mirror of
https://github.com/ruvnet/RuView
synced 2026-06-13 10:53:20 +00:00
Compare commits
157 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 85417b84a6 | |||
| 4fc491dea5 | |||
| 4f6780f884 | |||
| 085af0c2be | |||
| f4e636aaa2 | |||
| 582d51aed6 | |||
| b31efe5e92 | |||
| f03b484dd1 | |||
| 7a75277d58 | |||
| 73ce72d39c | |||
| 4e9e92d713 | |||
| 28368b2c70 | |||
| 4bb8c3303f | |||
| b9778c5ad2 | |||
| b6c032d665 | |||
| 9d70d621da | |||
| b4c9e7743f | |||
| 8f2de7e9f2 | |||
| 74c965f7ec | |||
| 73d4cb9fc2 | |||
| ba82fcfc37 | |||
| ccc543c0e7 | |||
| ade0fe82f6 | |||
| a73a17e264 | |||
| c63cf2ee77 | |||
| 9a2bc1839a | |||
| 77a2e7e4e9 | |||
| b46b789e9e | |||
| 6464023780 | |||
| 7b12b36889 | |||
| 27d17431c5 | |||
| a4bd2308b7 | |||
| 3733e54aef | |||
| cd84c35f8f | |||
| dd45160cc5 | |||
| 5e5781b28a | |||
| 6f23e89909 | |||
| 1dcf5d42eb | |||
| 9814d2bc62 | |||
| 7f02c87c6f | |||
| 9a074bdf4f | |||
| 3c02f6cfb0 | |||
| 23dedecf0c | |||
| c2e564a9f4 | |||
| 40f19622af | |||
| 022499b2f5 | |||
| e6068c5efe | |||
| 7a13877fa3 | |||
| 6c98c98920 | |||
| 5f3c90bf1c | |||
| 4713a30402 | |||
| 2b8a7cc458 | |||
| 8a84748a83 | |||
| 578d84c25e | |||
| 7eba8c7286 | |||
| a7d417837f | |||
| 4239dfa35a | |||
| 24ea88cbe0 | |||
| ef582b4429 | |||
| 8318f9c677 | |||
| 92a6986b79 | |||
| 66e2fa0835 | |||
| 7a97ffd8c7 | |||
| 2b3c3e4b45 | |||
| 024d2583f0 | |||
| 5b2aacd923 | |||
| 1d4af7c757 | |||
| 523be943b0 | |||
| a467dfed9f | |||
| d793c1f49f | |||
| 3457610c9f | |||
| e9d5ea3ad3 | |||
| 9cefb32815 | |||
| a7c74e0c57 | |||
| 98a2b0462c | |||
| e5e3d42ca2 | |||
| 7c1351fd5d | |||
| 6e03a47867 | |||
| 9d1140de2d | |||
| 952f27a1ce | |||
| f7d043d727 | |||
| ff91d4e8cf | |||
| fc92436f52 | |||
| 285bb0ad37 | |||
| b5ec4ef043 | |||
| 21aba2df8d | |||
| a28a875594 | |||
| e12749bf68 | |||
| 3b37aaf460 | |||
| d3c683cc7e | |||
| 56de77c0ad | |||
| 0b98917dff | |||
| da4255a54c | |||
| 26a7d6775a | |||
| 341d9e05a8 | |||
| bc5408bd80 | |||
| c82c4fc4ac | |||
| 0c85d9c86f | |||
| 65c6fa7a34 | |||
| 7659b0bbe2 | |||
| 75d4685d25 | |||
| 45c15b77a5 | |||
| 47223a98be | |||
| c45690ed4e | |||
| fb782e0d71 | |||
| 944076733e | |||
| a8f48a7897 | |||
| 7df316f13e | |||
| da54ea07d2 | |||
| bf4d64ad4b | |||
| 8b57a6f64c | |||
| 5fa61ba7ea | |||
| f771cf8461 | |||
| c257e9a215 | |||
| 6e76578dcf | |||
| c6f061a191 | |||
| 57141ff707 | |||
| b995adea87 | |||
| 6fea56c4a9 | |||
| d7a55fd646 | |||
| dc371a6751 | |||
| da7105d599 | |||
| 749007d708 | |||
| 26655d397e | |||
| aca1bbc82e | |||
| 2ad510782e | |||
| 8658cc3de0 | |||
| 2e9b34ec9a | |||
| 3eb8444f73 | |||
| cd7b914580 | |||
| 6d799c2917 | |||
| d00b733c99 | |||
| 90b5beb1d4 | |||
| b5af3bc528 | |||
| 7e43edf26a | |||
| a7fe8b6799 | |||
| c2e6546159 | |||
| f953a309fe | |||
| f995f69622 | |||
| ce171696b2 | |||
| b544545cb0 | |||
| b6f7b8a74a | |||
| 86f08303e6 | |||
| d4fb7d30d3 | |||
| 977da0f28e | |||
| 29b3e0a6fa | |||
| 3b74798ba6 | |||
| f1337ff1a2 | |||
| e94c7056f2 | |||
| d63d4d95d1 | |||
| 0c9b73a309 | |||
| 4b1005524e | |||
| 407b46b206 | |||
| 14902e6b4e | |||
| 086b0e690f | |||
| e0fe10b3dc | |||
| 915943cef4 |
@@ -0,0 +1 @@
|
||||
{"intelligence":7,"timestamp":1774922079152}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"running": true,
|
||||
"startedAt": "2026-02-28T15:54:19.353Z",
|
||||
"startedAt": "2026-03-09T15:26:00.921Z",
|
||||
"workers": {
|
||||
"map": {
|
||||
"runCount": 49,
|
||||
@@ -8,16 +8,16 @@
|
||||
"failureCount": 0,
|
||||
"averageDurationMs": 1.2857142857142858,
|
||||
"lastRun": "2026-02-28T16:13:19.194Z",
|
||||
"nextRun": "2026-02-28T16:28:19.195Z",
|
||||
"nextRun": "2026-03-09T15:56:00.928Z",
|
||||
"isRunning": false
|
||||
},
|
||||
"audit": {
|
||||
"runCount": 44,
|
||||
"runCount": 45,
|
||||
"successCount": 0,
|
||||
"failureCount": 44,
|
||||
"failureCount": 45,
|
||||
"averageDurationMs": 0,
|
||||
"lastRun": "2026-02-28T16:20:19.184Z",
|
||||
"nextRun": "2026-02-28T16:30:19.185Z",
|
||||
"lastRun": "2026-03-09T15:43:00.933Z",
|
||||
"nextRun": "2026-03-09T15:38:00.914Z",
|
||||
"isRunning": false
|
||||
},
|
||||
"optimize": {
|
||||
@@ -26,7 +26,7 @@
|
||||
"failureCount": 34,
|
||||
"averageDurationMs": 0,
|
||||
"lastRun": "2026-02-28T16:23:19.387Z",
|
||||
"nextRun": "2026-02-28T16:18:19.361Z",
|
||||
"nextRun": "2026-03-09T15:45:00.915Z",
|
||||
"isRunning": false
|
||||
},
|
||||
"consolidate": {
|
||||
@@ -35,7 +35,7 @@
|
||||
"failureCount": 0,
|
||||
"averageDurationMs": 0.6521739130434783,
|
||||
"lastRun": "2026-02-28T16:05:19.091Z",
|
||||
"nextRun": "2026-02-28T16:35:19.054Z",
|
||||
"nextRun": "2026-03-09T16:02:00.918Z",
|
||||
"isRunning": false
|
||||
},
|
||||
"testgaps": {
|
||||
@@ -44,8 +44,8 @@
|
||||
"failureCount": 27,
|
||||
"averageDurationMs": 0,
|
||||
"lastRun": "2026-02-28T16:08:19.369Z",
|
||||
"nextRun": "2026-02-28T16:22:19.355Z",
|
||||
"isRunning": true
|
||||
"nextRun": "2026-03-09T15:54:00.920Z",
|
||||
"isRunning": false
|
||||
},
|
||||
"predict": {
|
||||
"runCount": 0,
|
||||
@@ -64,8 +64,8 @@
|
||||
},
|
||||
"config": {
|
||||
"autoStart": false,
|
||||
"logDir": "/home/user/wifi-densepose/.claude-flow/logs",
|
||||
"stateFile": "/home/user/wifi-densepose/.claude-flow/daemon-state.json",
|
||||
"logDir": "/Users/cohen/GitHub/ruvnet/RuView/.claude-flow/logs",
|
||||
"stateFile": "/Users/cohen/GitHub/ruvnet/RuView/.claude-flow/daemon-state.json",
|
||||
"maxConcurrent": 2,
|
||||
"workerTimeoutMs": 300000,
|
||||
"resourceThresholds": {
|
||||
@@ -131,5 +131,5 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"savedAt": "2026-02-28T16:23:19.387Z"
|
||||
"savedAt": "2026-03-09T15:43:00.933Z"
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
166
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"timestamp": "2026-03-06T13:17:27.368Z",
|
||||
"mode": "local",
|
||||
"checks": {
|
||||
"envFilesProtected": true,
|
||||
"gitIgnoreExists": true,
|
||||
"noHardcodedSecrets": true
|
||||
},
|
||||
"riskLevel": "low",
|
||||
"recommendations": [],
|
||||
"note": "Install Claude Code CLI for AI-powered security analysis"
|
||||
}
|
||||
+13
-13
@@ -6,7 +6,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs pre-bash",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" pre-bash",
|
||||
"timeout": 5000
|
||||
}
|
||||
]
|
||||
@@ -18,7 +18,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs post-edit",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" post-edit",
|
||||
"timeout": 10000
|
||||
}
|
||||
]
|
||||
@@ -29,7 +29,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs route",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" route",
|
||||
"timeout": 10000
|
||||
}
|
||||
]
|
||||
@@ -40,12 +40,12 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs session-restore",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" session-restore",
|
||||
"timeout": 15000
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/auto-memory-hook.mjs import",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/auto-memory-hook.mjs\" import",
|
||||
"timeout": 8000
|
||||
}
|
||||
]
|
||||
@@ -56,7 +56,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs session-end",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" session-end",
|
||||
"timeout": 10000
|
||||
}
|
||||
]
|
||||
@@ -67,7 +67,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/auto-memory-hook.mjs sync",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/auto-memory-hook.mjs\" sync",
|
||||
"timeout": 10000
|
||||
}
|
||||
]
|
||||
@@ -79,11 +79,11 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs compact-manual"
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" compact-manual"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs session-end",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" session-end",
|
||||
"timeout": 5000
|
||||
}
|
||||
]
|
||||
@@ -93,11 +93,11 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs compact-auto"
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" compact-auto"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs session-end",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" session-end",
|
||||
"timeout": 6000
|
||||
}
|
||||
]
|
||||
@@ -108,7 +108,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/hook-handler.cjs status",
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/hook-handler.cjs\" status",
|
||||
"timeout": 3000
|
||||
}
|
||||
]
|
||||
@@ -117,7 +117,7 @@
|
||||
},
|
||||
"statusLine": {
|
||||
"type": "command",
|
||||
"command": "node .claude/helpers/statusline.cjs"
|
||||
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/helpers/statusline.cjs\""
|
||||
},
|
||||
"permissions": {
|
||||
"allow": [
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"enabledMcpjsonServers": [
|
||||
"claude-flow"
|
||||
],
|
||||
"enableAllProjectMcpServers": true
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
name: Desktop Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'desktop-v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version to release (e.g., 0.4.0)'
|
||||
required: true
|
||||
default: '0.4.0'
|
||||
attach_to_existing:
|
||||
description: 'Attach to existing release tag (leave empty to create new)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
build-macos:
|
||||
name: Build macOS
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
target: [aarch64-apple-darwin, x86_64-apple-darwin]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop/ui
|
||||
run: npm ci
|
||||
|
||||
- name: Build frontend
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop/ui
|
||||
run: npm run build
|
||||
|
||||
- name: Install Tauri CLI
|
||||
run: cargo install tauri-cli --version "^2.0.0"
|
||||
|
||||
- name: Build Tauri app
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop
|
||||
run: cargo tauri build --target ${{ matrix.target }}
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
|
||||
- name: Get architecture name
|
||||
id: arch
|
||||
run: |
|
||||
if [ "${{ matrix.target }}" = "aarch64-apple-darwin" ]; then
|
||||
echo "arch=arm64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "arch=x64" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Package macOS app
|
||||
run: |
|
||||
cd rust-port/wifi-densepose-rs/target/${{ matrix.target }}/release/bundle/macos
|
||||
zip -r "RuView-Desktop-${{ github.event.inputs.version || '0.4.0' }}-macos-${{ steps.arch.outputs.arch }}.zip" "RuView Desktop.app"
|
||||
|
||||
- name: Upload macOS artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ruview-macos-${{ steps.arch.outputs.arch }}
|
||||
path: rust-port/wifi-densepose-rs/target/${{ matrix.target }}/release/bundle/macos/*.zip
|
||||
|
||||
build-windows:
|
||||
name: Build Windows
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop/ui
|
||||
run: npm ci
|
||||
|
||||
- name: Build frontend
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop/ui
|
||||
run: npm run build
|
||||
|
||||
- name: Install Tauri CLI
|
||||
run: cargo install tauri-cli --version "^2.0.0"
|
||||
|
||||
- name: Build Tauri app
|
||||
working-directory: rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop
|
||||
run: cargo tauri build
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
|
||||
- name: Upload Windows MSI artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ruview-windows-msi
|
||||
path: rust-port/wifi-densepose-rs/target/release/bundle/msi/*.msi
|
||||
|
||||
- name: Upload Windows NSIS artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ruview-windows-nsis
|
||||
path: rust-port/wifi-densepose-rs/target/release/bundle/nsis/*.exe
|
||||
|
||||
create-release:
|
||||
name: Create Release
|
||||
needs: [build-macos, build-windows]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: List artifacts
|
||||
run: find artifacts -type f
|
||||
|
||||
- name: Create or Update Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
name: RuView Desktop v${{ github.event.inputs.version || '0.4.0' }}
|
||||
tag_name: ${{ github.event.inputs.attach_to_existing || format('desktop-v{0}', github.event.inputs.version || '0.4.0') }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
generate_release_notes: ${{ github.event.inputs.attach_to_existing == '' }}
|
||||
files: |
|
||||
artifacts/**/*.zip
|
||||
artifacts/**/*.msi
|
||||
artifacts/**/*.exe
|
||||
artifacts/**/*.dmg
|
||||
body: |
|
||||
## RuView Desktop v${{ github.event.inputs.version || '0.4.0' }}
|
||||
|
||||
WiFi-based human pose estimation desktop application.
|
||||
|
||||
### Downloads
|
||||
|
||||
| Platform | Architecture | Download |
|
||||
|----------|--------------|----------|
|
||||
| macOS | Apple Silicon (M1/M2/M3) | `RuView-Desktop-*-macos-arm64.zip` |
|
||||
| macOS | Intel | `RuView-Desktop-*-macos-x64.zip` |
|
||||
| Windows | x64 | `RuView-Desktop-*.msi` or `RuView-Desktop-*.exe` |
|
||||
|
||||
### Installation
|
||||
|
||||
**macOS:**
|
||||
1. Download the appropriate `.zip` file for your Mac
|
||||
2. Extract the zip file
|
||||
3. Move `RuView Desktop.app` to your Applications folder
|
||||
4. Right-click and select "Open" (first time only, to bypass Gatekeeper)
|
||||
|
||||
**Windows:**
|
||||
1. Download the `.msi` installer
|
||||
2. Run the installer
|
||||
3. Launch RuView Desktop from the Start menu
|
||||
|
||||
### Requirements
|
||||
- macOS 11.0+ (Big Sur or later)
|
||||
- Windows 10/11 (64-bit)
|
||||
@@ -0,0 +1,102 @@
|
||||
name: Firmware CI
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build ESP32-S3 Firmware
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: espressif/idf:v5.4
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build firmware
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
idf.py set-target esp32s3
|
||||
idf.py build
|
||||
|
||||
- name: Verify binary size (< 1100 KB gate)
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
BIN=build/esp32-csi-node.bin
|
||||
SIZE=$(stat -c%s "$BIN")
|
||||
MAX=$((1100 * 1024))
|
||||
echo "Binary size: $SIZE bytes ($(( SIZE / 1024 )) KB)"
|
||||
echo "Size limit: $MAX bytes (1100 KB — includes WASM runtime + HTTP client for Seed swarm bridge)"
|
||||
if [ "$SIZE" -gt "$MAX" ]; then
|
||||
echo "::error::Firmware binary exceeds 1100 KB size gate ($SIZE > $MAX)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Binary size OK: $SIZE <= $MAX"
|
||||
|
||||
- name: Verify flash image integrity
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
ERRORS=0
|
||||
BIN=build/esp32-csi-node.bin
|
||||
|
||||
# Check binary exists and is non-empty.
|
||||
if [ ! -s "$BIN" ]; then
|
||||
echo "::error::Binary not found or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check partition table magic (0xAA50 at offset 0).
|
||||
# Use od instead of xxd (xxd not available in espressif/idf container).
|
||||
PT=build/partition_table/partition-table.bin
|
||||
if [ -f "$PT" ]; then
|
||||
MAGIC=$(od -A n -t x1 -N 2 "$PT" | tr -d ' ')
|
||||
if [ "$MAGIC" != "aa50" ]; then
|
||||
echo "::warning::Partition table magic mismatch: $MAGIC (expected aa50)"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check bootloader exists.
|
||||
BL=build/bootloader/bootloader.bin
|
||||
if [ ! -s "$BL" ]; then
|
||||
echo "::warning::Bootloader binary missing or empty"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# Verify non-zero data in binary (not all 0xFF padding).
|
||||
NONZERO=$(od -A n -t x1 -N 1024 "$BIN" | tr -d ' f\n' | wc -c)
|
||||
if [ "$NONZERO" -lt 100 ]; then
|
||||
echo "::error::Binary appears to be mostly padding (non-zero chars: $NONZERO)"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
if [ "$ERRORS" -gt 0 ]; then
|
||||
echo "::warning::Flash image verification completed with $ERRORS warning(s)"
|
||||
else
|
||||
echo "Flash image integrity verified"
|
||||
fi
|
||||
|
||||
- name: Check QEMU ESP32-S3 support status
|
||||
run: |
|
||||
echo "::notice::ESP32-S3 QEMU support is experimental in ESP-IDF v5.4. "
|
||||
echo "Full smoke testing requires QEMU 8.2+ with xtensa-esp32s3 target."
|
||||
echo "See: https://github.com/espressif/qemu/wiki"
|
||||
|
||||
- name: Upload firmware artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: esp32-csi-node-firmware
|
||||
path: |
|
||||
firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
firmware/esp32-csi-node/build/bootloader/bootloader.bin
|
||||
firmware/esp32-csi-node/build/partition_table/partition-table.bin
|
||||
firmware/esp32-csi-node/build/ota_data_initial.bin
|
||||
retention-days: 90
|
||||
@@ -0,0 +1,370 @@
|
||||
name: Firmware QEMU Tests (ADR-061)
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- 'scripts/qemu-esp32s3-test.sh'
|
||||
- 'scripts/validate_qemu_output.py'
|
||||
- 'scripts/generate_nvs_matrix.py'
|
||||
- 'scripts/qemu_swarm.py'
|
||||
- 'scripts/swarm_health.py'
|
||||
- 'scripts/swarm_presets/**'
|
||||
- '.github/workflows/firmware-qemu.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- 'scripts/qemu-esp32s3-test.sh'
|
||||
- 'scripts/validate_qemu_output.py'
|
||||
- 'scripts/generate_nvs_matrix.py'
|
||||
- 'scripts/qemu_swarm.py'
|
||||
- 'scripts/swarm_health.py'
|
||||
- 'scripts/swarm_presets/**'
|
||||
- '.github/workflows/firmware-qemu.yml'
|
||||
|
||||
env:
|
||||
IDF_VERSION: "v5.4"
|
||||
QEMU_REPO: "https://github.com/espressif/qemu.git"
|
||||
QEMU_BRANCH: "esp-develop"
|
||||
|
||||
jobs:
|
||||
build-qemu:
|
||||
name: Build Espressif QEMU
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Cache QEMU build
|
||||
id: cache-qemu
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /opt/qemu-esp32
|
||||
# Include date component so cache refreshes monthly when branch updates
|
||||
key: qemu-esp32s3-${{ env.QEMU_BRANCH }}-v5
|
||||
restore-keys: |
|
||||
qemu-esp32s3-${{ env.QEMU_BRANCH }}-
|
||||
|
||||
- name: Install QEMU build dependencies
|
||||
if: steps.cache-qemu.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
git build-essential ninja-build pkg-config \
|
||||
libglib2.0-dev libpixman-1-dev libslirp-dev \
|
||||
libgcrypt20-dev \
|
||||
python3 python3-venv
|
||||
|
||||
- name: Clone and build Espressif QEMU
|
||||
if: steps.cache-qemu.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
git clone --depth 1 -b "$QEMU_BRANCH" "$QEMU_REPO" /tmp/qemu-esp
|
||||
cd /tmp/qemu-esp
|
||||
mkdir build && cd build
|
||||
../configure \
|
||||
--target-list=xtensa-softmmu \
|
||||
--prefix=/opt/qemu-esp32 \
|
||||
--enable-slirp \
|
||||
--disable-werror
|
||||
ninja -j$(nproc)
|
||||
ninja install
|
||||
|
||||
- name: Verify QEMU binary
|
||||
run: |
|
||||
file_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1" 2>/dev/null || wc -c < "$1"; }
|
||||
/opt/qemu-esp32/bin/qemu-system-xtensa --version
|
||||
echo "QEMU binary size: $(file_size /opt/qemu-esp32/bin/qemu-system-xtensa) bytes"
|
||||
|
||||
- name: Upload QEMU artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: qemu-esp32
|
||||
path: /opt/qemu-esp32/
|
||||
retention-days: 7
|
||||
|
||||
qemu-test:
|
||||
name: QEMU Test (${{ matrix.nvs_config }})
|
||||
needs: build-qemu
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: espressif/idf:v5.4
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
nvs_config:
|
||||
- default
|
||||
- full-adr060
|
||||
- edge-tier0
|
||||
- edge-tier1
|
||||
- tdm-3node
|
||||
- boundary-max
|
||||
- boundary-min
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: qemu-esp32
|
||||
path: /opt/qemu-esp32
|
||||
|
||||
- name: Make QEMU executable
|
||||
run: chmod +x /opt/qemu-esp32/bin/qemu-system-xtensa
|
||||
|
||||
- name: Verify QEMU works
|
||||
run: /opt/qemu-esp32/bin/qemu-system-xtensa --version
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
pip install esptool esp-idf-nvs-partition-gen
|
||||
|
||||
- name: Set target ESP32-S3
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
idf.py set-target esp32s3
|
||||
|
||||
- name: Build firmware (mock CSI mode)
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
idf.py \
|
||||
-D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" \
|
||||
build
|
||||
|
||||
- name: Generate NVS matrix
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
python3 scripts/generate_nvs_matrix.py \
|
||||
--output-dir firmware/esp32-csi-node/build/nvs_matrix \
|
||||
--only ${{ matrix.nvs_config }}
|
||||
|
||||
- name: Create merged flash image
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
|
||||
# Determine merge_bin arguments
|
||||
OTA_ARGS=""
|
||||
if [ -f build/ota_data_initial.bin ]; then
|
||||
OTA_ARGS="0xf000 build/ota_data_initial.bin"
|
||||
fi
|
||||
|
||||
python3 -m esptool --chip esp32s3 merge_bin \
|
||||
-o build/qemu_flash.bin \
|
||||
--flash_mode dio --flash_freq 80m --flash_size 8MB \
|
||||
--fill-flash-size 8MB \
|
||||
0x0 build/bootloader/bootloader.bin \
|
||||
0x8000 build/partition_table/partition-table.bin \
|
||||
$OTA_ARGS \
|
||||
0x20000 build/esp32-csi-node.bin
|
||||
|
||||
file_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1" 2>/dev/null || wc -c < "$1"; }
|
||||
echo "Flash image size: $(file_size build/qemu_flash.bin) bytes"
|
||||
|
||||
- name: Inject NVS partition
|
||||
if: matrix.nvs_config != 'default'
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
NVS_BIN="build/nvs_matrix/nvs_${{ matrix.nvs_config }}.bin"
|
||||
if [ -f "$NVS_BIN" ]; then
|
||||
file_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1" 2>/dev/null || wc -c < "$1"; }
|
||||
echo "Injecting NVS: $NVS_BIN ($(file_size "$NVS_BIN") bytes)"
|
||||
dd if="$NVS_BIN" of=build/qemu_flash.bin \
|
||||
bs=1 seek=$((0x9000)) conv=notrunc 2>/dev/null
|
||||
else
|
||||
echo "WARNING: NVS binary not found: $NVS_BIN"
|
||||
fi
|
||||
|
||||
- name: Run QEMU smoke test
|
||||
env:
|
||||
QEMU_PATH: /opt/qemu-esp32/bin/qemu-system-xtensa
|
||||
QEMU_TIMEOUT: "90"
|
||||
run: |
|
||||
echo "Starting QEMU (timeout: ${QEMU_TIMEOUT}s)..."
|
||||
|
||||
timeout "$QEMU_TIMEOUT" "$QEMU_PATH" \
|
||||
-machine esp32s3 \
|
||||
-nographic \
|
||||
-drive file=firmware/esp32-csi-node/build/qemu_flash.bin,if=mtd,format=raw \
|
||||
-serial mon:stdio \
|
||||
-nic user,model=open_eth,net=10.0.2.0/24 \
|
||||
-no-reboot \
|
||||
2>&1 | tee firmware/esp32-csi-node/build/qemu_output.log || true
|
||||
|
||||
echo "QEMU finished. Log size: $(wc -l < firmware/esp32-csi-node/build/qemu_output.log) lines"
|
||||
|
||||
- name: Validate QEMU output
|
||||
run: |
|
||||
python3 scripts/validate_qemu_output.py \
|
||||
firmware/esp32-csi-node/build/qemu_output.log
|
||||
|
||||
- name: Upload test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: qemu-logs-${{ matrix.nvs_config }}
|
||||
path: |
|
||||
firmware/esp32-csi-node/build/qemu_output.log
|
||||
firmware/esp32-csi-node/build/nvs_matrix/
|
||||
retention-days: 14
|
||||
|
||||
fuzz-test:
|
||||
name: Fuzz Testing (ADR-061 Layer 6)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install clang
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang
|
||||
|
||||
- name: Build fuzz targets
|
||||
working-directory: firmware/esp32-csi-node/test
|
||||
run: make all CC=clang
|
||||
|
||||
- name: Run serialize fuzzer (60s)
|
||||
working-directory: firmware/esp32-csi-node/test
|
||||
run: make run_serialize FUZZ_DURATION=60 || echo "FUZZER_CRASH=serialize" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run edge enqueue fuzzer (60s)
|
||||
working-directory: firmware/esp32-csi-node/test
|
||||
run: make run_edge FUZZ_DURATION=60 || echo "FUZZER_CRASH=edge" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run NVS config fuzzer (60s)
|
||||
working-directory: firmware/esp32-csi-node/test
|
||||
run: make run_nvs FUZZ_DURATION=60 || echo "FUZZER_CRASH=nvs" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Check for crashes
|
||||
working-directory: firmware/esp32-csi-node/test
|
||||
run: |
|
||||
CRASHES=$(find . -type f \( -name "crash-*" -o -name "oom-*" -o -name "timeout-*" \) 2>/dev/null | wc -l)
|
||||
echo "Crash artifacts found: $CRASHES"
|
||||
if [ "$CRASHES" -gt 0 ] || [ -n "${FUZZER_CRASH:-}" ]; then
|
||||
echo "::error::Fuzzer found $CRASHES crash/oom/timeout artifacts. FUZZER_CRASH=${FUZZER_CRASH:-none}"
|
||||
ls -la crash-* oom-* timeout-* 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload fuzz artifacts
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fuzz-crashes
|
||||
path: |
|
||||
firmware/esp32-csi-node/test/crash-*
|
||||
firmware/esp32-csi-node/test/oom-*
|
||||
firmware/esp32-csi-node/test/timeout-*
|
||||
retention-days: 30
|
||||
|
||||
nvs-matrix-validate:
|
||||
name: NVS Matrix Generation
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install NVS generator
|
||||
run: pip install esp-idf-nvs-partition-gen
|
||||
|
||||
- name: Generate all 14 NVS configs
|
||||
run: |
|
||||
python3 scripts/generate_nvs_matrix.py \
|
||||
--output-dir build/nvs_matrix
|
||||
|
||||
- name: Verify all binaries generated
|
||||
run: |
|
||||
EXPECTED=14
|
||||
ACTUAL=$(find build/nvs_matrix -type f -name "nvs_*.bin" 2>/dev/null | wc -l)
|
||||
echo "Generated $ACTUAL / $EXPECTED NVS binaries"
|
||||
ls -la build/nvs_matrix/
|
||||
|
||||
if [ "$ACTUAL" -lt "$EXPECTED" ]; then
|
||||
echo "::error::Only $ACTUAL of $EXPECTED NVS binaries generated"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify binary sizes
|
||||
run: |
|
||||
file_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1" 2>/dev/null || wc -c < "$1"; }
|
||||
for f in build/nvs_matrix/nvs_*.bin; do
|
||||
SIZE=$(file_size "$f")
|
||||
if [ "$SIZE" -ne 24576 ]; then
|
||||
echo "::error::$f has unexpected size $SIZE (expected 24576)"
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: $(basename $f) ($SIZE bytes)"
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ADR-062: QEMU Swarm Configurator Test
|
||||
#
|
||||
# Runs a lightweight 3-node swarm (ci_matrix preset) under QEMU to validate
|
||||
# multi-node orchestration, TDM slot coordination, and swarm-level health
|
||||
# assertions. Uses the pre-built QEMU binary from the build-qemu job and the
|
||||
# firmware built by qemu-test.
|
||||
#
|
||||
# The CI runner is non-root, so TAP bridge networking is unavailable.
|
||||
# The orchestrator (qemu_swarm.py) detects this and falls back to SLIRP
|
||||
# user-mode networking, which is sufficient for the ci_matrix preset.
|
||||
# ---------------------------------------------------------------------------
|
||||
swarm-test:
|
||||
name: Swarm Test (ADR-062)
|
||||
needs: [build-qemu]
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: espressif/idf:v5.4
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: qemu-esp32
|
||||
path: /opt/qemu-esp32
|
||||
|
||||
- name: Make QEMU executable
|
||||
run: chmod +x /opt/qemu-esp32/bin/qemu-system-xtensa
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
pip install pyyaml esptool esp-idf-nvs-partition-gen
|
||||
|
||||
- name: Build firmware for swarm
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
idf.py set-target esp32s3
|
||||
idf.py -D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" build
|
||||
python3 -m esptool --chip esp32s3 merge_bin \
|
||||
-o build/qemu_flash.bin \
|
||||
--flash_mode dio --flash_freq 80m --flash_size 8MB \
|
||||
--fill-flash-size 8MB \
|
||||
0x0 build/bootloader/bootloader.bin \
|
||||
0x8000 build/partition_table/partition-table.bin \
|
||||
0x20000 build/esp32-csi-node.bin
|
||||
|
||||
- name: Run swarm smoke test
|
||||
run: |
|
||||
. $IDF_PATH/export.sh
|
||||
EXIT_CODE=0
|
||||
python3 scripts/qemu_swarm.py --preset ci_matrix \
|
||||
--qemu-path /opt/qemu-esp32/bin/qemu-system-xtensa \
|
||||
--output-dir build/swarm-results || EXIT_CODE=$?
|
||||
# Exit 0=PASS, 1=WARN (acceptable in CI without real hardware)
|
||||
if [ "$EXIT_CODE" -gt 1 ]; then
|
||||
echo "Swarm test failed with exit code $EXIT_CODE"
|
||||
exit "$EXIT_CODE"
|
||||
fi
|
||||
timeout-minutes: 10
|
||||
|
||||
- name: Upload swarm results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: swarm-results
|
||||
path: |
|
||||
build/swarm-results/
|
||||
retention-days: 14
|
||||
@@ -0,0 +1,50 @@
|
||||
name: Update vendor submodules
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # Every 6 hours
|
||||
workflow_dispatch: # Manual trigger
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Update submodules to latest main
|
||||
run: git submodule update --remote --merge
|
||||
|
||||
- name: Check for changes
|
||||
id: check
|
||||
run: |
|
||||
if git diff --quiet; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create PR with updates
|
||||
if: steps.check.outputs.changed == 'true'
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
BRANCH="chore/update-submodules-$(date +%Y%m%d-%H%M%S)"
|
||||
git checkout -b "$BRANCH"
|
||||
git add vendor/
|
||||
git commit -m "chore: update vendor submodules to latest main"
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update vendor submodules" \
|
||||
--body "Automated submodule update to latest upstream main." \
|
||||
--base main \
|
||||
--head "$BRANCH"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
+49
-1
@@ -1,8 +1,41 @@
|
||||
# Local Claude config (contains WiFi credentials and machine-specific paths)
|
||||
CLAUDE.local.md
|
||||
|
||||
# ESP32 firmware build artifacts and local config (contains WiFi credentials)
|
||||
firmware/esp32-csi-node/build/
|
||||
firmware/esp32-csi-node/sdkconfig
|
||||
firmware/esp32-csi-node/sdkconfig.defaults
|
||||
firmware/esp32-csi-node/sdkconfig.old
|
||||
# Downloaded WASM3 source (fetched at configure time)
|
||||
firmware/esp32-csi-node/components/wasm3/wasm3-src/
|
||||
# ESP-IDF managed components (downloaded at build time)
|
||||
firmware/esp32-csi-node/managed_components/
|
||||
firmware/esp32-csi-node/dependencies.lock
|
||||
firmware/esp32-csi-node/sdkconfig.defaults.bak
|
||||
|
||||
# Claude Flow swarm runtime state
|
||||
.swarm/
|
||||
|
||||
# CSI recordings (local training data, machine-specific)
|
||||
rust-port/wifi-densepose-rs/data/recordings/
|
||||
|
||||
# NVS partition images and CSVs (contain WiFi credentials)
|
||||
nvs.bin
|
||||
nvs_config.csv
|
||||
nvs_provision.bin
|
||||
firmware/esp32-csi-node/nvs_seed.csv
|
||||
firmware/esp32-csi-node/nvs_seed.bin
|
||||
firmware/esp32-csi-node/nvs_config.bin
|
||||
firmware/esp32-csi-node/nvs_wifi.bin
|
||||
firmware/esp32-csi-node/nvs.bin
|
||||
# Catch any other NVS binaries/CSVs with credentials
|
||||
**/nvs_*.bin
|
||||
**/nvs_*.csv
|
||||
|
||||
# Working artifacts that should not land in root
|
||||
/*.wasm
|
||||
/esp32_*.txt
|
||||
/serial_error.txt
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
@@ -201,4 +234,19 @@ v1/src/sensing/mac_wifi
|
||||
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||
# refer to https://docs.cursor.com/context/ignore-files
|
||||
.cursorignore
|
||||
.cursorindexingignore
|
||||
.cursorindexingignore
|
||||
|
||||
# Claude Flow runtime artifacts (auto-generated, machine-specific)
|
||||
**/daemon.pid
|
||||
**/pending-insights.jsonl
|
||||
**/vectors.db
|
||||
**/memory.db
|
||||
**/.claude-flow/sessions/session-*.json
|
||||
**/.claude-flow/sessions/current.json
|
||||
|
||||
# Node modules (should use npm ci, not committed)
|
||||
**/node_modules/
|
||||
|
||||
# Local build scripts
|
||||
firmware/esp32-csi-node/build_firmware.batdata/
|
||||
models/
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
[submodule "vendor/midstream"]
|
||||
path = vendor/midstream
|
||||
url = https://github.com/ruvnet/midstream
|
||||
branch = main
|
||||
[submodule "vendor/ruvector"]
|
||||
path = vendor/ruvector
|
||||
url = https://github.com/ruvnet/ruvector
|
||||
branch = main
|
||||
[submodule "vendor/sublinear-time-solver"]
|
||||
path = vendor/sublinear-time-solver
|
||||
url = https://github.com/ruvnet/sublinear-time-solver
|
||||
branch = main
|
||||
Binary file not shown.
Vendored
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "QEMU ESP32-S3 Debug",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/firmware/esp32-csi-node/build/esp32-csi-node.elf",
|
||||
"cwd": "${workspaceFolder}/firmware/esp32-csi-node",
|
||||
"MIMode": "gdb",
|
||||
"miDebuggerPath": "xtensa-esp-elf-gdb",
|
||||
"miDebuggerServerAddress": "localhost:1234",
|
||||
"setupCommands": [
|
||||
{
|
||||
"description": "Set remote hardware breakpoint limit (ESP32-S3 has 2)",
|
||||
"text": "set remote hardware-breakpoint-limit 2",
|
||||
"ignoreFailures": false
|
||||
},
|
||||
{
|
||||
"description": "Set remote hardware watchpoint limit (ESP32-S3 has 2)",
|
||||
"text": "set remote hardware-watchpoint-limit 2",
|
||||
"ignoreFailures": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "QEMU ESP32-S3 Debug (attach)",
|
||||
"type": "cppdbg",
|
||||
"request": "attach",
|
||||
"program": "${workspaceFolder}/firmware/esp32-csi-node/build/esp32-csi-node.elf",
|
||||
"cwd": "${workspaceFolder}/firmware/esp32-csi-node",
|
||||
"MIMode": "gdb",
|
||||
"miDebuggerPath": "xtensa-esp-elf-gdb",
|
||||
"miDebuggerServerAddress": "localhost:1234",
|
||||
"setupCommands": [
|
||||
{
|
||||
"description": "Set remote hardware breakpoint limit (ESP32-S3 has 2)",
|
||||
"text": "set remote hardware-breakpoint-limit 2",
|
||||
"ignoreFailures": false
|
||||
},
|
||||
{
|
||||
"description": "Set remote hardware watchpoint limit (ESP32-S3 has 2)",
|
||||
"text": "set remote hardware-watchpoint-limit 2",
|
||||
"ignoreFailures": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+146
@@ -5,9 +5,150 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [v0.5.4-esp32] — 2026-04-02
|
||||
|
||||
### Added
|
||||
- **ADR-069: ESP32 CSI → Cognitum Seed RVF ingest pipeline** — Live-validated pipeline connecting ESP32-S3 CSI sensing to Cognitum Seed (Pi Zero 2 W) edge intelligence appliance. 339 vectors ingested, 100% kNN validation, SHA-256 witness chain verified.
|
||||
- **Feature vector packet (magic 0xC5110003)** — New 48-byte packet with 8 normalized dimensions (presence, motion, breathing, heart rate, phase variance, person count, fall, RSSI) sent at 1 Hz alongside vitals.
|
||||
- **`scripts/seed_csi_bridge.py`** — Python bridge: UDP listener → HTTPS ingest with bearer token auth, `--validate` (kNN + PIR ground truth), `--stats`, `--compact` modes, hash-based vector IDs, NaN/inf rejection, source IP filtering, retry logic.
|
||||
- **Arena Physica research** — 26 research documents in `docs/research/` covering Maxwell's equations in WiFi sensing, Arena Physica Studio analysis, SOTA WiFi sensing 2025-2026, GOAP implementation plan for ESP32 + Pi Zero.
|
||||
- **Cognitum Seed MCP integration** — 114-tool MCP proxy enables AI assistants to query sensing state, vectors, witness chain, and device status directly.
|
||||
|
||||
### Fixed
|
||||
- **Compressed frame magic collision** — Reassigned compressed frame magic from `0xC5110003` to `0xC5110005` to free `0xC5110003` for feature vectors.
|
||||
- **Uninitialized `s_top_k[0]` read** — Guarded variance computation against `s_top_k_count == 0` in `send_feature_vector()`.
|
||||
- **Presence score normalization** — Bridge now divides by 15.0 instead of clamping, preserving dynamic range for raw values 1.41-14.92.
|
||||
- **Stale magic references** — Updated ADR-039, DDD model to reflect `0xC5110005` for compressed frames.
|
||||
|
||||
### Security
|
||||
- **Credential exposure remediation** — Removed hardcoded WiFi passwords and bearer tokens from source files. Added NVS binary/CSV patterns to `.gitignore`. Environment variable fallback for bearer token.
|
||||
- **NaN/Inf injection prevention** — Bridge validates all feature dimensions are finite before Seed ingest.
|
||||
- **UDP source filtering** — `--allowed-sources` argument restricts packet acceptance to known ESP32 IPs.
|
||||
|
||||
### Changed
|
||||
- Wire format table now includes 6 magic numbers: `0xC5110001` (raw), `0xC5110002` (vitals), `0xC5110003` (features), `0xC5110004` (WASM events), `0xC5110005` (compressed), `0xC5110006` (fused vitals).
|
||||
|
||||
## [v0.5.3-esp32] — 2026-03-30
|
||||
|
||||
### Added
|
||||
- **Cross-node RSSI-weighted feature fusion** — Multiple ESP32 nodes fuse CSI features using RSSI-based weighting. Closer node gets higher weight. Reduces variance noise by 29%, keypoint jitter by 72%.
|
||||
- **DynamicMinCut person separation** — Uses `ruvector_mincut::DynamicMinCut` on the subcarrier temporal correlation graph to detect independent motion clusters. Replaces variance-based heuristic for multi-person counting.
|
||||
- **RSSI-based position tracking** — Skeleton position driven by RSSI differential between nodes. Walk between ESP32s and the skeleton follows you.
|
||||
- **Per-node state pipeline (ADR-068)** — Each ESP32 node gets independent `HashMap<u8, NodeState>` with frame history, classification, vitals, and person count. Fixes #249 (the #1 user-reported issue).
|
||||
- **RuVector Phase 1-3 integration** — Subcarrier importance weighting, temporal keypoint smoothing (EMA), coherence gating, skeleton kinematic constraints (Jakobsen relaxation), compressed pose history.
|
||||
- **Client-side lerp smoothing** — UI keypoints interpolate between frames (alpha=0.15) for fluid skeleton movement.
|
||||
- **Multi-node mesh tests** — 8 integration tests covering 1-255 node configurations.
|
||||
- **`wifi_densepose` Python package** — `from wifi_densepose import WiFiDensePose` now works (#314).
|
||||
|
||||
### Fixed
|
||||
- **Watchdog crash on busy LANs (#321)** — Batch-limited edge_dsp to 4 frames before 20ms yield. Fixed idle-path busy-spin (`pdMS_TO_TICKS(5)==0`).
|
||||
- **No detection from edge vitals (#323)** — Server now generates `sensing_update` from Tier 2+ vitals packets.
|
||||
- **RSSI byte offset mismatch (#332)** — Server parsed RSSI from wrong byte (was reading sequence counter).
|
||||
- **Stack overflow risk** — Moved 4KB of BPM scratch buffers from stack to static storage.
|
||||
- **Stale node memory leak** — `node_states` HashMap evicts nodes inactive >60s.
|
||||
- **Unsafe raw pointer removed** — Replaced with safe `.clone()` for adaptive model borrow.
|
||||
- **Firmware CI** — Upgraded to IDF v5.4, replaced `xxd` with `od` (#327).
|
||||
- **Person count double-counting** — Multi-node aggregation changed from `sum` to `max`.
|
||||
- **Skeleton jitter** — Removed tick-based noise, dampened procedural animation, recalibrated feature scaling for real ESP32 data.
|
||||
|
||||
### Changed
|
||||
- Motion-responsive skeleton: arm swing (0-80px) driven by CSI variance, leg kick (0-50px) by motion_band_power, vertical bob when walking.
|
||||
- Person count thresholds recalibrated for real ESP32 hardware (1→2 at 0.70, EMA alpha 0.04).
|
||||
- Vital sign filtering: larger median window (31), faster EMA (0.05), looser HR jump filter (15 BPM).
|
||||
- Vendored ruvector updated to v2.1.0-40 (316 commits ahead).
|
||||
|
||||
### Benchmarks (2-node mesh, COM6 + COM9, 30s)
|
||||
| Metric | Baseline | v0.5.3 | Improvement |
|
||||
|--------|----------|--------|-------------|
|
||||
| Variance noise | 109.4 | 77.6 | **-29%** |
|
||||
| Feature stability | std=154.1 | std=105.4 | **-32%** |
|
||||
| Keypoint jitter | std=4.5px | std=1.3px | **-72%** |
|
||||
| Confidence | 0.643 | 0.686 | **+7%** |
|
||||
| Presence accuracy | 93.4% | 94.6% | **+1.3pp** |
|
||||
|
||||
### Verified
|
||||
- Real hardware: COM6 (node 1) + COM9 (node 2) on ruv.net WiFi
|
||||
- All 284 Rust tests pass, 352 signal crate tests pass
|
||||
- Firmware builds clean at 843 KB
|
||||
- QEMU CI: 11/11 jobs green
|
||||
|
||||
## [v0.5.2-esp32] — 2026-03-28
|
||||
|
||||
### Fixed
|
||||
- RSSI byte offset in frame parser (#332)
|
||||
- Per-node state pipeline for multi-node sensing (#249)
|
||||
- Firmware CI upgraded to IDF v5.4 (#327)
|
||||
|
||||
## [v0.5.1-esp32] — 2026-03-27
|
||||
|
||||
### Fixed
|
||||
- Watchdog crash on busy LANs (#321)
|
||||
- No detection from edge vitals (#323)
|
||||
- `wifi_densepose` Python package import (#314)
|
||||
- Pre-compiled firmware binaries added to release
|
||||
|
||||
## [v0.5.0-esp32] — 2026-03-15
|
||||
|
||||
### Added
|
||||
- **60 GHz mmWave sensor fusion (ADR-063)** — Auto-detects Seeed MR60BHA2 (60 GHz, HR/BR/presence) and HLK-LD2410 (24 GHz, presence/distance) on UART at boot. Probes 115200 then 256000 baud, registers device capabilities, starts background parser.
|
||||
- **48-byte fused vitals packet** (magic `0xC5110004`) — Kalman-style fusion: mmWave 80% + CSI 20% when both available. Automatic fallback to standard 32-byte CSI-only packet.
|
||||
- **Server-side fusion bridge** (`scripts/mmwave_fusion_bridge.py`) — Reads two serial ports simultaneously for dual-sensor setups where mmWave runs on a separate ESP32.
|
||||
- **Multimodal ambient intelligence roadmap (ADR-064)** — 25+ applications from fall detection to sleep monitoring to RF tomography.
|
||||
|
||||
### Verified
|
||||
- Real hardware: ESP32-S3 (COM7) WiFi CSI + ESP32-C6/MR60BHA2 (COM4) 60 GHz mmWave running concurrently. HR=75 bpm, BR=25/min at 52 cm range. All 11 QEMU CI jobs green.
|
||||
|
||||
## [v0.4.3-esp32] — 2026-03-15
|
||||
|
||||
### Fixed
|
||||
- **Fall detection false positives (#263)** — Default threshold raised from 2.0 to 15.0 rad/s²; normal walking (2-5 rad/s²) no longer triggers alerts. Added 3-consecutive-frame debounce and 5-second cooldown between alerts. Verified on real ESP32-S3 hardware: 0 false alerts in 60s / 1,300+ live WiFi CSI frames.
|
||||
- **Kconfig default mismatch** — `CONFIG_EDGE_FALL_THRESH` Kconfig default was still 2000 (=2.0) while `nvs_config.c` fallback was updated to 15.0. Fixed Kconfig to 15000. Caught by real hardware testing — mock data did not reproduce.
|
||||
- **provision.py NVS generator API change** — `esp_idf_nvs_partition_gen` package changed its `generate()` signature; switched to subprocess-first invocation for cross-version compatibility.
|
||||
- **QEMU CI pipeline (11 jobs)** — Fixed all failures: fuzz test `esp_timer` stubs, QEMU `libgcrypt` dependency, NVS matrix generator, IDF container `pip` path, flash image padding, validation WARN handling, swarm `ip`/`cargo` missing.
|
||||
|
||||
### Added
|
||||
- **4MB flash support (#265)** — `partitions_4mb.csv` and `sdkconfig.defaults.4mb` for ESP32-S3 boards with 4MB flash (e.g. SuperMini). Dual OTA slots, 1.856 MB each. Thanks to @sebbu for the community workaround that confirmed feasibility.
|
||||
- **`--strict` flag** for `validate_qemu_output.py` — WARNs now pass by default in CI (no real WiFi in QEMU); use `--strict` to fail on warnings.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- **QEMU ESP32-S3 testing platform (ADR-061)** — 9-layer firmware testing without hardware
|
||||
- Mock CSI generator with 10 physics-based scenarios (empty room, walking, fall, multi-person, etc.)
|
||||
- Single-node QEMU runner with 16-check UART validation
|
||||
- Multi-node TDM mesh simulation (TAP networking, 2-6 nodes)
|
||||
- GDB remote debugging with VS Code integration
|
||||
- Code coverage via gcov/lcov + apptrace
|
||||
- Fuzz testing (3 libFuzzer targets + ASAN/UBSAN)
|
||||
- NVS provisioning matrix (14 configs)
|
||||
- Snapshot-based regression testing (sub-second VM restore)
|
||||
- Chaos testing with fault injection + health monitoring
|
||||
- **QEMU Swarm Configurator (ADR-062)** — YAML-driven multi-ESP32 test orchestration
|
||||
- 4 topologies: star, mesh, line, ring
|
||||
- 3 node roles: sensor, coordinator, gateway
|
||||
- 9 swarm-level assertions (boot, crashes, TDM, frame rate, fall detection, etc.)
|
||||
- 7 presets: smoke (2n/15s), standard (3n/60s), ci-matrix, large-mesh, line-relay, ring-fault, heterogeneous
|
||||
- Health oracle with cross-node validation
|
||||
- **QEMU installer** (`install-qemu.sh`) — auto-detects OS, installs deps, builds Espressif QEMU fork
|
||||
- **Unified QEMU CLI** (`qemu-cli.sh`) — single entry point for all 11 QEMU test commands
|
||||
- CI: `firmware-qemu.yml` workflow with QEMU test matrix, fuzz testing, NVS validation, and swarm test jobs
|
||||
- User guide: QEMU testing and swarm configurator section with plain-language walkthrough
|
||||
|
||||
### Fixed
|
||||
- Firmware now boots in QEMU: WiFi/UDP/OTA/display guards for mock CSI mode
|
||||
- 9 bugs in mock_csi.c (LFSR bias, MAC filter init, scenario loop, overflow burst timing)
|
||||
- 23 bugs from ADR-061 deep review (inject_fault.py writes, CI cache, snapshot log corruption, etc.)
|
||||
- 16 bugs from ADR-062 deep review (log filename mismatch, SLIRP port collision, heap false positives, etc.)
|
||||
- All scripts: `--help` flags, prerequisite checks with install hints, standardized exit codes
|
||||
|
||||
- **Sensing server UI API completion (ADR-043)** — 14 fully-functional REST endpoints for model management, CSI recording, and training control
|
||||
- Model CRUD: `GET /api/v1/models`, `GET /api/v1/models/active`, `POST /api/v1/models/load`, `POST /api/v1/models/unload`, `DELETE /api/v1/models/:id`, `GET /api/v1/models/lora/profiles`, `POST /api/v1/models/lora/activate`
|
||||
- CSI recording: `GET /api/v1/recording/list`, `POST /api/v1/recording/start`, `POST /api/v1/recording/stop`, `DELETE /api/v1/recording/:id`
|
||||
- Training control: `GET /api/v1/train/status`, `POST /api/v1/train/start`, `POST /api/v1/train/stop`
|
||||
- Recording writes CSI frames to `.jsonl` files via tokio background task
|
||||
- Model/recording directories scanned at startup, state managed via `Arc<RwLock<AppStateInner>>`
|
||||
- **ADR-044: Provisioning tool enhancements** — 5-phase plan for complete NVS coverage (7 missing keys), JSON config files, mesh presets, read-back/verify, and auto-detect
|
||||
- **25 real mobile tests** replacing `it.todo()` placeholders — 205 assertions covering components, services, stores, hooks, screens, and utils
|
||||
- **Project MERIDIAN (ADR-027)** — Cross-environment domain generalization for WiFi pose estimation (1,858 lines, 72 tests)
|
||||
- `HardwareNormalizer` — Catmull-Rom cubic interpolation resamples any hardware CSI to canonical 56 subcarriers; z-score + phase sanitization
|
||||
- `DomainFactorizer` + `GradientReversalLayer` — adversarial disentanglement of pose-relevant vs environment-specific features
|
||||
@@ -23,6 +164,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- ADR-025: macOS CoreWLAN WiFi Sensing (ORCA)
|
||||
|
||||
### Fixed
|
||||
- **sendto ENOMEM crash (Issue #127)** — CSI callbacks in promiscuous mode exhaust lwIP pbuf pool causing guru meditation crash. Fixed with 50 Hz rate limiter in `csi_collector.c` and 100 ms ENOMEM backoff in `stream_sender.c`. Hardware-verified on ESP32-S3 (200+ callbacks, zero crashes)
|
||||
- **Provisioning script missing TDM/edge flags (Issue #130)** — Added `--tdm-slot`, `--tdm-total`, `--edge-tier`, `--pres-thresh`, `--fall-thresh`, `--vital-win`, `--vital-int`, `--subk-count` to `provision.py`
|
||||
- **WebSocket "RECONNECTING" on Dashboard/Live Demo** — `sensingService.start()` now called on app init in `app.js` so WebSocket connects immediately instead of waiting for Sensing tab visit
|
||||
- **Mobile WebSocket port** — `ws.service.ts` `buildWsUrl()` uses same-origin port instead of hardcoded port 3001
|
||||
- **Mobile Jest config** — `testPathIgnorePatterns` no longer silently ignores the entire test directory
|
||||
- Removed synthetic byte counters from Python `MacosWifiCollector` — now reports `tx_bytes=0, rx_bytes=0` instead of fake incrementing values
|
||||
|
||||
---
|
||||
|
||||
@@ -57,7 +57,7 @@ All 5 ruvector crates integrated in workspace:
|
||||
- `ruvector-attention` → `model.rs` (apply_spatial_attention) + `bvp.rs`
|
||||
|
||||
### Architecture Decisions
|
||||
32 ADRs in `docs/adr/` (ADR-001 through ADR-032). Key ones:
|
||||
43 ADRs in `docs/adr/` (ADR-001 through ADR-043). Key ones:
|
||||
- ADR-014: SOTA signal processing (Accepted)
|
||||
- ADR-015: MM-Fi + Wi-Pose training datasets (Accepted)
|
||||
- ADR-016: RuVector training pipeline integration (Accepted — complete)
|
||||
@@ -70,6 +70,17 @@ All 5 ruvector crates integrated in workspace:
|
||||
- ADR-031: RuView sensing-first RF mode (Proposed)
|
||||
- ADR-032: Multistatic mesh security hardening (Proposed)
|
||||
|
||||
### Supported Hardware
|
||||
|
||||
| Device | Port | Chip | Role | Cost |
|
||||
|--------|------|------|------|------|
|
||||
| ESP32-S3 (8MB flash) | COM7 | Xtensa dual-core | WiFi CSI sensing node | ~$9 |
|
||||
| ESP32-S3 SuperMini (4MB) | — | Xtensa dual-core | WiFi CSI (compact) | ~$6 |
|
||||
| ESP32-C6 + Seeed MR60BHA2 | COM4 | RISC-V + 60 GHz FMCW | mmWave HR/BR/presence | ~$15 |
|
||||
| HLK-LD2410 | — | 24 GHz FMCW | Presence + distance | ~$3 |
|
||||
|
||||
**Not supported:** ESP32 (original), ESP32-C3 — single-core, can't run CSI DSP pipeline.
|
||||
|
||||
### Build & Test Commands (this repo)
|
||||
```bash
|
||||
# Rust — full workspace tests (1,031+ tests, ~2 min)
|
||||
@@ -79,11 +90,6 @@ cargo test --workspace --no-default-features
|
||||
# Rust — single crate check (no GPU needed)
|
||||
cargo check -p wifi-densepose-train --no-default-features
|
||||
|
||||
# Rust — publish crates (dependency order)
|
||||
cargo publish -p wifi-densepose-core --no-default-features
|
||||
cargo publish -p wifi-densepose-signal --no-default-features
|
||||
# ... see crate publishing order below
|
||||
|
||||
# Python — deterministic proof verification (SHA-256)
|
||||
python v1/data/proof/verify.py
|
||||
|
||||
@@ -91,6 +97,36 @@ python v1/data/proof/verify.py
|
||||
cd v1 && python -m pytest tests/ -x -q
|
||||
```
|
||||
|
||||
### ESP32 Firmware Build (Windows — Python subprocess required)
|
||||
```bash
|
||||
# Build 8MB firmware (real WiFi CSI mode, no mocks)
|
||||
# See CLAUDE.local.md for the full Python subprocess command
|
||||
# Key: must strip MSYSTEM env vars for ESP-IDF v5.4 on Git Bash
|
||||
|
||||
# Build 4MB firmware
|
||||
cp sdkconfig.defaults.4mb sdkconfig.defaults
|
||||
# then same build process
|
||||
|
||||
# Flash to COM7
|
||||
# [python, idf_py, '-p', 'COM7', 'flash']
|
||||
|
||||
# Provision WiFi
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourWiFi" --password "secret" --target-ip 192.168.1.20
|
||||
|
||||
# Monitor serial
|
||||
python -m serial.tools.miniterm COM7 115200
|
||||
```
|
||||
|
||||
### Firmware Release Process
|
||||
1. Build 8MB from `sdkconfig.defaults.template` (no mock)
|
||||
2. Build 4MB from `sdkconfig.defaults.4mb` (no mock)
|
||||
3. Save 6 binaries: `esp32-csi-node.bin`, `bootloader.bin`, `partition-table.bin`, `ota_data_initial.bin`, `esp32-csi-node-4mb.bin`, `partition-table-4mb.bin`
|
||||
4. Tag: `git tag v0.X.Y-esp32 && git push origin v0.X.Y-esp32`
|
||||
5. Release: `gh release create v0.X.Y-esp32 <binaries> --title "..." --notes-file ...`
|
||||
6. Verify on real hardware (COM7) before publishing
|
||||
7. **CRITICAL:** Always test with real WiFi CSI, not mock mode — mock missed the Kconfig threshold bug
|
||||
|
||||
### Crate Publishing Order
|
||||
Crates must be published in dependency order:
|
||||
1. `wifi-densepose-core` (no internal deps)
|
||||
@@ -173,7 +209,7 @@ Active feature branch: `ruvsense-full-implementation` (PR #77)
|
||||
## File Organization
|
||||
|
||||
- NEVER save to root folder — use the directories below
|
||||
- `docs/adr/` — Architecture Decision Records (32 ADRs)
|
||||
- `docs/adr/` — Architecture Decision Records (43 ADRs)
|
||||
- `docs/ddd/` — Domain-Driven Design models
|
||||
- `rust-port/wifi-densepose-rs/crates/` — Rust workspace crates (15 crates)
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-signal/src/ruvsense/` — RuvSense multistatic modules (14 files)
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 157 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 203 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.0 MiB |
File diff suppressed because one or more lines are too long
@@ -0,0 +1,141 @@
|
||||
## Introduction
|
||||
|
||||
RuView is a WiFi-based human pose estimation system built on ESP32 CSI (Channel State Information). Today, managing a RuView deployment requires juggling **6+ disconnected CLI tools**: `esptool.py` for flashing, `provision.py` for NVS configuration, `curl` for OTA and WASM management, `cargo run` for the sensing server, a browser for visualization, and manual IP tracking for node discovery. There is no single tool that provides a unified view of the entire deployment — from ESP32 hardware through the sensing pipeline to pose visualization.
|
||||
|
||||
This issue tracks the implementation of **RuView Desktop** — a Tauri v2 cross-platform desktop application that replaces all of these tools with a single, cohesive interface. The application is designed as the **control plane** for the RuView platform, managing the full lifecycle: discover, flash, provision, OTA, load WASM, observe sensing.
|
||||
|
||||
### Why Tauri (Not Electron/Flutter/Web)
|
||||
|
||||
| Requirement | Why Desktop is Required |
|
||||
|-------------|------------------------|
|
||||
| Serial port access | Browser/PWA cannot touch COM/tty ports for firmware flashing |
|
||||
| Raw UDP sockets | Node discovery via broadcast probes requires raw socket access |
|
||||
| Filesystem access | Firmware binaries, WASM modules, model files live on local disk |
|
||||
| Process management | Sensing server runs as a managed child process (sidecar) |
|
||||
| Small binary | Tauri ~20 MB vs Electron ~150 MB |
|
||||
| Rust integration | Shares crates with existing workspace |
|
||||
|
||||
### UI Design Language
|
||||
|
||||
The frontend uses a **Foundation Book** design scheme with **Unity Editor-inspired** UI panels. Think: clean typographic hierarchy, structured panels with dockable regions, monospaced data displays, and a professional dark theme with accent colors for status indicators. Powered by rUv.
|
||||
|
||||
---
|
||||
|
||||
## ADR-052 Deep Overview
|
||||
|
||||
The full architecture is documented in [ADR-052](https://github.com/ruvnet/RuView/blob/feat/tauri-desktop-frontend/docs/adr/ADR-052-tauri-desktop-frontend.md) with a companion [DDD bounded contexts appendix](https://github.com/ruvnet/RuView/blob/feat/tauri-desktop-frontend/docs/adr/ADR-052-ddd-bounded-contexts.md).
|
||||
|
||||
### Workspace Integration
|
||||
|
||||
The desktop app is a new Rust crate (`wifi-densepose-desktop`) in the existing workspace, sharing types with the sensing server and hardware crate. The frontend uses React + Vite + TypeScript with a Foundation Book / Unity-inspired design system.
|
||||
|
||||
### 6 Rust Command Groups
|
||||
|
||||
| Group | Commands | Bounded Context |
|
||||
|-------|----------|-----------------|
|
||||
| **Discovery** | `discover_nodes`, `get_node_status`, `watch_nodes` | Device Discovery |
|
||||
| **Flash** | `list_serial_ports`, `flash_firmware`, `read_chip_info` | Firmware Management |
|
||||
| **OTA** | `ota_update`, `ota_status`, `ota_batch_update` | Firmware Management |
|
||||
| **WASM** | `wasm_list`, `wasm_upload`, `wasm_control` | Edge Module |
|
||||
| **Server** | `start_server`, `stop_server`, `server_status` | Sensing Pipeline |
|
||||
| **Provision** | `provision_node`, `read_nvs` | Configuration |
|
||||
|
||||
### 7 Frontend Pages
|
||||
|
||||
| Page | Purpose |
|
||||
|------|---------|
|
||||
| **Dashboard** | Node count (online/offline), server status, quick actions, activity feed |
|
||||
| **Node Detail** | Single node deep-dive: firmware, health, TDM config, WASM modules |
|
||||
| **Flash Firmware** | 3-step wizard: select port, select firmware, flash with progress bar |
|
||||
| **WASM Modules** | Drag-and-drop upload, module list with start/stop/unload |
|
||||
| **Sensing View** | Live CSI heatmap, pose skeleton overlay, vital signs |
|
||||
| **Mesh Topology** | Force-directed graph: TDM slots, sync drift, node health |
|
||||
| **Settings** | Server ports, bind address, OTA PSK, UI theme |
|
||||
|
||||
### DDD Bounded Contexts
|
||||
|
||||
6 bounded contexts with 9 aggregates, 25+ domain events, and 3 anti-corruption layers. See the [DDD appendix](https://github.com/ruvnet/RuView/blob/feat/tauri-desktop-frontend/docs/adr/ADR-052-ddd-bounded-contexts.md) for full details.
|
||||
|
||||
| Context | Aggregate Root(s) | Key Events |
|
||||
|---------|--------------------|------------|
|
||||
| Device Discovery | `NodeRegistry` | `NodeDiscovered`, `NodeWentOffline`, `ScanCompleted` |
|
||||
| Firmware Management | `FlashSession`, `OtaSession`, `BatchOtaSession` | `FlashProgress`, `OtaCompleted`, `BatchOtaCompleted` |
|
||||
| Configuration | `ProvisioningSession` | `NodeProvisioned`, `ConfigReadBack` |
|
||||
| Sensing Pipeline | `SensingServer`, `WebSocketSession` | `ServerStarted`, `FrameReceived` |
|
||||
| Edge Module (WASM) | `ModuleRegistry` | `ModuleUploaded`, `ModuleStarted` |
|
||||
| Visualization | Query model (no aggregate) | Consumes all upstream events |
|
||||
|
||||
### Persistent Node Registry
|
||||
|
||||
Stored in `~/.ruview/nodes.db` (SQLite). On startup, previously known nodes load as Offline and reconcile against fresh discovery. The app remembers the mesh across restarts.
|
||||
|
||||
### OTA Safety Gate
|
||||
|
||||
The `TdmSafe` rolling update strategy updates even-slot nodes first, then odd-slot nodes, ensuring adjacent nodes are never offline simultaneously during mesh-wide firmware updates.
|
||||
|
||||
### Platform-Specific Considerations
|
||||
|
||||
| Platform | Concern | Solution |
|
||||
|----------|---------|----------|
|
||||
| macOS | USB serial drivers need signing on Sequoia+ | Document driver requirements |
|
||||
| Windows | COM port naming, UAC | Auto-detect via registry |
|
||||
| Linux | Serial port permissions | Bundle udev rules installer |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
| Phase | Scope | Priority |
|
||||
|-------|-------|----------|
|
||||
| 1. Skeleton | Tauri scaffolding, workspace integration, React window | P0 |
|
||||
| 2. Discovery | Serial ports, node discovery, dashboard cards | P0 |
|
||||
| 3. Flash | espflash integration, flashing wizard | P0 |
|
||||
| 4. Server | Sidecar sensing server, log viewer | P1 |
|
||||
| 5. OTA | HTTP OTA with PSK auth, batch TdmSafe | P1 |
|
||||
| 6. Provisioning | NVS GUI form, read-back, mesh presets | P1 |
|
||||
| 7. WASM | Module upload/list/control | P2 |
|
||||
| 8. Sensing | WebSocket, live charts, pose overlay | P2 |
|
||||
| 9. Mesh View | Topology graph, TDM visualization | P2 |
|
||||
| 10. Polish | App signing, auto-update, onboarding wizard | P3 |
|
||||
|
||||
Total estimated effort: ~11 weeks for a single developer.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] Tauri app builds on Windows, macOS, Linux
|
||||
- [ ] Can discover ESP32 nodes on local network
|
||||
- [ ] Node registry persists across restarts
|
||||
- [ ] Can flash firmware via serial port (no Python dependency)
|
||||
- [ ] Can push OTA updates with PSK authentication
|
||||
- [ ] Rolling OTA with TdmSafe strategy for mesh deployments
|
||||
- [ ] Can upload/manage WASM modules on nodes
|
||||
- [ ] Can start/stop sensing server and view live logs
|
||||
- [ ] Can view real-time sensing data via WebSocket
|
||||
- [ ] Can provision NVS config via GUI form
|
||||
- [ ] Mesh topology visualization shows TDM slots and health
|
||||
- [ ] Binary size less than 30 MB
|
||||
- [ ] Foundation Book / Unity-inspired UI design system
|
||||
- [ ] Each new Rust module has unit tests
|
||||
|
||||
## Dependencies
|
||||
|
||||
- ADR-012: ESP32 CSI Sensor Mesh
|
||||
- ADR-039: ESP32 Edge Intelligence
|
||||
- ADR-040: WASM Programmable Sensing
|
||||
- ADR-044: Provisioning Tool Enhancements
|
||||
- ADR-050: Quality Engineering Security Hardening
|
||||
- ADR-051: Sensing Server Decomposition
|
||||
- ADR-053: UI Design System (Foundation Book + Unity-inspired)
|
||||
|
||||
## Branch
|
||||
|
||||
[`feat/tauri-desktop-frontend`](https://github.com/ruvnet/RuView/tree/feat/tauri-desktop-frontend)
|
||||
|
||||
## References
|
||||
|
||||
- [ADR-052: Tauri Desktop Frontend](https://github.com/ruvnet/RuView/blob/feat/tauri-desktop-frontend/docs/adr/ADR-052-tauri-desktop-frontend.md)
|
||||
- [ADR-052 DDD Appendix](https://github.com/ruvnet/RuView/blob/feat/tauri-desktop-frontend/docs/adr/ADR-052-ddd-bounded-contexts.md)
|
||||
- [Tauri v2 Documentation](https://v2.tauri.app/)
|
||||
- [espflash crate](https://crates.io/crates/espflash)
|
||||
|
||||
Powered by **rUv**
|
||||
@@ -96,6 +96,13 @@ static void csi_data_callback(void *ctx, wifi_csi_info_t *info) {
|
||||
|
||||
**No on-device FFT** (contradicting ADR-012's optional feature extraction path): The Rust aggregator will do feature extraction using the SOTA `wifi-densepose-signal` pipeline. Raw I/Q is cheaper to stream at ESP32 sampling rates (~100 Hz at 56 subcarriers = ~35 KB/s per node).
|
||||
|
||||
**Rate-limiting and ENOMEM backoff** (Issue #127 fix):
|
||||
|
||||
CSI callbacks fire 100-500+ times/sec in promiscuous mode. Two safeguards prevent lwIP pbuf exhaustion:
|
||||
|
||||
1. **50 Hz rate limiter** (`csi_collector.c`): `sendto()` is skipped if less than 20 ms have elapsed since the last successful send. Excess CSI callbacks are dropped silently.
|
||||
2. **ENOMEM backoff** (`stream_sender.c`): When `sendto()` returns `ENOMEM` (errno 12), all sends are suppressed for 100 ms to let lwIP reclaim packet buffers. Without this, rapid-fire failed sends cause a guru meditation crash.
|
||||
|
||||
**`sdkconfig.defaults`** must enable:
|
||||
|
||||
```
|
||||
|
||||
@@ -74,6 +74,8 @@ static uint32_t s_dwell_ms = 50; // 50ms per channel
|
||||
|
||||
At 100 Hz raw CSI rate with 50 ms dwell across 3 channels, each channel yields ~33 frames/second. The existing ADR-018 binary frame format already carries `channel_freq_mhz` at offset 8, so no wire format change is needed.
|
||||
|
||||
> **Note (Issue #127 fix):** In promiscuous mode, CSI callbacks fire 100-500+ times/sec — far exceeding the channel dwell rate. The firmware now rate-limits UDP sends to 50 Hz and applies a 100 ms ENOMEM backoff if lwIP buffers are exhausted. This is essential for stable channel hopping under load.
|
||||
|
||||
**NDP frame injection:** `esp_wifi_80211_tx()` injects deterministic Null Data Packet frames (preamble-only, no payload, ~24 us airtime) at GPIO-triggered intervals. This is sensing-first: the primary RF emission purpose is CSI measurement, not data communication.
|
||||
|
||||
### 2.3 Multi-Band Frame Fusion
|
||||
@@ -364,6 +366,7 @@ No new workspace dependencies. All ruvector crates are already in the workspace
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|-------------|--------|------------|
|
||||
| ESP32 channel hop causes CSI gaps | Medium | Reduced effective rate | Measure gap duration; increase dwell if >5ms |
|
||||
| CSI callback rate exhausts lwIP pbufs | **Resolved** | Guru meditation crash | 50 Hz rate limiter + 100 ms ENOMEM backoff (Issue #127, PR #132) |
|
||||
| 5 GHz CSI unavailable on S3 | High | Lose frequency diversity | Fallback: 3-channel 2.4 GHz still provides 3x BW; ESP32-C6 for dual-band |
|
||||
| Model inference >40ms | Medium | Miss 20 Hz target | Run model at 10 Hz; Kalman predict at 20 Hz interpolates |
|
||||
| Two-person separation fails at 3 nodes | Low | Identity swaps | AETHER re-ID recovers; increase to 4-6 nodes |
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
# ADR-039: ESP32-S3 Edge Intelligence Pipeline
|
||||
|
||||
**Status**: Accepted (hardware-validated on RuView ESP32-S3)
|
||||
**Date**: 2026-03-02
|
||||
**Deciders**: @ruvnet
|
||||
|
||||
## Context
|
||||
|
||||
WiFi-DensePose captures Channel State Information (CSI) from ESP32-S3 nodes and streams raw I/Q data to a host server for processing. This architecture has limitations:
|
||||
|
||||
1. **Bandwidth**: Raw CSI at 20 Hz × 128 subcarriers × 2 bytes = ~5 KB/frame = ~100 KB/s per node. Multi-node deployments saturate low-bandwidth links.
|
||||
2. **Latency**: Server-side processing adds network round-trip delay for time-critical signals like fall detection.
|
||||
3. **Power**: Continuous raw streaming prevents duty-cycling for battery-powered deployments.
|
||||
4. **Scalability**: Server CPU scales linearly with node count for basic signal processing that could run on the ESP32-S3's dual cores.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a tiered edge processing pipeline on the ESP32-S3 that performs signal processing locally and sends compact results:
|
||||
|
||||
### Tier 0 — Raw Passthrough (default, backward compatible)
|
||||
No on-device processing. CSI frames streamed as-is (magic `0xC5110001`).
|
||||
|
||||
### Tier 1 — Basic Signal Processing
|
||||
- Phase extraction and unwrapping from I/Q pairs
|
||||
- Welford running variance per subcarrier
|
||||
- Top-K subcarrier selection by variance
|
||||
- Delta compression (XOR + RLE) for 30-50% bandwidth reduction (magic `0xC5110005`, reassigned from `0xC5110003` by ADR-069)
|
||||
|
||||
### Tier 2 — Full Edge Intelligence
|
||||
All of Tier 1, plus:
|
||||
- Biquad IIR bandpass filters: breathing (0.1-0.5 Hz), heart rate (0.8-2.0 Hz)
|
||||
- Zero-crossing BPM estimation
|
||||
- Presence detection with adaptive threshold calibration (1200 frames, 3-sigma)
|
||||
- Fall detection (phase acceleration exceeding configurable threshold)
|
||||
- Multi-person vitals via subcarrier group clustering (up to 4 persons)
|
||||
- 32-byte vitals packet at configurable interval (magic `0xC5110002`)
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
Core 0 (WiFi) Core 1 (DSP)
|
||||
┌─────────────────┐ ┌──────────────────────────┐
|
||||
│ CSI callback │──SPSC ring──▶│ Phase extract + unwrap │
|
||||
│ (wifi_csi_cb) │ buffer │ Welford variance │
|
||||
│ │ │ Top-K selection │
|
||||
│ UDP raw stream │ │ Biquad bandpass filters │
|
||||
│ (0xC5110001) │ │ Zero-crossing BPM │
|
||||
└─────────────────┘ │ Presence detection │
|
||||
│ Fall detection │
|
||||
│ Multi-person clustering │
|
||||
│ Delta compression │
|
||||
│ ──▶ UDP vitals (0xC5110002)│
|
||||
│ ──▶ UDP compressed (0x05) │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
### Wire Protocols
|
||||
|
||||
**Vitals Packet (32 bytes, magic `0xC5110002`)**:
|
||||
|
||||
| Offset | Type | Field |
|
||||
|--------|------|-------|
|
||||
| 0-3 | u32 LE | Magic `0xC5110002` |
|
||||
| 4 | u8 | Node ID |
|
||||
| 5 | u8 | Flags (bit0=presence, bit1=fall, bit2=motion) |
|
||||
| 6-7 | u16 LE | Breathing rate (BPM × 100) |
|
||||
| 8-11 | u32 LE | Heart rate (BPM × 10000) |
|
||||
| 12 | i8 | RSSI |
|
||||
| 13 | u8 | Number of detected persons |
|
||||
| 14-15 | u8[2] | Reserved |
|
||||
| 16-19 | f32 LE | Motion energy |
|
||||
| 20-23 | f32 LE | Presence score |
|
||||
| 24-27 | u32 LE | Timestamp (ms since boot) |
|
||||
| 28-31 | u32 LE | Reserved |
|
||||
|
||||
**Compressed Frame (magic `0xC5110005`, reassigned from `0xC5110003` by ADR-069)**:
|
||||
|
||||
| Offset | Type | Field |
|
||||
|--------|------|-------|
|
||||
| 0-3 | u32 LE | Magic `0xC5110005` |
|
||||
| 4 | u8 | Node ID |
|
||||
| 5 | u8 | WiFi channel |
|
||||
| 6-7 | u16 LE | Original I/Q length |
|
||||
| 8-9 | u16 LE | Compressed length |
|
||||
| 10+ | bytes | RLE-encoded XOR delta |
|
||||
|
||||
### Configuration
|
||||
|
||||
Six NVS keys in the `csi_cfg` namespace:
|
||||
|
||||
| NVS Key | Type | Default | Description |
|
||||
|---------|------|---------|-------------|
|
||||
| `edge_tier` | u8 | 2 | Processing tier (0/1/2) |
|
||||
| `pres_thresh` | u16 | 0 | Presence threshold × 1000 (0 = auto) |
|
||||
| `fall_thresh` | u16 | 2000 | Fall threshold × 1000 (rad/s²) |
|
||||
| `vital_win` | u16 | 256 | Phase history window |
|
||||
| `vital_int` | u16 | 1000 | Vitals interval (ms) |
|
||||
| `subk_count` | u8 | 8 | Top-K subcarrier count |
|
||||
|
||||
All configurable via `provision.py --edge-tier 2 --pres-thresh 0.05 ...`
|
||||
|
||||
### Additional Features
|
||||
|
||||
- **OTA Updates**: HTTP server on port 8032 (`POST /ota`, `GET /ota/status`) with rollback support
|
||||
- **Power Management**: WiFi modem sleep + automatic light sleep with configurable duty cycle
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Fall detection latency reduced from ~500 ms (network RTT) to <50 ms (on-device)
|
||||
- Bandwidth reduced 30-50% with delta compression, or 95%+ with vitals-only mode
|
||||
- Battery-powered deployments possible with duty-cycled light sleep
|
||||
- Server can handle 10x more nodes (only parses 32-byte vitals instead of ~5 KB CSI)
|
||||
|
||||
### Negative
|
||||
- Firmware complexity increases (edge_processing.c is ~750 lines)
|
||||
- ESP32-S3 RAM usage increases ~12 KB for ring buffer + filter state
|
||||
- Binary size increases from ~550 KB to ~925 KB with full WASM3 Tier 3 (10% free in 1 MB partition — see ADR-040)
|
||||
|
||||
### Risks
|
||||
- BPM accuracy depends on subject distance and movement; needs real-world validation
|
||||
- Fall detection heuristic may false-positive on environmental motion (doors, pets)
|
||||
- Multi-person separation via subcarrier clustering is approximate without calibration
|
||||
|
||||
## Implementation
|
||||
|
||||
- `firmware/esp32-csi-node/main/edge_processing.c` — DSP pipeline (~750 lines)
|
||||
- `firmware/esp32-csi-node/main/edge_processing.h` — Types and API
|
||||
- `firmware/esp32-csi-node/main/ota_update.c/h` — HTTP OTA endpoint
|
||||
- `firmware/esp32-csi-node/main/power_mgmt.c/h` — Power management
|
||||
- `rust-port/.../wifi-densepose-sensing-server/src/main.rs` — Vitals parser + REST endpoint
|
||||
- `scripts/provision.py` — Edge config CLI arguments
|
||||
- `.github/workflows/firmware-ci.yml` — CI build + size gate (updated to 950 KB for Tier 3)
|
||||
|
||||
### Tier 3 — WASM Programmable Sensing (ADR-040, ADR-041)
|
||||
|
||||
See [ADR-040](ADR-040-wasm-programmable-sensing.md) for hot-loadable WASM modules
|
||||
compiled from Rust, executed via WASM3 interpreter on-device. Core modules:
|
||||
gesture recognition, coherence monitoring, adversarial detection.
|
||||
|
||||
[ADR-041](ADR-041-wasm-module-collection.md) defines the curated module collection
|
||||
(37 modules across 6 categories). Phase 1 implemented modules:
|
||||
- `vital_trend.rs` — Clinical vital sign trend analysis (bradypnea, tachypnea, apnea)
|
||||
- `intrusion.rs` — State-machine intrusion detection (calibrate-monitor-arm-alert)
|
||||
- `occupancy.rs` — Spatial occupancy zone detection with per-zone variance analysis
|
||||
|
||||
## Hardware Benchmark (RuView ESP32-S3)
|
||||
|
||||
Measured on ESP32-S3 (QFN56 rev v0.2, 8 MB flash, 160 MHz, ESP-IDF v5.2).
|
||||
|
||||
### Boot Timing
|
||||
|
||||
| Milestone | Time (ms) |
|
||||
|-----------|-----------|
|
||||
| `app_main()` | 412 |
|
||||
| WiFi STA init | 627 |
|
||||
| WiFi connected + IP | 3,732 |
|
||||
| CSI collection init | 3,754 |
|
||||
| Edge DSP task started | 3,773 |
|
||||
| WASM runtime initialized | 3,857 |
|
||||
| **Total boot → ready** | **~3.9 s** |
|
||||
|
||||
### CSI Performance
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Frame rate | **28.5 Hz** (measured, ch 5 BW20) |
|
||||
| Frame sizes | 128 / 256 bytes |
|
||||
| RSSI range | -83 to -32 dBm (mean -62 dBm) |
|
||||
| Per-frame interval | 30.6 ms avg |
|
||||
|
||||
### Memory
|
||||
|
||||
| Region | Size |
|
||||
|--------|------|
|
||||
| RAM (main heap) | 256 KiB |
|
||||
| RAM (secondary) | 21 KiB |
|
||||
| DRAM | 32 KiB |
|
||||
| RTC RAM | 7 KiB |
|
||||
| **Total available** | **316 KiB** |
|
||||
| PSRAM | Not populated on test board |
|
||||
| WASM arena fallback | Internal heap (160 KB/slot × 4) |
|
||||
|
||||
### Firmware Binary
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Binary size | **925 KB** (0xE7440 bytes) |
|
||||
| Partition size | 1 MB (factory) |
|
||||
| Free space | 10% (99 KB) |
|
||||
| CI size gate | 950 KB (PASS) |
|
||||
| WASM3 interpreter | Included (full, ~100 KB) |
|
||||
| WASM binary (7 modules) | 13.8 KB (wasm32-unknown-unknown release) |
|
||||
|
||||
### WASM Runtime
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Init time | **106 ms** |
|
||||
| Module slots | 4 |
|
||||
| Arena per slot | 160 KB |
|
||||
| Frame budget | 10,000 µs (10 ms) |
|
||||
| Timer interval | 1,000 ms (1 Hz) |
|
||||
|
||||
### Findings
|
||||
|
||||
1. **Fall detection threshold too low** — default `fall_thresh=2000` (2.0 rad/s²) triggers 6.7 false positives/s in static indoor environment. Recommend increasing to 5000-8000 for typical deployments.
|
||||
2. **No PSRAM on test board** — WASM arena falls back to internal heap. Boards with PSRAM would support larger modules.
|
||||
3. **CSI rate exceeds spec** — measured 28.5 Hz vs. expected ~20 Hz. Performance headroom is better than estimated.
|
||||
4. **WiFi-to-Ethernet isolation** — some routers block UDP between WiFi and wired clients. Recommend same-subnet verification in deployment guide.
|
||||
5. **sendto ENOMEM crash (Issue #127)** — CSI callbacks in promiscuous mode fire 100-500+ times/sec, exhausting the lwIP pbuf pool and causing a guru meditation crash. Fixed with a dual approach: 50 Hz rate limiter in `csi_collector.c` (20 ms minimum send interval) and a 100 ms ENOMEM backoff in `stream_sender.c`. Binary size with fix: 947 KB. Hardware-verified stable for 200+ CSI callbacks with zero ENOMEM errors.
|
||||
@@ -0,0 +1,582 @@
|
||||
# ADR-040: WASM Programmable Sensing (Tier 3)
|
||||
|
||||
**Status**: Accepted
|
||||
**Date**: 2026-03-02
|
||||
**Deciders**: @ruvnet
|
||||
|
||||
## Context
|
||||
|
||||
ADR-039 implemented Tiers 0-2 of the ESP32-S3 edge intelligence pipeline:
|
||||
- **Tier 0**: Raw CSI passthrough (magic `0xC5110001`)
|
||||
- **Tier 1**: Basic DSP — phase unwrap, Welford stats, top-K, delta compression
|
||||
- **Tier 2**: Full pipeline — vitals, presence, fall detection, multi-person
|
||||
|
||||
The firmware uses ~820 KB of flash, leaving ~80 KB headroom in the 1 MB OTA partition. The ESP32-S3 has 8 MB PSRAM available for runtime data. New sensing algorithms (gesture recognition, signal coherence monitoring, adversarial detection) currently require a full firmware reflash — impractical for deployed sensor networks.
|
||||
|
||||
The project already has 35+ RuVector WASM crates and 28 pre-built `.wasm` binaries, but none are integrated into the ESP32 firmware.
|
||||
|
||||
## Decision
|
||||
|
||||
Add a **Tier 3 WASM programmable sensing layer** that executes hot-loadable algorithms compiled from Rust to `wasm32-unknown-unknown`, interpreted on-device via the WASM3 runtime.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
Core 1 (DSP Task)
|
||||
┌──────────────────────────────────────────────────┐
|
||||
│ Tier 2 Pipeline (existing) │
|
||||
│ Phase extract → Welford → Top-K → Biquad → │
|
||||
│ BPM → Presence → Fall → Multi-person │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────────────────┐ │
|
||||
│ │ Tier 3 WASM Runtime (new) │ │
|
||||
│ │ WASM3 Interpreter (MIT, ~100 KB flash) │ │
|
||||
│ │ ┌────────────┐ ┌────────────┐ │ │
|
||||
│ │ │ Module 0 │ │ Module 1 │ ...×4 │ │
|
||||
│ │ │ gesture.wm │ │ coherence │ │ │
|
||||
│ │ └─────┬──────┘ └─────┬──────┘ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ Host API ("csi" namespace) │ │
|
||||
│ │ csi_get_phase, csi_get_amplitude, ... │ │
|
||||
│ └──────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ UDP output (0xC5110004) │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Components
|
||||
|
||||
| Component | File | Description |
|
||||
|-----------|------|-------------|
|
||||
| WASM3 component | `components/wasm3/CMakeLists.txt` | ESP-IDF managed component, fetches WASM3 from GitHub |
|
||||
| Runtime host | `main/wasm_runtime.c/h` | WASM3 environment, module slots, host API bindings |
|
||||
| HTTP upload | `main/wasm_upload.c/h` | REST endpoints for module management on port 8032 |
|
||||
| Rust WASM crate | `wifi-densepose-wasm-edge/` | `no_std` sensing algorithms compiled to WASM |
|
||||
|
||||
### Host API (namespace "csi")
|
||||
|
||||
| Import | Signature | Description |
|
||||
|--------|-----------|-------------|
|
||||
| `csi_get_phase` | `(i32) -> f32` | Current phase for subcarrier index |
|
||||
| `csi_get_amplitude` | `(i32) -> f32` | Current amplitude |
|
||||
| `csi_get_variance` | `(i32) -> f32` | Welford running variance |
|
||||
| `csi_get_bpm_breathing` | `() -> f32` | Breathing BPM from Tier 2 |
|
||||
| `csi_get_bpm_heartrate` | `() -> f32` | Heart rate BPM from Tier 2 |
|
||||
| `csi_get_presence` | `() -> i32` | Presence flag (0/1) |
|
||||
| `csi_get_motion_energy` | `() -> f32` | Motion energy scalar |
|
||||
| `csi_get_n_persons` | `() -> i32` | Detected person count |
|
||||
| `csi_get_timestamp` | `() -> i32` | Milliseconds since boot |
|
||||
| `csi_emit_event` | `(i32, f32) -> void` | Emit custom event to host |
|
||||
| `csi_log` | `(i32, i32) -> void` | Debug log from WASM memory |
|
||||
| `csi_get_phase_history` | `(i32, i32) -> i32` | Copy phase history ring buffer |
|
||||
|
||||
### Module Lifecycle
|
||||
|
||||
| Export | Called | Description |
|
||||
|--------|--------|-------------|
|
||||
| `on_init()` | Once, when module starts | Initialize module state |
|
||||
| `on_frame(n_sc: i32)` | Per CSI frame (~20 Hz) | Process current frame |
|
||||
| `on_timer()` | At configurable interval | Periodic tasks |
|
||||
|
||||
### Wire Protocol (magic `0xC5110004`)
|
||||
|
||||
| Offset | Type | Field |
|
||||
|--------|------|-------|
|
||||
| 0-3 | u32 LE | Magic `0xC5110004` |
|
||||
| 4 | u8 | Node ID |
|
||||
| 5 | u8 | Module ID (slot index) |
|
||||
| 6-7 | u16 LE | Event count |
|
||||
| 8+ | Event[] | Array of (u8 type, f32 value) tuples |
|
||||
|
||||
### HTTP Endpoints (port 8032)
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| `POST` | `/wasm/upload` | Upload .wasm binary (max 128 KB) |
|
||||
| `GET` | `/wasm/list` | List loaded modules with status |
|
||||
| `POST` | `/wasm/start/:id` | Start a module |
|
||||
| `POST` | `/wasm/stop/:id` | Stop a module |
|
||||
| `DELETE` | `/wasm/:id` | Unload a module |
|
||||
|
||||
### WASM Crate Modules
|
||||
|
||||
| Module | Source | Events | Description |
|
||||
|--------|--------|--------|-------------|
|
||||
| `gesture.rs` | `ruvsense/gesture.rs` | 1 (Core) | DTW template matching for gesture recognition |
|
||||
| `coherence.rs` | `ruvector/viewpoint/coherence.rs` | 2 (Core) | Phase phasor coherence monitoring |
|
||||
| `adversarial.rs` | `ruvsense/adversarial.rs` | 3 (Core) | Signal anomaly/adversarial detection |
|
||||
| `vital_trend.rs` | ADR-041 Phase 1 | 100-111 (Medical) | Clinical vital sign trend analysis (bradypnea, tachypnea, bradycardia, tachycardia, apnea) |
|
||||
| `occupancy.rs` | ADR-041 Phase 1 | 300-302 (Building) | Spatial occupancy zone detection with per-zone variance analysis |
|
||||
| `intrusion.rs` | ADR-041 Phase 1 | 200-203 (Security) | State-machine intrusion detector (calibrate-monitor-arm-alert) |
|
||||
|
||||
### Memory Budget
|
||||
|
||||
| Component | SRAM | PSRAM | Flash |
|
||||
|-----------|------|-------|-------|
|
||||
| WASM3 interpreter | ~10 KB | — | ~100 KB |
|
||||
| WASM module storage (×4) | — | 512 KB | — |
|
||||
| WASM execution stack | 8 KB | — | — |
|
||||
| Host API bindings | 2 KB | — | ~15 KB |
|
||||
| HTTP upload handler | 1 KB | — | ~8 KB |
|
||||
| RVF parser + verifier | 1 KB | — | ~6 KB |
|
||||
| **Total Tier 3** | **~22 KB** | **512 KB** | **~129 KB** |
|
||||
| **Running total (Tier 0-3)** | **~34 KB** | **512 KB** | **~925 KB** |
|
||||
|
||||
**Measured binary size**: 925 KB (0xE7440 bytes), 10% free in 1 MB OTA partition.
|
||||
|
||||
### NVS Configuration
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `wasm_max` | u8 | 4 | Maximum concurrent WASM modules |
|
||||
| `wasm_verify` | u8 | 1 | Require signature verification (secure-by-default) |
|
||||
| `wasm_pubkey` | blob(32) | — | Signing public key for WASM verification |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Deploy new sensing algorithms to 1000+ nodes without reflashing firmware
|
||||
- 20-year extensibility horizon — new algorithms via .wasm uploads
|
||||
- Algorithms developed/tested in Rust, compiled to portable WASM
|
||||
- PSRAM utilization (previously unused 8 MB) for module storage
|
||||
- Hot-swap algorithms for A/B testing in production deployments
|
||||
- Same `no_std` Rust code runs on ESP32 (WASM3) and in browser (wasm-pack)
|
||||
|
||||
### Negative
|
||||
- WASM3 interpreter overhead: ~10× slower than native C for compute-heavy code
|
||||
- Adds ~123 KB flash footprint (firmware approaches 950 KB of 1 MB limit)
|
||||
- Additional attack surface via WASM module upload endpoint
|
||||
- Debugging WASM modules on ESP32 is harder than native C
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| WASM3 memory management may fragment PSRAM over time | Fixed 160 KB arenas pre-allocated at boot per slot — no runtime malloc/free cycles |
|
||||
| Complex WASM modules (>64 KB) may cause stack overflow in interpreter | `WASM_STACK_SIZE` = 8 KB, `d_m3MaxFunctionStackHeight` = 128; modules validated at load time |
|
||||
| HTTP upload endpoint requires network security | Ed25519 signature verification enabled by default (`wasm_verify=1`); disable only via NVS for lab/dev |
|
||||
| Runaway WASM module blocks DSP pipeline | Per-frame budget guard (10 ms default); module auto-stopped after 10 consecutive faults |
|
||||
| Denial-of-service via rapid upload/unload cycles | Max 4 concurrent slots; upload handler validates size before PSRAM copy |
|
||||
|
||||
## Implementation
|
||||
|
||||
- `firmware/esp32-csi-node/components/wasm3/CMakeLists.txt` — WASM3 ESP-IDF component
|
||||
- `firmware/esp32-csi-node/main/wasm_runtime.c/h` — Runtime host with 12 API bindings + manifest
|
||||
- `firmware/esp32-csi-node/main/wasm_upload.c/h` — HTTP REST endpoints (RVF-aware)
|
||||
- `firmware/esp32-csi-node/main/rvf_parser.c/h` — RVF container parser and verifier
|
||||
- `rust-port/.../wifi-densepose-wasm-edge/` — Rust WASM crate (gesture, coherence, adversarial, rvf, occupancy, vital_trend, intrusion)
|
||||
- `rust-port/.../wifi-densepose-sensing-server/src/main.rs` — `0xC5110004` parser
|
||||
- `docs/adr/ADR-039-esp32-edge-intelligence.md` — Updated with Tier 3 reference
|
||||
|
||||
---
|
||||
|
||||
## Appendix A: Production Hardening
|
||||
|
||||
The initial Tier 3 implementation addresses five production-readiness concerns:
|
||||
|
||||
### A.1 Fixed PSRAM Arenas
|
||||
|
||||
Dynamic `heap_caps_malloc` / `free` cycles on PSRAM fragment memory over days of
|
||||
continuous operation. Instead, each module slot pre-allocates a **160 KB fixed arena**
|
||||
at boot (`WASM_ARENA_SIZE`). The WASM binary and WASM3 runtime heap both live inside
|
||||
this arena. Unloading a module zeroes the arena but never frees it — the slot is
|
||||
reused on the next `wasm_runtime_load()`.
|
||||
|
||||
```
|
||||
Boot: [arena0: 160 KB][arena1: 160 KB][arena2: 160 KB][arena3: 160 KB]
|
||||
Total: 640 KB PSRAM
|
||||
Load: [module0 binary | wasm3 heap | ...padding... ]
|
||||
Unload:[zeroed .......................................] ← slot reusable
|
||||
```
|
||||
|
||||
This eliminates fragmentation at the cost of reserving 640 KB PSRAM at boot
|
||||
(8% of 8 MB). The remaining 7.36 MB is available for future use.
|
||||
|
||||
### A.2 Per-Frame Budget Guard
|
||||
|
||||
Each `on_frame()` call is measured with `esp_timer_get_time()`. If execution
|
||||
exceeds `WASM_FRAME_BUDGET_US` (default 10 ms = 10,000 us), a budget fault is
|
||||
recorded. After **10 consecutive faults**, the module is auto-stopped with
|
||||
`WASM_MODULE_ERROR` state. This prevents a runaway WASM module from blocking the
|
||||
Tier 2 DSP pipeline.
|
||||
|
||||
```c
|
||||
int64_t t_start = esp_timer_get_time();
|
||||
m3_CallV(slot->fn_on_frame, n_sc);
|
||||
uint32_t elapsed_us = (uint32_t)(esp_timer_get_time() - t_start);
|
||||
|
||||
slot->total_us += elapsed_us;
|
||||
if (elapsed_us > slot->max_us) slot->max_us = elapsed_us;
|
||||
|
||||
if (elapsed_us > WASM_FRAME_BUDGET_US) {
|
||||
slot->budget_faults++;
|
||||
if (slot->budget_faults >= 10) {
|
||||
slot->state = WASM_MODULE_ERROR; // auto-stop
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The budget is configurable via `WASM_FRAME_BUDGET_US` (Kconfig or NVS override).
|
||||
|
||||
### A.3 Per-Module Telemetry
|
||||
|
||||
The `/wasm/list` endpoint and `wasm_module_info_t` struct expose per-module
|
||||
telemetry:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `frame_count` | u32 | Total on_frame calls since start |
|
||||
| `event_count` | u32 | Total csi_emit_event calls |
|
||||
| `error_count` | u32 | WASM3 runtime errors |
|
||||
| `total_us` | u32 | Cumulative execution time (microseconds) |
|
||||
| `max_us` | u32 | Worst-case single frame execution time |
|
||||
| `budget_faults` | u32 | Times frame budget was exceeded |
|
||||
|
||||
Mean execution time = `total_us / frame_count`. This enables remote monitoring
|
||||
of module health and performance regression detection.
|
||||
|
||||
### A.4 Secure-by-Default
|
||||
|
||||
`wasm_verify` defaults to **1** in both Kconfig and the NVS fallback path.
|
||||
Uploaded `.wasm` binaries must include a valid Ed25519 signature (same key as
|
||||
OTA firmware). Disable only for lab/dev use via:
|
||||
|
||||
```bash
|
||||
python provision.py --port COM7 --wasm-verify # NVS: wasm_verify=1 (default)
|
||||
# To disable in dev: write wasm_verify=0 to NVS directly
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Appendix B: Adaptive Budget Architecture (Mincut-Driven)
|
||||
|
||||
### B.1 Design Principle
|
||||
|
||||
One control loop turns **sensing into a bounded compute budget**, spends that
|
||||
budget on **sparse or spiking inference**, and exports **only deltas**. The
|
||||
budget is driven by the **mincut eigenvalue gap** (Δλ = λ₂ − λ₁ of the CSI
|
||||
graph Laplacian), which reflects scene complexity: a quiet room has Δλ ≈ 0,
|
||||
a busy room has large Δλ.
|
||||
|
||||
### B.2 Control Loop
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
CSI frames ───→ │ Tier 2 DSP (existing) │
|
||||
│ Welford stats, top-K, presence │
|
||||
└──────────┬────────────────────────┘
|
||||
│
|
||||
┌──────────────▼──────────────────────┐
|
||||
│ Budget Controller │
|
||||
│ │
|
||||
│ Inputs: │
|
||||
│ Δλ = mincut eigenvalue gap │
|
||||
│ A = anomaly_score (adversarial) │
|
||||
│ T = thermal_pressure (0.0-1.0) │
|
||||
│ P = battery_pressure (0.0-1.0) │
|
||||
│ │
|
||||
│ Output: │
|
||||
│ B = frame compute budget (μs) │
|
||||
│ │
|
||||
│ B = clamp(B₀ + k₁·max(0,Δλ) │
|
||||
│ + k₂·A │
|
||||
│ − k₃·T │
|
||||
│ − k₄·P, │
|
||||
│ B_min, B_max) │
|
||||
└──────────────┬──────────────────────┘
|
||||
│
|
||||
┌──────────────▼──────────────────────┐
|
||||
│ WASM Module Dispatch │
|
||||
│ Budget B split across active modules│
|
||||
│ Each module gets B/N μs per frame │
|
||||
└──────────────┬──────────────────────┘
|
||||
│
|
||||
┌──────────────▼──────────────────────┐
|
||||
│ Delta Export │
|
||||
│ Only emit events when Δ > threshold │
|
||||
│ Quiet room → near-zero UDP traffic │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### B.3 Budget Formula
|
||||
|
||||
```
|
||||
B = clamp(B₀ + k₁·max(0, Δλ) + k₂·A − k₃·T − k₄·P, B_min, B_max)
|
||||
```
|
||||
|
||||
| Symbol | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| B₀ | 5,000 μs | Base budget (5 ms) |
|
||||
| k₁ | 2,000 | Δλ sensitivity (more scene change → more budget) |
|
||||
| k₂ | 3,000 | Anomaly boost (detected anomaly → more compute) |
|
||||
| k₃ | 4,000 | Thermal penalty (chip hot → less compute) |
|
||||
| k₄ | 3,000 | Battery penalty (low SoC → less compute) |
|
||||
| B_min | 1,000 μs | Floor: always run at least 1 ms |
|
||||
| B_max | 15,000 μs | Ceiling: never exceed 15 ms |
|
||||
|
||||
### B.4 Where Δλ Comes From
|
||||
|
||||
The mincut graph is the **top-K subcarrier correlation graph** already
|
||||
maintained by Tier 1/2 DSP. Subcarriers are nodes; edge weights are
|
||||
pairwise Pearson correlation magnitudes over the Welford window. The
|
||||
algebraic connectivity (Fiedler value λ₂) of this graph's Laplacian
|
||||
approximates the mincut value. On ESP32-S3 with K=8 subcarriers, this
|
||||
is an 8×8 eigenvalue problem — solvable with power iteration in <100 μs.
|
||||
|
||||
### B.5 Spiking and Sparse Optimizations
|
||||
|
||||
When the budget is tight (Δλ ≈ 0, quiet room), WASM modules should:
|
||||
|
||||
1. **Skip on_frame entirely** if Δλ < ε (no scene change → no computation)
|
||||
2. **Sparse inference**: Only process the top-K subcarriers that changed
|
||||
(already tracked by Tier 1 delta compression)
|
||||
3. **Spiking semantics**: Modules emit events only when state transitions
|
||||
occur, not on every frame. The host tracks a per-module "last emitted"
|
||||
state and suppresses duplicate events.
|
||||
|
||||
### B.6 Thermal and Power Hooks
|
||||
|
||||
ESP32-S3 provides:
|
||||
- `temp_sensor_read()` — on-chip temperature (°C)
|
||||
- ADC reading of battery voltage (if wired)
|
||||
|
||||
Thermal pressure: `T = clamp((temp_celsius - 60) / 20, 0, 1)` — ramps
|
||||
from 0 at 60°C to 1.0 at 80°C (thermal throttle zone).
|
||||
|
||||
Battery pressure: `P = clamp((3.3 - battery_volts) / 0.6, 0, 1)` — ramps
|
||||
from 0 at 3.3V to 1.0 at 2.7V (brownout zone).
|
||||
|
||||
### B.7 Transport Strategy
|
||||
|
||||
WASM output packets (`0xC5110004`) adopt **delta-only export**:
|
||||
|
||||
- Events are only emitted when the value changes by more than a
|
||||
configurable dead-band (default: 5% of previous value)
|
||||
- Quiet room = zero WASM UDP packets (only Tier 2 vitals at 1 Hz)
|
||||
- Busy room = bursty WASM events, naturally rate-limited by budget B
|
||||
|
||||
Future work: QUIC-lite transport with 0-RTT connection resumption and
|
||||
congestion-aware pacing, replacing raw UDP for WASM event streams.
|
||||
|
||||
---
|
||||
|
||||
## Appendix C: Hardware Benchmark (RuView ESP32-S3)
|
||||
|
||||
Measured on ESP32-S3 (QFN56 rev v0.2, 8 MB flash, 160 MHz, ESP-IDF v5.2,
|
||||
board without PSRAM). WiFi connected to AP at RSSI -25 dBm, channel 5 BW20.
|
||||
|
||||
### WASM Runtime Performance
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| WASM runtime init | **106 ms** |
|
||||
| Total boot to ready | **3.9 s** (including WiFi connect) |
|
||||
| Module slots | 4 × 160 KB (heap fallback, no PSRAM) |
|
||||
| WASM binary size (7 modules) | **13.8 KB** (wasm32-unknown-unknown release) |
|
||||
| Frame budget | 10,000 µs (10 ms) |
|
||||
| Timer interval | 1,000 ms (1 Hz) |
|
||||
|
||||
### CSI Throughput
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Frame rate | **28.5 Hz** (exceeds 20 Hz estimate) |
|
||||
| Frame sizes | 128 / 256 bytes |
|
||||
| Per-frame interval | 30.6 ms avg |
|
||||
| RSSI range | -83 to -32 dBm (mean -62 dBm) |
|
||||
|
||||
### Rust Test Results
|
||||
|
||||
| Crate | Tests | Status |
|
||||
|-------|-------|--------|
|
||||
| wifi-densepose-wasm-edge (std) | 14 | All pass, 0 warnings |
|
||||
| Full workspace | 1,411 | All pass, 0 failed |
|
||||
|
||||
### Known Issues
|
||||
|
||||
1. **Fall threshold too sensitive** — default 2.0 rad/s² produces 6.7 false positives/s in static environment. Recommend 5.0-8.0 for deployment.
|
||||
2. **No PSRAM on test board** — WASM arenas fall back to internal heap (316 KiB total). Production boards with 8 MB PSRAM will use dedicated PSRAM arenas.
|
||||
3. **WiFi-Ethernet isolation** — some consumer routers block bridging between WiFi and wired clients. Verify network path during deployment.
|
||||
|
||||
### B.8 Implementation Plan
|
||||
|
||||
| Step | Scope | Effort |
|
||||
|------|-------|--------|
|
||||
| 1 | Add `edge_compute_fiedler()` in `edge_processing.c` — power iteration on 8×8 Laplacian | ~50 lines C |
|
||||
| 2 | Add budget controller struct and update formula in `wasm_runtime.c` | ~30 lines C |
|
||||
| 3 | Wire thermal/battery sensors into budget inputs | ~20 lines C |
|
||||
| 4 | Add delta-export dead-band filter in `wasm_runtime_on_frame()` | ~15 lines C |
|
||||
| 5 | NVS keys for k₁-k₄, B_min, B_max, dead-band threshold | ~10 lines C |
|
||||
|
||||
Total: ~125 lines of C, no new files. All constants configurable via NVS.
|
||||
|
||||
### B.9 Failure Modes
|
||||
|
||||
| Failure | Behavior |
|
||||
|---------|----------|
|
||||
| Δλ estimate wrong (correlation noise) | Budget oscillates — clamped by B_min/B_max |
|
||||
| Thermal sensor absent | T defaults to 0 (no throttle) |
|
||||
| Battery ADC not wired | P defaults to 0 (always-on mode) |
|
||||
| All WASM modules budget-faulted | DSP pipeline runs Tier 2 only — graceful degradation |
|
||||
|
||||
---
|
||||
|
||||
## Appendix C: RVF Container Format
|
||||
|
||||
### C.1 Problem
|
||||
|
||||
Raw `.wasm` uploads over HTTP are remote code execution. Signatures solve
|
||||
authenticity, but without a manifest the host has no way to enforce budgets,
|
||||
check API compatibility, or identify what it's running. RVF wraps the WASM
|
||||
payload with governance metadata in a single artifact.
|
||||
|
||||
### C.2 Binary Layout
|
||||
|
||||
```
|
||||
Offset Size Type Field
|
||||
────────────────────────────────────────────
|
||||
0 4 [u8;4] Magic "RVF\x01" (0x01465652 LE)
|
||||
4 2 u16 LE format_version (1)
|
||||
6 2 u16 LE flags (bit 0: has_signature, bit 1: has_test_vectors)
|
||||
8 4 u32 LE manifest_len (always 96)
|
||||
12 4 u32 LE wasm_len
|
||||
16 4 u32 LE signature_len (0 or 64)
|
||||
20 4 u32 LE test_vectors_len (0 if none)
|
||||
24 4 u32 LE total_len (header + manifest + wasm + sig + tvec)
|
||||
28 4 u32 LE reserved (0)
|
||||
────────────────────────────────────────────
|
||||
32 96 struct Manifest (see below)
|
||||
128 N bytes WASM payload ("\0asm" magic)
|
||||
128+N 0|64 bytes Ed25519 signature (signs bytes 0..128+N-1)
|
||||
128+N+S M bytes Test vectors (optional)
|
||||
```
|
||||
|
||||
Total overhead: 32 (header) + 96 (manifest) + 64 (signature) = **192 bytes**.
|
||||
|
||||
### C.3 Manifest (96 bytes, packed)
|
||||
|
||||
| Offset | Size | Type | Field |
|
||||
|--------|------|------|-------|
|
||||
| 0 | 32 | char[] | `module_name` — null-terminated ASCII |
|
||||
| 32 | 2 | u16 | `required_host_api` — version (1 = current) |
|
||||
| 34 | 4 | u32 | `capabilities` — RVF_CAP_* bitmask |
|
||||
| 38 | 4 | u32 | `max_frame_us` — requested per-frame budget (0 = use default) |
|
||||
| 42 | 2 | u16 | `max_events_per_sec` — rate limit (0 = unlimited) |
|
||||
| 44 | 2 | u16 | `memory_limit_kb` — max WASM heap (0 = use default) |
|
||||
| 46 | 2 | u16 | `event_schema_version` — for receiver compatibility |
|
||||
| 48 | 32 | [u8;32] | `build_hash` — SHA-256 of WASM payload |
|
||||
| 80 | 2 | u16 | `min_subcarriers` — minimum required (0 = any) |
|
||||
| 82 | 2 | u16 | `max_subcarriers` — maximum expected (0 = any) |
|
||||
| 84 | 10 | char[] | `author` — null-padded ASCII |
|
||||
| 94 | 2 | [u8;2] | reserved (0) |
|
||||
|
||||
### C.4 Capability Bitmask
|
||||
|
||||
| Bit | Flag | Host API functions |
|
||||
|-----|------|--------------------|
|
||||
| 0 | `READ_PHASE` | `csi_get_phase` |
|
||||
| 1 | `READ_AMPLITUDE` | `csi_get_amplitude` |
|
||||
| 2 | `READ_VARIANCE` | `csi_get_variance` |
|
||||
| 3 | `READ_VITALS` | `csi_get_bpm_*`, `csi_get_presence`, `csi_get_n_persons` |
|
||||
| 4 | `READ_HISTORY` | `csi_get_phase_history` |
|
||||
| 5 | `EMIT_EVENTS` | `csi_emit_event` |
|
||||
| 6 | `LOG` | `csi_log` |
|
||||
|
||||
Modules declare which host APIs they need. Future firmware versions may
|
||||
refuse to link imports that aren't declared in capabilities — defense in
|
||||
depth against supply-chain attacks.
|
||||
|
||||
### C.5 On-Device Flow
|
||||
|
||||
```
|
||||
HTTP POST /wasm/upload
|
||||
│
|
||||
▼
|
||||
┌────────────────────────┐
|
||||
│ Check first 4 bytes │
|
||||
│ "RVF\x01" → RVF path │
|
||||
│ "\0asm" → raw path │
|
||||
└───────┬────────────────┘
|
||||
│
|
||||
┌────▼────┐ ┌───────────┐
|
||||
│ RVF │ │ Raw WASM │
|
||||
│ parse │ │ (dev only,│
|
||||
│ header │ │ verify=0) │
|
||||
└────┬────┘ └─────┬─────┘
|
||||
│ │
|
||||
┌────▼────┐ │
|
||||
│ Verify │ │
|
||||
│ SHA-256 │ │
|
||||
│ hash │ │
|
||||
└────┬────┘ │
|
||||
│ │
|
||||
┌────▼────┐ │
|
||||
│ Verify │ │
|
||||
│ Ed25519 │ │
|
||||
│ sig │ │
|
||||
└────┬────┘ │
|
||||
│ │
|
||||
┌────▼────┐ │
|
||||
│ Check │ │
|
||||
│ host API│ │
|
||||
│ version │ │
|
||||
└────┬────┘ │
|
||||
│ │
|
||||
├────────────────┘
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ wasm_runtime_load │
|
||||
│ set_manifest │
|
||||
│ start module │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
### C.6 Rollback Support
|
||||
|
||||
Each slot stores the SHA-256 build hash from the manifest. The `/wasm/list`
|
||||
endpoint returns this hash. Fleet management systems can:
|
||||
|
||||
1. Push an RVF to a node
|
||||
2. Verify the installed hash matches via GET `/wasm/list`
|
||||
3. Roll back by pushing the previous RVF (same slot reused after unload)
|
||||
|
||||
Two-slot strategy: maintain slot 0 as "last known good" and slot 1 as
|
||||
"candidate". Promote by stopping slot 0 and starting slot 1.
|
||||
|
||||
### C.7 Rust Builder
|
||||
|
||||
The `wifi-densepose-wasm-edge` crate provides `rvf::builder::build_rvf()`
|
||||
(behind the `std` feature) to package a `.wasm` binary into an `.rvf`:
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::rvf::builder::{build_rvf, RvfConfig};
|
||||
|
||||
let wasm = std::fs::read("target/wasm32-unknown-unknown/release/module.wasm")?;
|
||||
let rvf = build_rvf(&wasm, &RvfConfig {
|
||||
module_name: "gesture".into(),
|
||||
author: "rUv".into(),
|
||||
capabilities: CAP_READ_PHASE | CAP_EMIT_EVENTS,
|
||||
max_frame_us: 5000,
|
||||
..Default::default()
|
||||
});
|
||||
std::fs::write("gesture.rvf", &rvf)?;
|
||||
// Then sign externally with Ed25519 and patch_signature()
|
||||
```
|
||||
|
||||
### C.8 Implementation Files
|
||||
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `firmware/.../main/rvf_parser.h` | RVF types, capability flags, parse/verify API |
|
||||
| `firmware/.../main/rvf_parser.c` | Header/manifest parser, SHA-256 hash check |
|
||||
| `wifi-densepose-wasm-edge/src/rvf.rs` | Format constants, builder (std), tests |
|
||||
|
||||
### C.9 Failure Modes
|
||||
|
||||
| Failure | Behavior |
|
||||
|---------|----------|
|
||||
| RVF too large for PSRAM buffer | Rejected at receive with 400 |
|
||||
| Build hash mismatch | Rejected at parse with `ESP_ERR_INVALID_CRC` |
|
||||
| Signature absent when `wasm_verify=1` | Rejected with 403 |
|
||||
| Host API version too new | Rejected with `ESP_ERR_NOT_SUPPORTED` |
|
||||
| Raw WASM when `wasm_verify=1` | Rejected with 403 |
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,600 @@
|
||||
# ADR-042: Coherent Human Channel Imaging (CHCI) — Beyond WiFi CSI
|
||||
|
||||
**Status**: Proposed
|
||||
**Date**: 2026-03-03
|
||||
**Deciders**: @ruvnet
|
||||
**Supersedes**: None
|
||||
**Related**: ADR-014, ADR-017, ADR-029, ADR-039, ADR-040, ADR-041
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
WiFi-DensePose currently relies on passive Channel State Information (CSI) extracted from standard 802.11 traffic frames. CSI is one specific way of estimating a channel response, but it is fundamentally constrained by a protocol designed for throughput and interoperability — not for sensing.
|
||||
|
||||
### Fundamental Limitations of Passive WiFi CSI
|
||||
|
||||
| Constraint | Root Cause | Impact on Sensing |
|
||||
|-----------|-----------|-------------------|
|
||||
| MAC-layer jitter | CSMA/CA random backoff, retransmissions | Non-uniform sample timing, aliased Doppler |
|
||||
| Rate adaptation | MCS selection varies bandwidth and modulation | Inconsistent subcarrier count per frame |
|
||||
| LO phase drift | Independent oscillators at TX and RX | Phase noise floor ~5° on ESP32, limiting displacement sensitivity to ~0.87 mm at 2.4 GHz |
|
||||
| Frame overhead | 802.11 preamble, headers, FCS | Wasted airtime that could carry sensing symbols |
|
||||
| Bandwidth fragmentation | Channel bonding decisions by AP | Variable spectral coverage per observation |
|
||||
| Multi-node asynchrony | No shared timing reference | TDM coordination requires statistical phase correction (current `phase_align.rs`) |
|
||||
|
||||
These constraints impose a hard floor on sensing fidelity. Breathing detection (4–12 mm chest displacement) is reliable, but heartbeat detection (0.2–0.5 mm) is marginal. Pose estimation accuracy is limited by amplitude-only tomography rather than coherent phase imaging.
|
||||
|
||||
### What We Actually Want
|
||||
|
||||
The real objective is **coherent multipath sensing** — measuring the complex-valued impulse response of the human-occupied channel with sufficient phase stability and temporal resolution to reconstruct body surface geometry and sub-millimeter physiological motion.
|
||||
|
||||
WiFi is optimized for throughput and interoperability. DensePose is optimized for phase stability and micro-Doppler fidelity. Those goals are not aligned.
|
||||
|
||||
### IEEE 802.11bf Changes the Landscape
|
||||
|
||||
IEEE Std 802.11bf-2025 was published on September 26, 2025, defining WLAN Sensing as a first-class MAC/PHY capability. Key provisions:
|
||||
|
||||
- **Null Data PPDU (NDP) sounding**: Deterministic, known waveforms with no data payload — purpose-built for channel measurement
|
||||
- **Sensing Measurement Setup (SMS)**: Negotiation protocol between sensing initiator and responder with unique session IDs
|
||||
- **Trigger-Based Sensing Measurement Exchange (TB SME)**: AP-coordinated sounding with Sensing Availability Windows (SAW)
|
||||
- **Multiband support**: Sub-7 GHz (2.4, 5, 6 GHz) plus 60 GHz mmWave
|
||||
- **Bistatic and multistatic modes**: Standard-defined multi-node sensing
|
||||
|
||||
This transforms WiFi sensing from passive traffic sniffing into an intentional, standards-compliant sensing protocol. The question is whether to adopt 802.11bf incrementally or to design a purpose-built coherent sensing architecture that goes beyond what 802.11bf specifies.
|
||||
|
||||
### ESPARGOS Proves Phase Coherence at ESP32 Cost
|
||||
|
||||
The ESPARGOS project (University of Stuttgart, IEEE 2024) demonstrates that phase-coherent WiFi sensing is achievable with commodity ESP32 hardware:
|
||||
|
||||
- 8 antennas per board, each on an ESP32-S2
|
||||
- Phase coherence via shared 40 MHz reference clock + 2.4 GHz phase reference signal distributed over coaxial cable
|
||||
- Multiple boards combinable into larger coherent arrays
|
||||
- Public datasets with reference positioning labels
|
||||
- Ultra-low cost compared to commercial radar platforms
|
||||
|
||||
This proves the hardware architecture described in this ADR is feasible at the ESP32-S3 price point ($3–5 per node).
|
||||
|
||||
### SOTA Displacement Sensitivity
|
||||
|
||||
| Technology | Frequency | Displacement Resolution | Range | Cost/Node |
|
||||
|-----------|-----------|------------------------|-------|-----------|
|
||||
| Passive WiFi CSI (current) | 2.4/5 GHz | ~0.87 mm (limited by 5° phase noise) | 1–8 m | $3 |
|
||||
| 802.11bf NDP sounding | 2.4/5/6 GHz | ~0.4 mm (coherent averaging) | 1–8 m | $3 |
|
||||
| ESPARGOS phase-coherent | 2.4 GHz | ~0.1 mm (8-antenna coherent) | Room-scale | $5 |
|
||||
| CW Doppler radar (ISM) | 2.4 GHz | ~10 μm | 1–5 m | $15 |
|
||||
| Infineon BGT60TR13C | 58–63.5 GHz | Sub-mm | Up to 15 m | $20 |
|
||||
| Vayyar 4D imaging | 3–81 GHz | High (4D imaging) | Room-scale | $200+ |
|
||||
| Novelda X4 UWB | 7.29/8.748 GHz | Sub-mm | 0.4–10 m | $15–50 |
|
||||
|
||||
The gap between passive WiFi CSI (~0.87 mm) and coherent phase processing (~0.1 mm) represents a 9x improvement in displacement sensitivity — the difference between marginal and reliable heartbeat detection at ISM bands.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
We define **Coherent Human Channel Imaging (CHCI)** — a purpose-built coherent RF sensing protocol optimized for structural human motion, vital sign extraction, and body surface reconstruction. CHCI is not WiFi in the traditional sense. It is a sensing protocol that operates within ISM band regulatory constraints and can optionally maintain backward compatibility with 802.11bf.
|
||||
|
||||
### Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ CHCI System Architecture │
|
||||
├─────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ CHCI Node │ │ CHCI Node │ │ CHCI Node │ │
|
||||
│ │ (TX + RX) │ │ (TX + RX) │ │ (TX + RX) │ │
|
||||
│ │ ESP32-S3 │ │ ESP32-S3 │ │ ESP32-S3 │ │
|
||||
│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │
|
||||
│ │ │ │ │
|
||||
│ └───────────┬───────┴───────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────┴────────┐ │
|
||||
│ │ Reference Clock │ ← 40 MHz TCXO + PLL distribution │
|
||||
│ │ Distribution │ ← 2.4/5 GHz phase reference │
|
||||
│ └────────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────────────┴──────────────────────────────┐ │
|
||||
│ │ Waveform Controller │ │
|
||||
│ │ ┌────────────┐ ┌────────────┐ ┌────────────┐ │ │
|
||||
│ │ │ NDP Sound │ │ Micro-Burst│ │ Chirp Gen │ │ │
|
||||
│ │ │ (802.11bf) │ │ (5 kHz) │ │ (Multi-BW) │ │ │
|
||||
│ │ └────────────┘ └────────────┘ └────────────┘ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ └──────────────┼───────────────┘ │ │
|
||||
│ │ ▼ │ │
|
||||
│ │ ┌─────────────────┐ │ │
|
||||
│ │ │ Cognitive Engine │ ← Scene state │ │
|
||||
│ │ │ (Waveform Adapt) │ feedback loop │ │
|
||||
│ │ └─────────────────┘ │ │
|
||||
│ └───────────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────────────────────────────────────────┐ │
|
||||
│ │ Signal Processing Pipeline │ │
|
||||
│ │ ┌──────────┐ ┌───────────┐ ┌────────────────┐ │ │
|
||||
│ │ │ Coherent │ │ Multi-Band│ │ Diffraction │ │ │
|
||||
│ │ │ Phase │ │ Fusion │ │ Tomography │ │ │
|
||||
│ │ │ Alignment │ │ (2.4+5+6) │ │ (Complex CSI) │ │ │
|
||||
│ │ └──────────┘ └───────────┘ └────────────────┘ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ └──────────────┼───────────────┘ │ │
|
||||
│ │ ▼ │ │
|
||||
│ │ ┌─────────────────┐ │ │
|
||||
│ │ │ Body Model │ │ │
|
||||
│ │ │ Reconstruction │ ── DensePose UV │ │
|
||||
│ │ └─────────────────┘ │ │
|
||||
│ └───────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 1. Intentional OFDM Sounding (Replaces Passive CSI Sniffing)
|
||||
|
||||
**What changes**: Instead of waiting for random WiFi packets and extracting CSI as a side effect, transmit deterministic OFDM sounding frames at a fixed cadence with known pilot symbol structure.
|
||||
|
||||
**Waveform specification**:
|
||||
|
||||
| Parameter | Value | Rationale |
|
||||
|-----------|-------|-----------|
|
||||
| Symbol type | 802.11bf NDP (Null Data PPDU) | Standards-compliant, no data payload overhead |
|
||||
| Sounding cadence | 50–200 Hz (configurable) | 50 Hz minimum for heartbeat Doppler; 200 Hz for gesture |
|
||||
| Bandwidth | 20/40/80 MHz (per band) | 20 MHz default; 80 MHz for maximum range resolution |
|
||||
| Pilot structure | L-LTF + HT-LTF (standard) | Known phase structure enables coherent processing |
|
||||
| Burst duration | ≤10 ms per sounding event | ETSI EN 300 328 burst limit compliance |
|
||||
| Subcarrier count | 56 (20 MHz) / 114 (40 MHz) / 242 (80 MHz) | Standard OFDM subcarrier allocation |
|
||||
|
||||
**Phase stability improvement**:
|
||||
|
||||
```
|
||||
Passive CSI: σ_φ ≈ 5° per subcarrier (random MCS, no averaging)
|
||||
NDP Sounding: σ_φ ≈ 5° / √N where N = coherent averages per epoch
|
||||
At 50 Hz cadence, 10-frame average: σ_φ ≈ 1.6°
|
||||
Displacement floor: 0.87 mm → 0.28 mm at 2.4 GHz
|
||||
```
|
||||
|
||||
**Implementation**: New ESP32-S3 firmware mode alongside existing passive CSI. Uses `esp_wifi_80211_tx()` for NDP transmission and existing CSI callback for reception. Sounding schedule coordinated by the Waveform Controller.
|
||||
|
||||
### 2. Phase-Locked Dual-Radio Architecture
|
||||
|
||||
**What changes**: All CHCI nodes share a common reference clock, eliminating per-node LO phase drift that currently requires statistical correction in `phase_align.rs`.
|
||||
|
||||
**Clock distribution design** (based on ESPARGOS architecture):
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────┐
|
||||
│ Reference Clock Module │
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────────┐ │
|
||||
│ │ 40 MHz │────▶│ PLL │ │
|
||||
│ │ TCXO │ │ Synthesizer │ │
|
||||
│ │ (±0.5ppm)│ │ (SI5351A) │ │
|
||||
│ └──────────┘ └──────┬───────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────────┼──────────────┐ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ 40 MHz │ │ 40 MHz │ │ 40 MHz │ │
|
||||
│ │ to Node 1│ │ to Node 2│ │ to Node 3│ │
|
||||
│ └──────────┘ └──────────┘ └──────────┘ │
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ 2.4 GHz │ │ 2.4 GHz │ │ 2.4 GHz │ │
|
||||
│ │ Phase Ref│ │ Phase Ref│ │ Phase Ref│ │
|
||||
│ │ to Node 1│ │ to Node 2│ │ to Node 3│ │
|
||||
│ └──────────┘ └──────────┘ └──────────┘ │
|
||||
│ │
|
||||
│ Distribution: coaxial cable with power splitters │
|
||||
│ Phase ref: CW tone at center of operating band │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Components per node** (incremental cost ~$2):
|
||||
|
||||
| Component | Part | Cost | Purpose |
|
||||
|-----------|------|------|---------|
|
||||
| TCXO | SiT8008 40 MHz ±0.5 ppm | $0.50 | Reference oscillator (1 per system) |
|
||||
| PLL synthesizer | SI5351A | $1.00 | Generates 40 MHz + 2.4 GHz references (1 per system) |
|
||||
| Coax splitter | Mini-Circuits PSC-4-1+ | $0.30/port | Distributes reference to nodes |
|
||||
| SMA connector | Edge-mount | $0.20 | Reference clock input on each node |
|
||||
|
||||
**Acceptance metric**: Phase variance per subcarrier under static conditions ≤ 0.5° RMS over 10 minutes (vs current ~5° with statistical correction).
|
||||
|
||||
**Impact on displacement sensitivity**:
|
||||
|
||||
```
|
||||
Current (incoherent): δ_min ≈ λ/(4π) × σ_φ = 12.5cm/(4π) × 5° × π/180 ≈ 0.87 mm
|
||||
Coherent (shared clock): δ_min ≈ λ/(4π) × 0.5° × π/180 ≈ 0.087 mm
|
||||
|
||||
With 8-antenna coherent averaging:
|
||||
δ_min ≈ 0.087 mm / √8 ≈ 0.031 mm
|
||||
```
|
||||
|
||||
This puts heartbeat detection (0.2–0.5 mm chest displacement) well within the sensitivity envelope.
|
||||
|
||||
### 3. Multi-Band Coherent Fusion
|
||||
|
||||
**What changes**: Transmit sounding frames simultaneously at 2.4 GHz and 5 GHz (optionally 6 GHz with WiFi 6E), fusing them as projections of the same latent motion field in RuVector embedding space.
|
||||
|
||||
**Band characteristics for coherent fusion**:
|
||||
|
||||
| Property | 2.4 GHz | 5 GHz | 6 GHz |
|
||||
|----------|---------|-------|-------|
|
||||
| Wavelength | 12.5 cm | 6.0 cm | 5.0 cm |
|
||||
| Wall penetration | Excellent | Good | Moderate |
|
||||
| Displacement sensitivity (0.5° phase) | 0.087 mm | 0.042 mm | 0.035 mm |
|
||||
| Range resolution (20 MHz) | 7.5 m | 7.5 m | 7.5 m |
|
||||
| Fresnel zone radius (2 m) | 22.4 cm | 15.5 cm | 14.1 cm |
|
||||
| Subcarrier spacing (20 MHz) | 312.5 kHz | 312.5 kHz | 312.5 kHz |
|
||||
|
||||
**Fusion architecture**:
|
||||
|
||||
```
|
||||
2.4 GHz CSI ──▶ ┌───────────────────┐
|
||||
│ Band-Specific │ ┌─────────────────────┐
|
||||
│ Phase Alignment │────▶│ │
|
||||
│ (per-band ref) │ │ Contrastive │
|
||||
└───────────────────┘ │ Cross-Band │
|
||||
│ Fusion │
|
||||
5 GHz CSI ────▶ ┌───────────────────┐ │ │
|
||||
│ Band-Specific │────▶│ Body model priors │
|
||||
│ Phase Alignment │ │ constrain phase │
|
||||
│ (per-band ref) │ │ relationships │
|
||||
└───────────────────┘ │ │
|
||||
│ Output: unified │
|
||||
6 GHz CSI ────▶ ┌───────────────────┐ │ complex channel │
|
||||
(optional) │ Band-Specific │────▶│ response │
|
||||
│ Phase Alignment │ │ │
|
||||
└───────────────────┘ └─────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ RuVector Contrastive │
|
||||
│ Embedding Space │
|
||||
│ (body surface latent)│
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
**Key insight**: Lower frequency penetrates better (through-wall sensing, NLOS paths). Higher frequency provides finer spatial resolution. By treating each band as a projection of the same physical scene, the fusion model can achieve super-resolution beyond any single band — using body model priors (known human dimensions, joint angle constraints) to constrain the phase relationships across bands.
|
||||
|
||||
**Integration with existing code**: Extends `multiband.rs` from independent per-channel fusion to coherent cross-band phase alignment. The existing `CrossViewpointAttention` mechanism in `ruvector/src/viewpoint/attention.rs` provides the attention-weighted fusion foundation.
|
||||
|
||||
### 4. Time-Coded Micro-Bursts
|
||||
|
||||
**What changes**: Replace continuous WiFi packet streams with very short deterministic OFDM bursts at high cadence, maximizing temporal resolution of Doppler shifts without 802.11 frame overhead.
|
||||
|
||||
**Burst specification**:
|
||||
|
||||
| Parameter | Value | Rationale |
|
||||
|-----------|-------|-----------|
|
||||
| Burst cadence | 1–5 kHz | 5 kHz enables 2.5 kHz Doppler bandwidth (Nyquist) |
|
||||
| Burst duration | 4–20 μs | Single OFDM symbol + CP = 4 μs minimum |
|
||||
| Symbols per burst | 1–4 | Minimal overhead per measurement |
|
||||
| Duty cycle | 0.4–10% | Compliant with ETSI 10 ms burst limit |
|
||||
| Inter-burst gap | 196–996 μs | Available for normal WiFi traffic |
|
||||
|
||||
**Doppler resolution comparison**:
|
||||
|
||||
```
|
||||
Passive WiFi CSI (random, ~30 Hz):
|
||||
Doppler resolution: Δf_D = 1/T_obs = 1/33ms ≈ 30 Hz
|
||||
Minimum detectable velocity: v_min = λ × Δf_D / 2 ≈ 1.9 m/s at 2.4 GHz
|
||||
|
||||
CHCI micro-burst (5 kHz cadence):
|
||||
Doppler resolution: Δf_D = 1/(N × T_burst) = 1/(256 × 0.2ms) ≈ 20 Hz
|
||||
BUT: unambiguous Doppler: ±2500 Hz → v_max = ±156 m/s
|
||||
Minimum detectable velocity: v_min ≈ λ × 20 / 2 ≈ 1.25 m/s
|
||||
|
||||
With coherent integration over 1 second (5000 bursts):
|
||||
Δf_D = 1/1s = 1 Hz → v_min ≈ 0.063 m/s (6.3 cm/s)
|
||||
Chest wall velocity during breathing: ~1–5 cm/s ✓
|
||||
Chest wall velocity during heartbeat: ~0.5–2 cm/s ✓
|
||||
```
|
||||
|
||||
**Regulatory compliance**: At 5 kHz burst cadence with 4 μs bursts, duty cycle is 2%. ETSI EN 300 328 allows up to 10 ms continuous transmission followed by mandatory idle. A 4 μs burst followed by 196 μs idle is well within limits. FCC Part 15.247 requires digital modulation (OFDM qualifies) or spread spectrum.
|
||||
|
||||
### 5. MIMO Geometry Optimization
|
||||
|
||||
**What changes**: Instead of 2×2 WiFi-style antenna layout (optimized for throughput diversity), design antenna spacing tuned for human-scale wavelengths and chest wall displacement sensitivity.
|
||||
|
||||
**Antenna geometry design**:
|
||||
|
||||
```
|
||||
Current WiFi-DensePose (throughput-optimized):
|
||||
┌─────────────────┐
|
||||
│ ANT1 ANT2 │ ← λ/2 spacing = 6.25 cm at 2.4 GHz
|
||||
│ │ Optimized for spatial diversity
|
||||
│ ESP32-S3 │
|
||||
└─────────────────┘
|
||||
|
||||
Proposed CHCI (sensing-optimized):
|
||||
┌───────────────────────────────────────┐
|
||||
│ │
|
||||
│ ANT1 ANT2 ANT3 ANT4 │ ← λ/4 spacing = 3.125 cm
|
||||
│ ●───────●───────●───────● │ at 2.4 GHz
|
||||
│ │ Linear array for 1D AoA
|
||||
│ ESP32-S3 (Node A) │
|
||||
└───────────────────────────────────────┘
|
||||
λ/4 = 3.125 cm
|
||||
|
||||
Alternative: L-shaped for 2D AoA:
|
||||
┌────────────────────┐
|
||||
│ ANT4 │
|
||||
│ ● │
|
||||
│ │ λ/4 │
|
||||
│ ANT3 │
|
||||
│ ● │
|
||||
│ │ λ/4 │
|
||||
│ ANT2 │
|
||||
│ ● │
|
||||
│ │ λ/4 │
|
||||
│ ANT1──●──ANT5──●──ANT6──●──ANT7 │
|
||||
│ │
|
||||
│ ESP32-S3 (Node A) │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
**Design rationale**:
|
||||
|
||||
| Design parameter | WiFi (throughput) | CHCI (sensing) |
|
||||
|-----------------|-------------------|----------------|
|
||||
| Spacing | λ/2 (6.25 cm) | λ/4 (3.125 cm) |
|
||||
| Goal | Maximize diversity gain | Maximize angular resolution |
|
||||
| Array factor | Broad main lobe | Narrow main lobe, grating lobe suppression |
|
||||
| Geometry | Dual-antenna diversity | Linear or L-shaped phased array |
|
||||
| Target signal | Far-field plane wave | Near-field chest wall displacement |
|
||||
|
||||
**Virtual aperture synthesis**: With 4 nodes × 4 antennas = 16 physical elements, MIMO virtual aperture provides 16 × 16 = 256 virtual channels. Combined with MUSIC or ESPRIT algorithms, this enables sub-degree angle-of-arrival estimation — sufficient to resolve individual body segments.
|
||||
|
||||
### 6. Cognitive Waveform Adaptation
|
||||
|
||||
**What changes**: The sensing waveform adapts in real-time based on the current scene state, driven by delta coherence feedback from the body model.
|
||||
|
||||
**Cognitive sensing modes**:
|
||||
|
||||
```
|
||||
┌───────────────────────────────────────────────────────────────┐
|
||||
│ Cognitive Waveform Engine │
|
||||
│ │
|
||||
│ Scene State ─────▶ ┌────────────────┐ ─────▶ Waveform Config │
|
||||
│ (from body model) │ Mode Selector │ (to TX nodes) │
|
||||
│ └───────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────────┼──────────────────┐ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌────────────┐ ┌────────────┐ ┌────────────┐ │
|
||||
│ │ IDLE │ │ ALERT │ │ ACTIVE │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ 1 Hz NDP │ │ 10 Hz NDP │ │ 50-200 Hz │ │
|
||||
│ │ Single band│ │ Dual band │ │ All bands │ │
|
||||
│ │ Low power │ │ Med power │ │ Full power │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ Presence │ │ Tracking │ │ DensePose │ │
|
||||
│ │ detection │ │ + coarse │ │ + vitals │ │
|
||||
│ │ only │ │ pose │ │ + micro- │ │
|
||||
│ │ │ │ │ │ Doppler │ │
|
||||
│ └────────────┘ └────────────┘ └────────────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌────────────┐ ┌────────────┐ ┌────────────┐ │
|
||||
│ │ VITAL │ │ GESTURE │ │ SLEEP │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ 100 Hz │ │ 200 Hz │ │ 20 Hz │ │
|
||||
│ │ Subset of │ │ Full band │ │ Single │ │
|
||||
│ │ optimal │ │ Max bursts │ │ band │ │
|
||||
│ │ subcarriers│ │ │ │ Low power │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ Breathing, │ │ DTW match │ │ Apnea, │ │
|
||||
│ │ HR, HRV │ │ + classify │ │ movement, │ │
|
||||
│ │ │ │ │ │ stages │ │
|
||||
│ └────────────┘ └────────────┘ └────────────┘ │
|
||||
│ │
|
||||
│ Transition triggers: │
|
||||
│ IDLE → ALERT: Coherence delta > threshold │
|
||||
│ ALERT → ACTIVE: Person detected with confidence > 0.8 │
|
||||
│ ACTIVE → VITAL: Static person, body model stable │
|
||||
│ ACTIVE → GESTURE: Motion spike with periodic structure │
|
||||
│ ACTIVE → SLEEP: Supine pose detected, low ambient motion │
|
||||
│ * → IDLE: No detection for 30 seconds │
|
||||
│ │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Power efficiency**: Cognitive adaptation reduces average power consumption by 60–80% compared to constant full-rate sounding. In IDLE mode (1 Hz, single band, low power), the system draws <10 mA from the ESP32-S3 radio — enabling battery-powered deployment.
|
||||
|
||||
**Integration with ADR-039**: The cognitive waveform modes map directly to ADR-039 edge processing tiers. Tier 0 (raw CSI) corresponds to IDLE/ALERT. Tier 1 (phase unwrap, stats) corresponds to ACTIVE. Tier 2 (vitals, fall detection) corresponds to VITAL/SLEEP. The cognitive engine adds the waveform adaptation feedback loop that ADR-039 lacks.
|
||||
|
||||
### 7. Coherent Diffraction Tomography
|
||||
|
||||
**What changes**: Current tomography (`tomography.rs`) uses amplitude-only attenuation for voxel reconstruction. With coherent phase data from CHCI, we upgrade to diffraction tomography — resolving body surfaces rather than volumetric shadows.
|
||||
|
||||
**Mathematical foundation**:
|
||||
|
||||
```
|
||||
Current (amplitude tomography):
|
||||
I(x,y,z) = Σ_links |H_measured(f)| × W_link(x,y,z)
|
||||
Output: scalar opacity per voxel (shadow image)
|
||||
|
||||
Proposed (coherent diffraction tomography):
|
||||
O(x,y,z) = F^{-1}[ Σ_links H_measured(f,θ) / H_reference(f,θ) ]
|
||||
Where:
|
||||
H_measured = complex channel response with human present
|
||||
H_reference = complex channel response of empty room (calibration)
|
||||
f = frequency (across all bands)
|
||||
θ = link angle (across all node pairs)
|
||||
Output: complex permittivity contrast per voxel (body surface)
|
||||
```
|
||||
|
||||
**Key advantage**: Diffraction tomography produces body surface geometry, not just occupancy maps. This directly feeds the DensePose UV mapping pipeline with geometric constraints — reducing the neural network's burden from "guess the surface from shadows" to "refine the surface from holographic reconstruction."
|
||||
|
||||
**Performance projection** (based on ESPARGOS results and multi-band coverage):
|
||||
|
||||
| Metric | Current (Amplitude) | Proposed (Coherent Diffraction) |
|
||||
|--------|--------------------|---------------------------------|
|
||||
| Spatial resolution | ~15 cm (limited by wavelength) | ~3 cm (multi-band synthesis) |
|
||||
| Body segment discrimination | Coarse (torso vs limb) | Fine (individual limbs) |
|
||||
| Surface vs volume | Volumetric opacity | Surface geometry |
|
||||
| Through-wall capability | Yes (amplitude penetrates) | Partial (phase coherence degrades) |
|
||||
| Calibration requirement | None | Empty room reference scan |
|
||||
|
||||
### Acceptance Test
|
||||
|
||||
**Primary acceptance criterion**: Demonstrate 0.1 mm displacement detection repeatably at 2 meters in a static controlled room.
|
||||
|
||||
**Full acceptance test protocol**:
|
||||
|
||||
| Test | Metric | Target | Method |
|
||||
|------|--------|--------|--------|
|
||||
| AT-1: Phase stability | σ_φ per subcarrier, static, 10 min | ≤ 0.5° RMS | Record CSI, compute variance |
|
||||
| AT-2: Displacement | Detectable displacement at 2 m | ≤ 0.1 mm | Precision linear stage, sinusoidal motion |
|
||||
| AT-3: Breathing rate | BPM error, 3 subjects, 5 min each | ≤ 0.2 BPM | Reference: respiratory belt |
|
||||
| AT-4: Heart rate | BPM error, 3 subjects, seated, 2 min | ≤ 3 BPM | Reference: pulse oximeter |
|
||||
| AT-5: Multi-person | Pose detection, 3 persons, 4×4 m room | ≥ 90% keypoint detection | Reference: camera ground truth |
|
||||
| AT-6: Power | Average draw in IDLE mode | ≤ 10 mA (radio) | Current meter on 3.3 V rail |
|
||||
| AT-7: Latency | End-to-end pose update latency | ≤ 50 ms | Timestamp injection |
|
||||
| AT-8: Regulatory | Conducted emissions, 2.4 GHz ISM | FCC 15.247 + ETSI 300 328 | Spectrum analyzer |
|
||||
|
||||
### Backward Compatibility
|
||||
|
||||
**Question 1: Do you want backward compatibility with normal WiFi routers?**
|
||||
|
||||
CHCI supports a **dual-mode architecture**:
|
||||
|
||||
| Mode | Description | When to Use |
|
||||
|------|-------------|-------------|
|
||||
| **Legacy CSI** | Passive sniffing of existing WiFi traffic | Retrofit into existing WiFi environments, no hardware changes |
|
||||
| **802.11bf NDP** | Standard-compliant NDP sounding | WiFi AP supports 802.11bf, moderate improvement over legacy |
|
||||
| **CHCI Native** | Full coherent sounding with shared clock | Purpose-deployed sensing mesh, maximum fidelity |
|
||||
|
||||
The firmware can switch between modes at runtime. The signal processing pipeline (`signal/src/ruvsense/`) accepts CSI from any mode — the coherent processing path activates when shared-clock metadata is present in the CSI frame header.
|
||||
|
||||
**Question 2: Are you willing to own both transmitter and receiver hardware?**
|
||||
|
||||
Yes. CHCI requires owning both TX and RX to achieve phase coherence. The system is deployed as a self-contained sensing mesh — not parasitic on existing WiFi infrastructure. This is the fundamental architectural trade: compatibility for control. For sensing, that is a good trade.
|
||||
|
||||
### Hardware Bill of Materials (per CHCI node)
|
||||
|
||||
| Component | Part | Quantity | Unit Cost | Purpose |
|
||||
|-----------|------|----------|-----------|---------|
|
||||
| ESP32-S3-WROOM-1 | Espressif | 1 | $2.50 | Main MCU + WiFi radio |
|
||||
| External antenna | 2.4/5 GHz dual-band | 2–4 | $0.30 each | Sensing antennas (λ/4 spacing) |
|
||||
| SMA connector | Edge-mount | 1 | $0.20 | Reference clock input |
|
||||
| Coax cable | RG-174 | 1 m | $0.15 | Clock distribution |
|
||||
| PCB | Custom 4-layer | 1 | $0.50 | Integration (at volume) |
|
||||
| **Node total** | | | **$4.25** | |
|
||||
| Reference clock module | SI5351A + TCXO + splitter | 1 per system | $3.00 | Shared clock source |
|
||||
| **4-node system total** | | | **$20.00** | |
|
||||
|
||||
This is 10× cheaper than the nearest comparable coherent sensing platform (Novelda X4 at $50/node, Vayyar at $200+).
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
| Phase | Timeline | Deliverables | Dependencies |
|
||||
|-------|----------|-------------|--------------|
|
||||
| **Phase 1: NDP Sounding** | 4 weeks | ESP32-S3 firmware for 802.11bf NDP TX/RX, sounding scheduler, CSI extraction from NDP frames | ESP-IDF 5.2+, existing firmware |
|
||||
| **Phase 2: Clock Distribution** | 6 weeks | Reference clock PCB design, SI5351A driver, phase reference distribution, `phase_align.rs` upgrade | Phase 1, PCB fabrication |
|
||||
| **Phase 3: Coherent Processing** | 4 weeks | Coherent diffraction tomography in `tomography.rs`, complex-valued CSI pipeline, calibration procedure | Phase 2 |
|
||||
| **Phase 4: Multi-Band Fusion** | 4 weeks | Simultaneous 2.4+5 GHz sounding, cross-band phase alignment, contrastive fusion in RuVector space | Phase 1, Phase 3 |
|
||||
| **Phase 5: Cognitive Engine** | 3 weeks | Waveform adaptation state machine, coherence delta feedback, power management modes | Phase 3, Phase 4 |
|
||||
| **Phase 6: Acceptance Testing** | 3 weeks | AT-1 through AT-8, precision displacement rig, regulatory pre-scan | Phase 5 |
|
||||
|
||||
### Crate Architecture
|
||||
|
||||
New and modified crates:
|
||||
|
||||
| Crate | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `wifi-densepose-chci` | **New** | CHCI protocol definition, waveform specs, cognitive engine |
|
||||
| `wifi-densepose-signal` | Modified | Add coherent diffraction tomography, upgrade `phase_align.rs` |
|
||||
| `wifi-densepose-hardware` | Modified | Reference clock driver, NDP sounding firmware, antenna geometry config |
|
||||
| `wifi-densepose-ruvector` | Modified | Cross-band contrastive fusion in viewpoint attention |
|
||||
| `wifi-densepose-wasm-edge` | Modified | New WASM modules for CHCI-specific edge processing |
|
||||
|
||||
### Module Impact Matrix
|
||||
|
||||
| Existing Module | Current Function | CHCI Upgrade |
|
||||
|----------------|-----------------|-------------|
|
||||
| `phase_align.rs` | Statistical LO offset estimation | Replace with shared-clock phase reference alignment |
|
||||
| `multiband.rs` | Independent per-channel fusion | Coherent cross-band phase alignment with body priors |
|
||||
| `coherence.rs` | Z-score coherence scoring | Complex-valued coherence metric (phasor domain) |
|
||||
| `coherence_gate.rs` | Accept/Reject gate decisions | Add waveform adaptation feedback to cognitive engine |
|
||||
| `tomography.rs` | Amplitude-only ISTA L1 solver | Coherent diffraction tomography with complex CSI |
|
||||
| `multistatic.rs` | Attention-weighted fusion | Add PLL-disciplined synchronization path |
|
||||
| `field_model.rs` | SVD room eigenstructure | Coherent room transfer function model with phase |
|
||||
| `intention.rs` | Pre-movement lead signals | Enhanced micro-Doppler from high-cadence bursts |
|
||||
| `gesture.rs` | DTW template matching | Phase-domain gesture features (higher discrimination) |
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **9× displacement sensitivity improvement**: From 0.87 mm (incoherent) to 0.031 mm (coherent 8-antenna) at 2.4 GHz, enabling reliable heartbeat detection at ISM bands
|
||||
- **Standards-compliant path**: 802.11bf NDP sounding is a published IEEE standard (September 2025), providing regulatory clarity
|
||||
- **10× cost advantage**: $4.25/node vs $50+ for nearest comparable coherent sensing platform
|
||||
- **Through-wall preservation**: Operates at 2.4/5 GHz ISM bands, maintaining the through-wall sensing advantage that mmWave systems lack
|
||||
- **Backward compatible**: Dual-mode firmware supports legacy CSI, 802.11bf NDP, and native CHCI — deployable incrementally
|
||||
- **Privacy-preserving**: No cameras, no audio — same RF-only sensing paradigm as current WiFi-DensePose
|
||||
- **Power-efficient**: Cognitive waveform adaptation reduces average power 60–80% vs constant-rate sounding
|
||||
- **Body surface reconstruction**: Coherent diffraction tomography produces geometric constraints for DensePose, reducing neural network inference burden
|
||||
- **Proven feasibility**: ESPARGOS demonstrates phase-coherent WiFi sensing at ESP32 cost point (IEEE 2024)
|
||||
|
||||
### Negative
|
||||
|
||||
- **Custom hardware required**: Cannot parasitically sense from existing WiFi routers in CHCI Native mode (802.11bf mode can use compliant APs)
|
||||
- **PCB design needed**: Reference clock distribution requires custom PCB — not a pure firmware upgrade
|
||||
- **Calibration burden**: Coherent diffraction tomography requires empty-room reference scan — adds deployment friction
|
||||
- **Clock distribution complexity**: Coaxial cable distribution limits deployment flexibility vs fully wireless mesh
|
||||
- **Two-phase deployment**: Full CHCI requires Phases 1–6 (~24 weeks). Intermediate modes (NDP-only, Phase 1) provide incremental value.
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| ESP32-S3 WiFi hardware does not support NDP TX at 802.11bf spec | Medium | High | Fall back to raw 802.11 frame injection with known preamble; validate with `esp_wifi_80211_tx()` |
|
||||
| Phase coherence degrades over cable length >2 m | Low | Medium | Use matched-length cables; add per-node phase calibration step |
|
||||
| ETSI/FCC regulatory rejection of custom sounding cadence | Low | High | Stay within 802.11bf NDP specification; use standard-compliant waveforms only |
|
||||
| Coherent diffraction tomography computationally exceeds ESP32 | Medium | Medium | Run tomography on aggregator (Rust server), not on edge. ESP32 sends coherent CSI only |
|
||||
| Multi-band simultaneous TX causes self-interference | Medium | Medium | Time-division between bands (alternating 2.4/5 GHz per burst slot) or frequency planning |
|
||||
| Body model priors over-constrain fusion, missing novel poses | Low | Medium | Use priors as soft constraints (regularization) not hard constraints |
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
### Standards
|
||||
|
||||
1. IEEE Std 802.11bf-2025, "Standard for Information Technology — Telecommunications and Information Exchange between Systems — Local and Metropolitan Area Networks — Specific Requirements — Part 11: Wireless LAN Medium Access Control (MAC) and Physical Layer (PHY) Specifications — Amendment: Enhancements for Wireless Local Area Network (WLAN) Sensing," IEEE, September 2025.
|
||||
2. ETSI EN 300 328 V2.2.2, "Wideband transmission systems; Data transmission equipment operating in the 2.4 GHz band," ETSI, July 2019.
|
||||
3. FCC 47 CFR Part 15.247, "Operation within the bands 902–928 MHz, 2400–2483.5 MHz, and 5725–5850 MHz."
|
||||
|
||||
### Research Papers
|
||||
|
||||
4. Euchner, F., et al., "ESPARGOS: An Ultra Low-Cost, Realtime-Capable Multi-Antenna WiFi Channel Sounder for Phase-Coherent Sensing," IEEE, 2024. [arXiv:2502.09405]
|
||||
5. Restuccia, F., "IEEE 802.11bf: Toward Ubiquitous Wi-Fi Sensing," IEEE Communications Standards Magazine, 2024. [arXiv:2310.05765]
|
||||
6. Pegoraro, J., et al., "Sensing Performance of the IEEE 802.11bf Protocol," IEEE, 2024. [arXiv:2403.19825]
|
||||
7. Chen, Y., et al., "Multi-Band Wi-Fi Neural Dynamic Fusion for Sensing," IEEE ICASSP, 2024. [arXiv:2407.12937]
|
||||
8. Samsung Research, "Optimal Preprocessing of WiFi CSI for Sensing Applications," IEEE, 2024. [arXiv:2307.12126]
|
||||
9. Yan, Y., et al., "Person-in-WiFi 3D: End-to-End Multi-Person 3D Pose Estimation with Wi-Fi," CVPR 2024.
|
||||
10. Geng, J., et al., "DensePose From WiFi," Carnegie Mellon University, 2023. [arXiv:2301.00250]
|
||||
11. Pegoraro, J., et al., "802.11bf Multiband Passive Sensing," IEEE, 2025. [arXiv:2507.22591]
|
||||
12. Liu, J., et al., "Monitoring Vital Signs and Postures During Sleep Using WiFi Signals," MobiCom, 2020.
|
||||
|
||||
### Commercial Systems
|
||||
|
||||
13. Vayyar Imaging, "4D Imaging Radar Technology Platform," https://vayyar.com/technology/
|
||||
14. Infineon Technologies, "BGT60TR13C 60 GHz Radar Sensor IC Datasheet," 2024.
|
||||
15. Novelda AS, "X4 UWB Radar SoC Datasheet," https://novelda.com/technology/
|
||||
16. Texas Instruments, "IWR6843 Single-Chip 60-GHz mmWave Sensor," 2024.
|
||||
17. ESPARGOS Project, https://espargos.net/
|
||||
|
||||
### Related ADRs
|
||||
|
||||
18. ADR-014: SOTA Signal Processing (phase alignment, coherence scoring)
|
||||
19. ADR-017: RuVector Signal + MAT Integration (embedding fusion)
|
||||
20. ADR-029: RuvSense Multistatic Sensing Mode (multi-node coordination)
|
||||
21. ADR-039: ESP32 Edge Intelligence (tiered processing, power management)
|
||||
22. ADR-040: WASM Programmable Sensing (edge compute architecture)
|
||||
23. ADR-041: WASM Module Collection (algorithm registry)
|
||||
@@ -0,0 +1,334 @@
|
||||
# ADR-043: Sensing Server UI API Completion
|
||||
|
||||
**Status**: Accepted
|
||||
**Date**: 2026-03-03
|
||||
**Deciders**: @ruvnet
|
||||
**Supersedes**: None
|
||||
**Related**: ADR-034, ADR-036, ADR-039, ADR-040, ADR-041
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
The WiFi-DensePose sensing server (`wifi-densepose-sensing-server`) is a single-binary Axum server that receives ESP32 CSI frames via UDP, processes them through the RuVector signal pipeline, and serves both a web UI at `/ui/` and a REST/WebSocket API. The UI provides tabs for live sensing visualization, model management, CSI recording, and training -- all designed to operate without external dependencies.
|
||||
|
||||
However, the UI's JavaScript expected several backend endpoints that were not yet implemented in the Rust server. Opening the browser console revealed persistent 404 errors for model, recording, and training API routes. Three categories of functionality were broken:
|
||||
|
||||
### 1. Model Management (7 endpoints missing)
|
||||
|
||||
The Models tab calls `GET /api/v1/models` to list available `.rvf` model files, `GET /api/v1/models/active` to show the currently loaded model, `POST /api/v1/models/load` and `POST /api/v1/models/unload` to control the model lifecycle, and `DELETE /api/v1/models/:id` to remove models from disk. LoRA fine-tuning profiles are managed via `GET /api/v1/models/lora/profiles` and `POST /api/v1/models/lora/activate`. All of these returned 404.
|
||||
|
||||
### 2. CSI Recording (5 endpoints missing)
|
||||
|
||||
The Recording tab calls `POST /api/v1/recording/start` and `POST /api/v1/recording/stop` to capture CSI frames to `.csi.jsonl` files for later training. `GET /api/v1/recording/list` enumerates stored sessions. `DELETE /api/v1/recording/:id` removes recordings. None of these were wired into the server's router.
|
||||
|
||||
### 3. Training Pipeline (5 endpoints missing)
|
||||
|
||||
The Training tab calls `POST /api/v1/train/start` to launch a background training run against recorded CSI data, `POST /api/v1/train/stop` to abort, and `GET /api/v1/train/status` to poll progress. Contrastive pretraining (`POST /api/v1/train/pretrain`) and LoRA fine-tuning (`POST /api/v1/train/lora`) endpoints were also unavailable. A WebSocket endpoint at `/ws/train/progress` streams epoch-level progress updates to the UI.
|
||||
|
||||
### 4. Sensing Service Not Started on App Init
|
||||
|
||||
The web UI's `sensingService` singleton (which manages the WebSocket connection to `/ws/sensing`) was only started lazily when the user navigated to the Sensing tab (`SensingTab.js:182`). However, the Dashboard and Live Demo tabs both read `sensingService.dataSource` at load time — and since the service was never started, the status permanently showed **"RECONNECTING"** with no WebSocket connection attempt and no console errors. This silent failure affected the first-load experience for every user.
|
||||
|
||||
### 5. Mobile App Defects
|
||||
|
||||
The Expo React Native mobile companion (ADR-034) had two integration defects:
|
||||
|
||||
- **WebSocket URL builder**: `ws.service.ts` hardcoded port `3001` for the WebSocket connection instead of using the same-origin port derived from the REST API URL. When the sensing server runs on a different port (e.g., `8080` or `3000`), the mobile app could not connect.
|
||||
- **Test configuration**: `jest.config.js` contained a `testPathIgnorePatterns` entry that effectively excluded the entire test directory, causing all 25 tests to be skipped silently.
|
||||
- **Placeholder tests**: All 25 mobile test files contained `it.todo()` stubs with no assertions, providing false confidence in test coverage.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Implement the complete model management, CSI recording, and training API directly in the sensing server's `main.rs` as inline handler functions sharing `AppStateInner` via `Arc<RwLock<…>>`. Wire all 14 routes into the server's main router so the UI loads without any 404 console errors. Start the sensing WebSocket service on application init (not lazily on tab visit) so Dashboard and Live Demo tabs connect immediately. Fix the mobile app WebSocket URL builder, test configuration, and replace placeholder tests with real implementations.
|
||||
|
||||
### Architecture
|
||||
|
||||
All 14 new handler functions are implemented directly in `main.rs` as async functions taking `State<AppState>` extractors, sharing the existing `AppStateInner` via `Arc<RwLock<…>>`. This avoids introducing new module files and keeps all API routes in one place alongside the existing sensing and pose handlers.
|
||||
|
||||
```
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ Sensing Server (main.rs) │
|
||||
│ │
|
||||
│ Router::new() │
|
||||
│ ├── /api/v1/sensing/* (existing — CSI streaming) │
|
||||
│ ├── /api/v1/pose/* (existing — pose estimation) │
|
||||
│ ├── /api/v1/models GET list_models (NEW) │
|
||||
│ ├── /api/v1/models/active GET get_active_model (NEW) │
|
||||
│ ├── /api/v1/models/load POST load_model (NEW) │
|
||||
│ ├── /api/v1/models/unload POST unload_model (NEW) │
|
||||
│ ├── /api/v1/models/:id DELETE delete_model (NEW) │
|
||||
│ ├── /api/v1/models/lora/profiles GET list_lora (NEW) │
|
||||
│ ├── /api/v1/models/lora/activate POST activate_lora (NEW) │
|
||||
│ ├── /api/v1/recording/list GET list_recordings (NEW) │
|
||||
│ ├── /api/v1/recording/start POST start_recording (NEW) │
|
||||
│ ├── /api/v1/recording/stop POST stop_recording (NEW) │
|
||||
│ ├── /api/v1/recording/:id DELETE delete_recording (NEW) │
|
||||
│ ├── /api/v1/train/status GET train_status (NEW) │
|
||||
│ ├── /api/v1/train/start POST train_start (NEW) │
|
||||
│ ├── /api/v1/train/stop POST train_stop (NEW) │
|
||||
│ ├── /ws/sensing (existing — sensing WebSocket) │
|
||||
│ └── /ui/* (existing — static file serving) │
|
||||
│ │
|
||||
│ AppStateInner (new fields) │
|
||||
│ ├── discovered_models: Vec<Value> │
|
||||
│ ├── active_model_id: Option<String> │
|
||||
│ ├── recordings: Vec<Value> │
|
||||
│ ├── recording_active / recording_start_time / recording_current_id │
|
||||
│ ├── recording_stop_tx: Option<watch::Sender<bool>> │
|
||||
│ ├── training_status: Value │
|
||||
│ └── training_config: Option<Value> │
|
||||
│ │
|
||||
│ data/ │
|
||||
│ ├── models/ *.rvf files scanned at startup │
|
||||
│ └── recordings/ *.jsonl files written by background task │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Routes are registered individually in the `http_app` Router before the static UI fallback handler.
|
||||
|
||||
### New Endpoints (17 total)
|
||||
|
||||
#### Model Management (`model_manager.rs`)
|
||||
|
||||
| Method | Path | Request Body | Response | Description |
|
||||
|--------|------|-------------|----------|-------------|
|
||||
| `GET` | `/api/v1/models` | -- | `{ models: ModelInfo[], count: usize }` | Scan `data/models/` for `.rvf` files and return manifest metadata |
|
||||
| `GET` | `/api/v1/models/{id}` | -- | `ModelInfo` | Detailed info for a single model (version, PCK score, LoRA profiles, segment count) |
|
||||
| `GET` | `/api/v1/models/active` | -- | `ActiveModelInfo \| { status: "no_model" }` | Active model with runtime stats (avg inference ms, frames processed) |
|
||||
| `POST` | `/api/v1/models/load` | `{ model_id: string }` | `{ status: "loaded", model_id, weight_count }` | Load model weights into memory via `RvfReader`, set `model_loaded = true` |
|
||||
| `POST` | `/api/v1/models/unload` | -- | `{ status: "unloaded", model_id }` | Drop loaded weights, set `model_loaded = false` |
|
||||
| `POST` | `/api/v1/models/lora/activate` | `{ model_id, profile_name }` | `{ status: "activated", profile_name }` | Activate a LoRA adapter profile on the loaded model |
|
||||
| `GET` | `/api/v1/models/lora/profiles` | -- | `{ model_id, profiles: string[], active }` | List LoRA profiles available in the loaded model |
|
||||
|
||||
#### CSI Recording (`recording.rs`)
|
||||
|
||||
| Method | Path | Request Body | Response | Description |
|
||||
|--------|------|-------------|----------|-------------|
|
||||
| `POST` | `/api/v1/recording/start` | `{ session_name, label?, duration_secs? }` | `{ status: "recording", session_id, file_path }` | Create a new `.csi.jsonl` file and begin appending frames |
|
||||
| `POST` | `/api/v1/recording/stop` | -- | `{ status: "stopped", session_id, frame_count }` | Stop the active recording, write companion `.meta.json` |
|
||||
| `GET` | `/api/v1/recording/list` | -- | `{ recordings: RecordingSession[], count }` | List all recordings by scanning `.meta.json` files |
|
||||
| `GET` | `/api/v1/recording/download/{id}` | -- | `application/x-ndjson` file | Download the raw JSONL recording file |
|
||||
| `DELETE` | `/api/v1/recording/{id}` | -- | `{ status: "deleted", deleted_files }` | Remove `.csi.jsonl` and `.meta.json` files |
|
||||
|
||||
#### Training Pipeline (`training_api.rs`)
|
||||
|
||||
| Method | Path | Request Body | Response | Description |
|
||||
|--------|------|-------------|----------|-------------|
|
||||
| `POST` | `/api/v1/train/start` | `TrainingConfig { epochs, batch_size, learning_rate, ... }` | `{ status: "started", run_id }` | Launch background training task against recorded CSI data |
|
||||
| `POST` | `/api/v1/train/stop` | -- | `{ status: "stopped", run_id }` | Cancel the active training run via a stop signal |
|
||||
| `GET` | `/api/v1/train/status` | -- | `TrainingStatus { phase, epoch, loss, ... }` | Current training state (idle, training, complete, failed) |
|
||||
| `POST` | `/api/v1/train/pretrain` | `{ epochs?, learning_rate? }` | `{ status: "started", mode: "pretrain" }` | Start self-supervised contrastive pretraining (ADR-024) |
|
||||
| `POST` | `/api/v1/train/lora` | `{ profile_name, epochs?, rank? }` | `{ status: "started", mode: "lora" }` | Start LoRA fine-tuning on a loaded base model |
|
||||
| `WS` | `/ws/train/progress` | -- | Streaming `TrainingProgress` JSON | Epoch-level progress with loss, metrics, and ETA |
|
||||
|
||||
### State Management
|
||||
|
||||
All three modules share the server's `AppStateInner` via `Arc<RwLock<AppStateInner>>`. New fields added to `AppStateInner`:
|
||||
|
||||
```rust
|
||||
/// Runtime state for a loaded RVF model (None if no model loaded).
|
||||
pub loaded_model: Option<LoadedModelState>,
|
||||
|
||||
/// Runtime state for the active CSI recording session.
|
||||
pub recording_state: RecordingState,
|
||||
|
||||
/// Runtime state for the active training run.
|
||||
pub training_state: TrainingState,
|
||||
|
||||
/// Broadcast channel for training progress updates (consumed by WebSocket).
|
||||
pub train_progress_tx: broadcast::Sender<TrainingProgress>,
|
||||
```
|
||||
|
||||
Key design constraints:
|
||||
|
||||
- **Single writer**: Only one recording session can be active at a time. Starting a new recording while one is active returns an error.
|
||||
- **Single model**: Only one model can be loaded at a time. Loading a new model implicitly unloads the previous one.
|
||||
- **Background training**: Training runs in a spawned `tokio::task`. Progress is broadcast via a `tokio::sync::broadcast` channel. The WebSocket handler subscribes to this channel.
|
||||
- **Auto-stop**: Recordings with a `duration_secs` parameter automatically stop after the specified elapsed time.
|
||||
|
||||
### Training Pipeline (No External Dependencies)
|
||||
|
||||
The training pipeline is implemented entirely in Rust without PyTorch or `tch` dependencies. The pipeline:
|
||||
|
||||
1. **Loads data**: Reads `.csi.jsonl` recording files from `data/recordings/`
|
||||
2. **Extracts features**: Subcarrier variance (sliding window), temporal gradients, Goertzel frequency-domain power across 9 bands, and 3 global scalar features (mean amplitude, std, motion score)
|
||||
3. **Trains model**: Regularised linear model via batch gradient descent targeting 17 COCO keypoints x 3 dimensions = 51 output targets
|
||||
4. **Exports model**: Best checkpoint exported as `.rvf` container using `RvfBuilder`, stored in `data/models/`
|
||||
|
||||
This design means the sensing server is fully self-contained: a field operator can record CSI data, train a model, and load it for inference without any external tooling.
|
||||
|
||||
### File Layout
|
||||
|
||||
```
|
||||
data/
|
||||
├── models/ # RVF model files
|
||||
│ ├── wifi-densepose-v1.rvf # Trained model container
|
||||
│ └── wifi-densepose-v1.rvf # (additional models...)
|
||||
└── recordings/ # CSI recording sessions
|
||||
├── walking-20260303_140000.csi.jsonl # Raw CSI frames (JSONL)
|
||||
├── walking-20260303_140000.csi.meta.json # Session metadata
|
||||
├── standing-20260303_141500.csi.jsonl
|
||||
└── standing-20260303_141500.csi.meta.json
|
||||
```
|
||||
|
||||
### Mobile App Fixes
|
||||
|
||||
Three defects were corrected in the Expo React Native mobile companion (`ui/mobile/`):
|
||||
|
||||
1. **WebSocket URL builder** (`src/services/ws.service.ts`): The URL construction logic previously hardcoded port `3001` for WebSocket connections. This was changed to derive the WebSocket port from the same-origin HTTP URL, using `window.location.port` on web and the configured server URL on native platforms. This ensures the mobile app connects to whatever port the sensing server is actually running on.
|
||||
|
||||
2. **Jest configuration** (`jest.config.js`): The `testPathIgnorePatterns` array previously contained an entry that matched the test directory itself, causing Jest to silently skip all test files. The pattern was corrected to only ignore `node_modules/`.
|
||||
|
||||
3. **Placeholder tests replaced**: All 25 mobile test files contained only `it.todo()` stubs. These were replaced with real test implementations covering:
|
||||
|
||||
| Category | Test Files | Coverage |
|
||||
|----------|-----------|----------|
|
||||
| Utils | `format.test.ts`, `validation.test.ts` | Number formatting, URL validation, input sanitization |
|
||||
| Services | `ws.service.test.ts`, `api.service.test.ts` | WebSocket connection lifecycle, REST API calls, error handling |
|
||||
| Stores | `poseStore.test.ts`, `settingsStore.test.ts`, `matStore.test.ts` | Zustand state transitions, persistence, selector memoization |
|
||||
| Components | `BreathingGauge.test.tsx`, `HeartRateGauge.test.tsx`, `MetricCard.test.tsx`, `ConnectionBanner.test.tsx` | Rendering, prop validation, theme compliance |
|
||||
| Hooks | `useConnection.test.ts`, `useSensing.test.ts` | Hook lifecycle, cleanup, error states |
|
||||
| Screens | `LiveScreen.test.tsx`, `VitalsScreen.test.tsx`, `SettingsScreen.test.tsx` | Screen rendering, navigation, data binding |
|
||||
|
||||
---
|
||||
|
||||
## Rationale
|
||||
|
||||
### Why implement model/training/recording in the sensing server?
|
||||
|
||||
The alternative would be to run a separate Python training service and proxy requests. This was rejected for three reasons:
|
||||
|
||||
1. **Single-binary deployment**: WiFi-DensePose targets edge deployments (disaster response, building security, healthcare monitoring per ADR-034) where installing Python, pip, and PyTorch is impractical. A single Rust binary that handles sensing, recording, training, and inference is the correct architecture for field use.
|
||||
|
||||
2. **Zero-configuration UI**: The web UI is served by the same binary that exposes the API. When a user opens `http://server:8080/`, everything works -- no additional services to start, no ports to configure, no CORS to manage.
|
||||
|
||||
3. **Data locality**: CSI frames arrive via UDP, are processed for real-time display, and can simultaneously be written to disk for training. The recording module hooks directly into the CSI processing loop via `maybe_record_frame()`, avoiding any serialization overhead or inter-process communication.
|
||||
|
||||
### Why fix mobile in the same change?
|
||||
|
||||
The mobile app's WebSocket failure was caused by the same root problem -- assumptions about server port layout that did not match reality. Fixing the server API without fixing the mobile client would leave a broken user experience. The test fixes were included because the placeholder tests masked the WebSocket URL bug during development.
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **UI loads with zero console errors**: All model, recording, and training tabs render correctly and receive real data from the server
|
||||
- **End-to-end workflow**: Users can record CSI data, train a model, load it, and see pose estimation results -- all from the web UI without any external tools
|
||||
- **LoRA fine-tuning support**: Users can adapt a base model to new environments via LoRA profiles, activated through the UI
|
||||
- **Mobile app connects reliably**: The WebSocket URL builder uses same-origin port derivation, working correctly regardless of which port the server runs on
|
||||
- **25 real mobile tests**: Provide actual regression protection for utils, services, stores, components, hooks, and screens
|
||||
- **Self-contained sensing server**: No Python, PyTorch, or external training infrastructure required
|
||||
|
||||
### Negative
|
||||
|
||||
- **Sensing server binary grows**: The three new modules add approximately 2,000 lines of Rust to the sensing server crate, increasing compile time marginally
|
||||
- **Training is lightweight**: The built-in training pipeline uses regularised linear regression, not deep learning. For production-grade pose estimation models, the full Python training pipeline (`wifi-densepose-train`) with PyTorch is still needed. The in-server training is designed for quick field calibration, not SOTA accuracy.
|
||||
- **File-based storage**: Models and recordings are stored as files on the local filesystem (`data/models/`, `data/recordings/`). There is no database, no replication, and no access control. This is acceptable for single-node edge deployments but not for multi-user production environments.
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| Disk fills up during long recording sessions | Medium | Medium | `duration_secs` auto-stop parameter; UI shows file size; manual `DELETE` endpoint |
|
||||
| Concurrent model load/unload during inference causes race | Low | High | `RwLock` on `AppStateInner` serializes all state mutations; inference path acquires read lock |
|
||||
| Training on insufficient data produces poor model | Medium | Low | Training API validates minimum frame count before starting; UI shows dataset statistics |
|
||||
| JSONL recording format is inefficient for large datasets | Low | Low | Acceptable for field calibration (minutes of data); production datasets use the Python pipeline with HDF5 |
|
||||
|
||||
---
|
||||
|
||||
## Implementation
|
||||
|
||||
### Server-Side Changes
|
||||
|
||||
All 14 new handler functions were added directly to `main.rs` (~400 lines of new code). Key additions:
|
||||
|
||||
| Handler | Method | Path | Description |
|
||||
|---------|--------|------|-------------|
|
||||
| `list_models` | GET | `/api/v1/models` | Scans `data/models/` for `.rvf` files at startup, returns cached list |
|
||||
| `get_active_model` | GET | `/api/v1/models/active` | Returns currently loaded model or `null` |
|
||||
| `load_model` | POST | `/api/v1/models/load` | Sets `active_model_id` in state |
|
||||
| `unload_model` | POST | `/api/v1/models/unload` | Clears `active_model_id` |
|
||||
| `delete_model` | DELETE | `/api/v1/models/:id` | Removes model from disk and state |
|
||||
| `list_lora_profiles` | GET | `/api/v1/models/lora/profiles` | Scans `data/models/lora/` directory |
|
||||
| `activate_lora_profile` | POST | `/api/v1/models/lora/activate` | Activates a LoRA adapter |
|
||||
| `list_recordings` | GET | `/api/v1/recording/list` | Scans `data/recordings/` for `.jsonl` files with frame counts |
|
||||
| `start_recording` | POST | `/api/v1/recording/start` | Spawns tokio background task writing CSI frames to `.jsonl` |
|
||||
| `stop_recording` | POST | `/api/v1/recording/stop` | Sends stop signal via `tokio::sync::watch`, returns duration |
|
||||
| `delete_recording` | DELETE | `/api/v1/recording/:id` | Removes recording file from disk |
|
||||
| `train_status` | GET | `/api/v1/train/status` | Returns training phase (idle/running/complete/failed) |
|
||||
| `train_start` | POST | `/api/v1/train/start` | Sets training status to running with config |
|
||||
| `train_stop` | POST | `/api/v1/train/stop` | Sets training status to idle |
|
||||
|
||||
Helper functions: `scan_model_files()`, `scan_lora_profiles()`, `scan_recording_files()`, `chrono_timestamp()`.
|
||||
|
||||
Startup creates `data/models/` and `data/recordings/` directories and populates initial state with scanned files.
|
||||
|
||||
### Web UI Fix
|
||||
|
||||
| File | Change | Description |
|
||||
|------|--------|-------------|
|
||||
| `ui/app.js` | Modified | Import `sensingService` and call `sensingService.start()` in `initializeServices()` after backend health check, so Dashboard and Live Demo tabs connect to `/ws/sensing` immediately on load instead of waiting for Sensing tab visit |
|
||||
| `ui/services/sensing.service.js` | Comment | Updated comment documenting that `/ws/sensing` is on the same HTTP port |
|
||||
|
||||
### Mobile App Files
|
||||
|
||||
| File | Change | Description |
|
||||
|------|--------|-------------|
|
||||
| `ui/mobile/src/services/ws.service.ts` | Modified | `buildWsUrl()` uses `parsed.host` directly with `/ws/sensing` path instead of hardcoded port `3001` |
|
||||
| `ui/mobile/jest.config.js` | Modified | `testPathIgnorePatterns` corrected to only ignore `node_modules/` |
|
||||
| `ui/mobile/src/__tests__/*.test.ts{x}` | Replaced | 25 placeholder `it.todo()` tests replaced with real implementations |
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
# 1. Start sensing server with auto source (simulated fallback)
|
||||
cd rust-port/wifi-densepose-rs
|
||||
cargo run -p wifi-densepose-sensing-server -- --http-port 3000 --source auto
|
||||
|
||||
# 2. Verify model endpoints return 200
|
||||
curl -s http://localhost:3000/api/v1/models | jq '.count'
|
||||
curl -s http://localhost:3000/api/v1/models/active | jq '.status'
|
||||
|
||||
# 3. Verify recording endpoints return 200
|
||||
curl -s http://localhost:3000/api/v1/recording/list | jq '.count'
|
||||
curl -s -X POST http://localhost:3000/api/v1/recording/start \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"session_name":"test","duration_secs":5}' | jq '.status'
|
||||
|
||||
# 4. Verify training endpoint returns 200
|
||||
curl -s http://localhost:3000/api/v1/train/status | jq '.phase'
|
||||
|
||||
# 5. Verify LoRA endpoints return 200
|
||||
curl -s http://localhost:3000/api/v1/models/lora/profiles | jq '.'
|
||||
|
||||
# 6. Open UI — check browser console for zero 404 errors
|
||||
# Navigate to http://localhost:3000/ui/
|
||||
|
||||
# 7. Run mobile tests
|
||||
cd ../../ui/mobile
|
||||
npx jest --no-coverage
|
||||
|
||||
# 8. Run Rust workspace tests (must pass, 1031+ tests)
|
||||
cd ../../rust-port/wifi-densepose-rs
|
||||
cargo test --workspace --no-default-features
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- ADR-034: Expo React Native Mobile Application (mobile companion architecture)
|
||||
- ADR-036: RVF Training Pipeline UI (training pipeline design)
|
||||
- ADR-039: ESP32-S3 Edge Intelligence Pipeline (CSI frame format and processing tiers)
|
||||
- ADR-040: WASM Programmable Sensing (Tier 3 edge compute)
|
||||
- ADR-041: WASM Module Collection (module catalog)
|
||||
- `crates/wifi-densepose-sensing-server/src/main.rs` -- all 14 new handler functions (model, recording, training)
|
||||
- `ui/app.js` -- sensing service early initialization fix
|
||||
- `ui/mobile/src/services/ws.service.ts` -- mobile WebSocket URL fix
|
||||
@@ -0,0 +1,214 @@
|
||||
# ADR-044: Provisioning Tool Enhancements
|
||||
|
||||
**Status**: Proposed
|
||||
**Date**: 2026-03-03
|
||||
**Deciders**: @ruvnet
|
||||
**Supersedes**: None
|
||||
**Related**: ADR-029, ADR-032, ADR-039, ADR-040
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
The ESP32-S3 CSI node provisioning script (`firmware/esp32-csi-node/provision.py`) is the primary tool for configuring pre-built firmware binaries without recompiling. It writes NVS key-value pairs that the firmware reads at boot.
|
||||
|
||||
After #131 added TDM and edge intelligence flags, the script now covers the most-requested NVS keys. However, there remain gaps between what the firmware reads from NVS (`nvs_config.c`, 20 keys) and what the provisioning script can write (13 keys). Additionally, the script lacks usability features that would help field operators deploying multi-node meshes.
|
||||
|
||||
### Gap 1: Missing NVS Keys (7 keys)
|
||||
|
||||
The firmware reads these NVS keys at boot but the provisioning script has no corresponding CLI flags:
|
||||
|
||||
| NVS Key | Type | Firmware Default | Purpose |
|
||||
|---------|------|-----------------|---------|
|
||||
| `hop_count` | u8 | 1 (no hop) | Number of channels to hop through |
|
||||
| `chan_list` | blob (u8[6]) | {1,6,11} | Channel numbers for hopping sequence |
|
||||
| `dwell_ms` | u32 | 100 | Time to dwell on each channel before hopping (ms) |
|
||||
| `power_duty` | u8 | 100 | Power duty cycle percentage (10-100%) for battery life |
|
||||
| `wasm_max` | u8 | 4 | Max concurrent WASM modules (ADR-040) |
|
||||
| `wasm_verify` | u8 | 0 | Require Ed25519 signature for WASM uploads (0/1) |
|
||||
| `wasm_pubkey` | blob (32B) | zeros | Ed25519 public key for WASM signature verification |
|
||||
|
||||
### Gap 2: No Read-Back
|
||||
|
||||
There is no way to read the current NVS configuration from a device. Field operators must remember what was provisioned or reflash everything. This is especially problematic for multi-node meshes where each node has different TDM slots.
|
||||
|
||||
### Gap 3: No Verification
|
||||
|
||||
After flashing, there is no automated check that the device booted successfully with the new configuration. Operators must manually run a serial monitor and inspect logs.
|
||||
|
||||
### Gap 4: No Config File Support
|
||||
|
||||
Provisioning a 6-node mesh requires running the script 6 times with largely overlapping flags (same SSID, password, target IP) and only TDM slot varying. There is no way to define a mesh configuration in a file.
|
||||
|
||||
### Gap 5: No Presets
|
||||
|
||||
Common deployment scenarios (single-node basic, 3-node mesh, 6-node mesh with vitals) require operators to know which flags to combine. Named presets would lower the barrier to entry.
|
||||
|
||||
### Gap 6: No Auto-Detect
|
||||
|
||||
The `--port` flag is required even though the script could auto-detect connected ESP32-S3 devices via `esptool.py`.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Enhance `provision.py` with the following capabilities, implemented incrementally.
|
||||
|
||||
### Phase 1: Complete NVS Coverage
|
||||
|
||||
Add flags for all remaining firmware NVS keys:
|
||||
|
||||
```
|
||||
--hop-count N Channel hop count (1=no hop, default: 1)
|
||||
--channels 1,6,11 Comma-separated channel list for hopping
|
||||
--dwell-ms N Dwell time per channel in ms (default: 100)
|
||||
--power-duty N Power duty cycle 10-100% (default: 100)
|
||||
--wasm-max N Max concurrent WASM modules 1-8 (default: 4)
|
||||
--wasm-verify Require Ed25519 signature for WASM uploads
|
||||
--wasm-pubkey FILE Path to Ed25519 public key file (32 bytes raw or PEM)
|
||||
```
|
||||
|
||||
Validation:
|
||||
- `--channels` length must match `--hop-count`
|
||||
- `--power-duty` clamped to 10-100
|
||||
- `--wasm-pubkey` implies `--wasm-verify`
|
||||
|
||||
### Phase 2: Config File and Mesh Provisioning
|
||||
|
||||
Add `--config FILE` to load settings from a JSON or TOML file:
|
||||
|
||||
```json
|
||||
{
|
||||
"common": {
|
||||
"ssid": "SensorNet",
|
||||
"password": "secret",
|
||||
"target_ip": "192.168.1.20",
|
||||
"target_port": 5005,
|
||||
"edge_tier": 2
|
||||
},
|
||||
"nodes": [
|
||||
{ "port": "COM7", "node_id": 0, "tdm_slot": 0 },
|
||||
{ "port": "COM8", "node_id": 1, "tdm_slot": 1 },
|
||||
{ "port": "COM9", "node_id": 2, "tdm_slot": 2 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`--config mesh.json` provisions all listed nodes in sequence, computing `tdm_total` automatically from the `nodes` array length.
|
||||
|
||||
### Phase 3: Presets
|
||||
|
||||
Add `--preset NAME` for common deployment profiles:
|
||||
|
||||
| Preset | What It Sets |
|
||||
|--------|-------------|
|
||||
| `basic` | Single node, edge_tier=0, no TDM, no hopping |
|
||||
| `vitals` | Single node, edge_tier=2, vital_int=1000, subk_count=32 |
|
||||
| `mesh-3` | 3-node TDM, edge_tier=1, hop_count=3, channels=1,6,11 |
|
||||
| `mesh-6-vitals` | 6-node TDM, edge_tier=2, hop_count=3, channels=1,6,11, vital_int=500 |
|
||||
|
||||
Presets set defaults that can be overridden by explicit flags.
|
||||
|
||||
### Phase 4: Read-Back and Verify
|
||||
|
||||
Add `--read` to dump the current NVS configuration from a connected device:
|
||||
|
||||
```bash
|
||||
python provision.py --port COM7 --read
|
||||
# Output:
|
||||
# ssid: SensorNet
|
||||
# target_ip: 192.168.1.20
|
||||
# tdm_slot: 0
|
||||
# tdm_nodes: 3
|
||||
# edge_tier: 2
|
||||
# ...
|
||||
```
|
||||
|
||||
Implementation: use `esptool.py read_flash` to read the NVS partition, then parse the NVS binary format to extract key-value pairs.
|
||||
|
||||
Add `--verify` to provision and then confirm the device booted:
|
||||
|
||||
```bash
|
||||
python provision.py --port COM7 --ssid "Net" --password "pass" --target-ip 192.168.1.20 --verify
|
||||
# After flash, opens serial monitor for 5 seconds
|
||||
# Checks for "CSI streaming active" log line
|
||||
# Reports PASS or FAIL
|
||||
```
|
||||
|
||||
### Phase 5: Auto-Detect Port
|
||||
|
||||
When `--port` is omitted, scan for connected ESP32-S3 devices:
|
||||
|
||||
```bash
|
||||
python provision.py --ssid "Net" --password "pass" --target-ip 192.168.1.20
|
||||
# Auto-detected ESP32-S3 on COM7 (Silicon Labs CP210x)
|
||||
# Proceed? [Y/n]
|
||||
```
|
||||
|
||||
Implementation: use `esptool.py` or `serial.tools.list_ports` to enumerate ports.
|
||||
|
||||
---
|
||||
|
||||
## Rationale
|
||||
|
||||
### Why incremental phases?
|
||||
|
||||
Phase 1 is a small diff that closes the NVS coverage gap immediately. Phases 2-5 add progressively more UX polish. Each phase is independently useful and can be shipped separately.
|
||||
|
||||
### Why JSON config over YAML/TOML?
|
||||
|
||||
JSON requires no additional Python dependencies (stdlib `json` module). TOML requires `tomllib` (Python 3.11+) or `tomli`. JSON is sufficient for this use case.
|
||||
|
||||
### Why not a GUI?
|
||||
|
||||
The target users are embedded developers and field operators who are already running `esptool` from the command line. A TUI/GUI would add dependencies and complexity for minimal benefit.
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Complete NVS coverage**: Every firmware-readable key can be set from the provisioning tool
|
||||
- **Mesh provisioning in one command**: `--config mesh.json` replaces 6 separate invocations
|
||||
- **Lower barrier to entry**: Presets eliminate the need to know which flags to combine
|
||||
- **Auditability**: `--read` lets operators inspect and verify deployed configurations
|
||||
- **Fewer mis-provisions**: `--verify` catches flashing failures before the operator walks away
|
||||
|
||||
### Negative
|
||||
|
||||
- **NVS binary parsing** (Phase 4) requires understanding the ESP-IDF NVS binary format, which is not officially documented as a stable API
|
||||
- **Auto-detect** (Phase 5) may produce false positives if other ESP32 variants are connected
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| NVS binary format changes in ESP-IDF v6 | Low | Medium | Pin to known ESP-IDF NVS page format; add format version check |
|
||||
| `--verify` serial parsing is fragile | Medium | Low | Match on stable log tag `[CSI_MAIN]`; timeout after 10s |
|
||||
| Config file credentials in plaintext | Medium | Medium | Document that config files should not be committed; add `.gitignore` pattern |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
| Phase | Effort | Impact | Priority |
|
||||
|-------|--------|--------|----------|
|
||||
| Phase 1: Complete NVS coverage | Small (1 file, ~50 lines) | High — closes feature gap | P0 |
|
||||
| Phase 2: Config file + mesh | Medium (~100 lines) | High — biggest UX win | P1 |
|
||||
| Phase 3: Presets | Small (~40 lines) | Medium — convenience | P2 |
|
||||
| Phase 4: Read-back + verify | Medium (~150 lines) | Medium — debugging aid | P2 |
|
||||
| Phase 5: Auto-detect | Small (~30 lines) | Low — minor convenience | P3 |
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- `firmware/esp32-csi-node/main/nvs_config.h` — NVS config struct (20 fields)
|
||||
- `firmware/esp32-csi-node/main/nvs_config.c` — NVS read logic (20 keys)
|
||||
- `firmware/esp32-csi-node/provision.py` — Current provisioning script (13 of 20 keys)
|
||||
- ADR-029: RuvSense multistatic sensing mode (TDM, channel hopping)
|
||||
- ADR-032: Multistatic mesh security hardening (mesh keys)
|
||||
- ADR-039: ESP32-S3 edge intelligence (edge tiers, vitals)
|
||||
- ADR-040: WASM programmable sensing (WASM modules, signature verification)
|
||||
- Issue #130: Provisioning script doesn't support TDM
|
||||
@@ -0,0 +1,110 @@
|
||||
# ADR-045: AMOLED Display Support for ESP32-S3 CSI Node
|
||||
|
||||
## Status
|
||||
|
||||
Proposed
|
||||
|
||||
## Context
|
||||
|
||||
The ESP32-S3 board (LilyGO T-Display-S3 AMOLED) has an integrated RM67162 QSPI AMOLED display (536x240) and 8MB octal PSRAM that were unused by the CSI firmware. Users want real-time on-device visualization of CSI statistics, vital signs, and system health without relying on an external server.
|
||||
|
||||
### Constraints
|
||||
|
||||
- Binary was 947 KB in a 1 MB partition — needed 8MB flash + custom partition table
|
||||
- SPIRAM was disabled in sdkconfig despite hardware having 8MB PSRAM
|
||||
- Core 1 is pinned to DSP (edge processing) — display must use Core 0
|
||||
- Existing CSI pipeline must not be affected
|
||||
|
||||
### Available APIs
|
||||
|
||||
Thread-safe edge APIs already exist (`edge_get_vitals()`, `edge_get_multi_person()`) — the display task only reads from these, no new synchronization needed.
|
||||
|
||||
## Decision
|
||||
|
||||
Add optional AMOLED display support with the following architecture:
|
||||
|
||||
### Hardware Abstraction Layer
|
||||
|
||||
- `display_hal.c/h`: RM67162 QSPI panel driver + CST816S capacitive touch via I2C
|
||||
- Auto-detect at boot: probe RM67162 and check SPIRAM; log warning and skip if absent
|
||||
|
||||
### UI Layer
|
||||
|
||||
- `display_ui.c/h`: LVGL 8.3 with 4 swipeable views via tileview widget
|
||||
- Dark theme (#0a0a0f) with cyan (#00d4ff) accent for three.js-like aesthetic
|
||||
- Views: Dashboard (CSI amplitude chart + stats), Vitals (breathing + HR line graphs), Presence (4x4 occupancy grid), System (CPU, heap, PSRAM, WiFi, uptime, FPS)
|
||||
|
||||
### Task Layer
|
||||
|
||||
- `display_task.c/h`: FreeRTOS task on Core 0, priority 1 (lowest)
|
||||
- LVGL pump loop at configurable FPS (default 30)
|
||||
- Double-buffered draw buffers allocated in SPIRAM
|
||||
|
||||
### Compile-Time Control
|
||||
|
||||
- `CONFIG_DISPLAY_ENABLE=y` (default): compiles display code, auto-detects hardware at boot
|
||||
- `CONFIG_DISPLAY_ENABLE=n`: zero-cost — no display code compiled
|
||||
- `CONFIG_SPIRAM_IGNORE_NOTFOUND=y`: boots fine on boards without PSRAM
|
||||
|
||||
### Flash Layout
|
||||
|
||||
8MB partition table (`partitions_display.csv`):
|
||||
- Dual OTA partitions: 2 x 2MB (supports larger binaries with LVGL)
|
||||
- SPIFFS: 1.9MB (for future font/asset storage)
|
||||
- NVS + otadata + phy: standard sizes
|
||||
|
||||
### Core/Task Layout
|
||||
|
||||
| Task | Core | Priority | Impact |
|
||||
|------|------|----------|--------|
|
||||
| WiFi/LwIP | 0 | 18-23 | unchanged |
|
||||
| OTA httpd | 0 | 5 | unchanged |
|
||||
| **display_task** | **0** | **1** | **NEW — lowest priority** |
|
||||
| edge_task (DSP) | 1 | 5 | unchanged |
|
||||
|
||||
### Dependencies
|
||||
|
||||
- LVGL ~8.3 (via ESP-IDF managed components)
|
||||
- espressif/esp_lcd_touch_cst816s ^1.0
|
||||
- espressif/esp_lcd_touch ^1.0
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Real-time on-device stats without network dependency
|
||||
- Zero impact on CSI pipeline (display reads thread-safe APIs, runs at lowest priority)
|
||||
- Graceful degradation: works on boards without display or PSRAM
|
||||
- SPIRAM enabled for all boards (benefits WASM runtime too)
|
||||
- 8MB flash + dual OTA 2MB partitions give headroom for future features
|
||||
|
||||
### Negative
|
||||
|
||||
- Binary size increase (~200-300 KB with LVGL)
|
||||
- SPIRAM + 8MB flash config is specific to T-Display-S3 AMOLED boards
|
||||
- Boards with only 4MB flash need `CONFIG_DISPLAY_ENABLE=n` and the old partition table
|
||||
|
||||
### Risks
|
||||
|
||||
- RM67162 init sequence is board-specific; other AMOLED panels may need different commands
|
||||
- QSPI bus conflicts if other peripherals use SPI2_HOST (currently unused)
|
||||
|
||||
## New Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `main/display_hal.c/h` | RM67162 QSPI + CST816S touch HAL |
|
||||
| `main/display_ui.c/h` | LVGL 4-view UI |
|
||||
| `main/display_task.c/h` | FreeRTOS task, LVGL pump |
|
||||
| `main/lv_conf.h` | LVGL compile config |
|
||||
| `partitions_display.csv` | 8MB partition table |
|
||||
| `idf_component.yml` | Managed component deps |
|
||||
|
||||
## Modified Files
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `sdkconfig.defaults` | 8MB flash, SPIRAM, custom partitions |
|
||||
| `main/CMakeLists.txt` | Conditional display sources + deps |
|
||||
| `main/main.c` | +1 include, +5 lines guarded init |
|
||||
| `main/Kconfig.projbuild` | "AMOLED Display" menu |
|
||||
@@ -0,0 +1,263 @@
|
||||
# ADR-046: Android TV Box / Armbian Deployment Target
|
||||
|
||||
## Status
|
||||
|
||||
Proposed
|
||||
|
||||
## Context
|
||||
|
||||
Issue [#138](https://github.com/ruvnet/wifi-densepose/issues/138) requests ESP8266 and mobile device support. The ESP8266 lacks CSI capability and sufficient resources, but the discussion revealed a compelling deployment target: **Android TV boxes** (Amlogic/Allwinner/Rockchip SoCs) running **Armbian** (Debian for ARM).
|
||||
|
||||
These devices cost $15–35, are always-on mains-powered, include 802.11ac WiFi, 2–4 GB RAM, quad-core ARM Cortex-A53/A55 CPUs, and HDMI output. They are widely available as consumer "IPTV boxes" (T95, H96 Max, X96, MXQ Pro, etc.) and can boot Armbian from SD card without modifying the factory Android installation.
|
||||
|
||||
### Current deployment model
|
||||
|
||||
```
|
||||
[ESP32-S3 nodes] --UDP CSI--> [Laptop/PC running sensing-server] --browser--> [UI]
|
||||
```
|
||||
|
||||
This requires a general-purpose computer ($300+) to run the Rust sensing server, NN inference, and web dashboard. For permanent installations (elder care, smart home, security), dedicating a laptop is impractical.
|
||||
|
||||
### Proposed deployment model
|
||||
|
||||
```
|
||||
[ESP32-S3 nodes] --UDP CSI--> [TV Box running Armbian + sensing-server] --HDMI--> [Display]
|
||||
$25, always-on, fanless
|
||||
```
|
||||
|
||||
### Future: custom WiFi firmware for standalone operation
|
||||
|
||||
Many TV box WiFi chipsets (Realtek RTL8822CS, MediaTek MT7661, Broadcom BCM43455) can potentially be patched for CSI extraction when running under Linux with custom drivers. This would eliminate the ESP32 dependency entirely for basic sensing:
|
||||
|
||||
```
|
||||
[TV Box with patched WiFi driver] --CSI extraction--> [sensing-server on same box] --HDMI--> [Display]
|
||||
$25 total, single device
|
||||
```
|
||||
|
||||
This ADR covers Phase 1 (TV box as aggregator) and Phase 2 (custom WiFi firmware for CSI). Phase 2 is speculative and requires per-chipset R&D.
|
||||
|
||||
## Decision
|
||||
|
||||
### Phase 1: TV Box as Aggregator (Armbian)
|
||||
|
||||
1. **Cross-compile the sensing server** for `aarch64-unknown-linux-gnu` using `cross` or Docker-based cross-compilation.
|
||||
|
||||
2. **Create an Armbian deployment package** containing:
|
||||
- Pre-built `wifi-densepose-sensing-server` binary (aarch64)
|
||||
- systemd service file for auto-start on boot
|
||||
- Kiosk-mode Chromium configuration for HDMI dashboard display
|
||||
- Network configuration for ESP32 UDP reception (port 5005)
|
||||
- Optional: `hostapd` config to create a dedicated WiFi AP for the ESP32 mesh
|
||||
|
||||
3. **Define minimum hardware requirements:**
|
||||
|
||||
| Component | Minimum | Recommended |
|
||||
|-----------|---------|-------------|
|
||||
| SoC | Amlogic S905W (A53 quad) | Amlogic S905X3 (A55 quad) |
|
||||
| RAM | 2 GB | 4 GB |
|
||||
| Storage | 8 GB eMMC + 8 GB SD | 16 GB eMMC + 16 GB SD |
|
||||
| WiFi | 802.11n 2.4 GHz | 802.11ac dual-band |
|
||||
| Ethernet | 100 Mbps | Gigabit |
|
||||
| USB | 1x USB 2.0 | 2x USB 3.0 |
|
||||
| HDMI | 1.4 | 2.0 |
|
||||
|
||||
4. **Tested reference devices** (initial target list):
|
||||
|
||||
| Device | SoC | WiFi Chip | Price | Armbian Support |
|
||||
|--------|-----|-----------|-------|-----------------|
|
||||
| T95 Max+ | S905X3 | RTL8822CS | ~$30 | Good (meson-sm1) |
|
||||
| H96 Max X3 | S905X3 | RTL8822CS | ~$35 | Good (meson-sm1) |
|
||||
| X96 Max+ | S905X3 | RTL8822CS | ~$28 | Good (meson-sm1) |
|
||||
| Tanix TX6S | H616 | MT7668 | ~$25 | Moderate (sun50i-h616) |
|
||||
|
||||
5. **New Rust compilation target** in workspace CI:
|
||||
- Add `aarch64-unknown-linux-gnu` to cross-compilation matrix
|
||||
- Binary size target: <15 MB stripped (fits easily in SD card)
|
||||
- No GPU dependency — CPU-only inference using `candle` or ONNX Runtime for ARM
|
||||
|
||||
### Phase 2: Custom WiFi Firmware for CSI Extraction (Future)
|
||||
|
||||
1. **CSI extraction feasibility by chipset:**
|
||||
|
||||
| Chipset | Driver | CSI Support | Monitor Mode | Effort |
|
||||
|---------|--------|-------------|--------------|--------|
|
||||
| Broadcom BCM43455 | brcmfmac | **Proven** (Nexmon CSI) | Yes | Low — patches exist |
|
||||
| Realtek RTL8822CS | rtw88 | **Moderate** — driver is open-source, CSI hooks need adding | Yes (patched) | Medium |
|
||||
| MediaTek MT7661 | mt76 | **Unknown** — MediaTek has released CSI tools for some chips | Yes | Medium-High |
|
||||
|
||||
2. **CSI extraction architecture** (Linux kernel driver modification):
|
||||
|
||||
```
|
||||
[WiFi chipset firmware] → [Modified kernel driver] → [Netlink/procfs CSI export]
|
||||
↓
|
||||
[userspace CSI reader]
|
||||
↓
|
||||
[sensing-server UDP input]
|
||||
```
|
||||
|
||||
The CSI data would be reformatted into the existing ESP32 binary protocol (ADR-018 header, magic `0xC5100001`) so the sensing server treats it identically to ESP32 frames. This means zero changes to the ingestion context.
|
||||
|
||||
3. **Hybrid mode**: When the TV box has both patched WiFi CSI and ESP32 UDP input, the sensing server's multi-node architecture (already supporting multiple `node_id` values) handles both sources transparently. The TV box's own WiFi becomes an additional viewpoint in the multistatic array.
|
||||
|
||||
### Phase 3: Android Companion App (Optional)
|
||||
|
||||
For users who want mobile monitoring without Armbian:
|
||||
|
||||
1. **PWA (Progressive Web App)**: The sensing server already serves a web UI. Adding a PWA manifest with offline caching makes it installable on any Android device. No native app needed.
|
||||
|
||||
2. **Native Android app** (future): Only if PWA proves insufficient. Would use Kotlin + Jetpack Compose, consuming the existing REST API and WebSocket endpoints.
|
||||
|
||||
## Deployment Architecture
|
||||
|
||||
### Single-Room Deployment (Phase 1)
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Room │
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ ESP32-S3 │ │ ESP32-S3 │ │ ESP32-S3 │ CSI sensor mesh │
|
||||
│ │ Node 1 │ │ Node 2 │ │ Node 3 │ ($10 each) │
|
||||
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
|
||||
│ │ │ │ │
|
||||
│ └──────────────┼──────────────┘ │
|
||||
│ │ UDP port 5005 │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────┐ │
|
||||
│ │ Android TV Box (Armbian) │ │
|
||||
│ │ │ │
|
||||
│ │ ┌──────────────────────────────┐ │ │
|
||||
│ │ │ wifi-densepose-sensing- │ │ │
|
||||
│ │ │ server (aarch64 binary) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ • CSI ingestion (UDP) │ │ │
|
||||
│ │ │ • Feature extraction │ │ │
|
||||
│ │ │ • NN inference (CPU) │ │ │
|
||||
│ │ │ • WebSocket streaming │ │ │
|
||||
│ │ │ • REST API │ │ │
|
||||
│ │ │ • Web UI (:3000) │ │ │
|
||||
│ │ └──────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌──────────────────────────────┐ │ │
|
||||
│ │ │ Chromium Kiosk Mode │───│──→ HDMI out │
|
||||
│ │ │ (localhost:3000) │ │ to display │
|
||||
│ │ └──────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ Cost: $25-35 │ │
|
||||
│ │ Power: 5-10W (USB-C or barrel) │ │
|
||||
│ │ Form: fits behind TV/monitor │ │
|
||||
│ └──────────────────────────────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
|
||||
Total system cost: $55-65 (3 ESP32 nodes + 1 TV box)
|
||||
```
|
||||
|
||||
### Multi-Room Deployment
|
||||
|
||||
```
|
||||
┌──────────────┐
|
||||
│ Router │
|
||||
│ (WiFi AP) │
|
||||
└──────┬───────┘
|
||||
│ LAN
|
||||
┌──────────────┼──────────────┐
|
||||
│ │ │
|
||||
┌───────▼───────┐ ┌───▼────────┐ ┌──▼──────────┐
|
||||
│ Room A │ │ Room B │ │ Room C │
|
||||
│ TV Box + │ │ TV Box + │ │ TV Box + │
|
||||
│ 3x ESP32 │ │ 3x ESP32 │ │ 3x ESP32 │
|
||||
│ HDMI display │ │ HDMI │ │ HDMI │
|
||||
└───────────────┘ └────────────┘ └─────────────┘
|
||||
|
||||
Each room: self-contained sensing + display
|
||||
Central dashboard: aggregate all rooms via REST API
|
||||
```
|
||||
|
||||
### Standalone Mode (Phase 2 — Custom WiFi FW)
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────┐
|
||||
│ Android TV Box (Armbian) │
|
||||
│ │
|
||||
│ ┌────────────────────┐ │
|
||||
│ │ Patched WiFi │ │
|
||||
│ │ Driver │ │
|
||||
│ │ (CSI extraction) │ │
|
||||
│ └─────────┬──────────┘ │
|
||||
│ │ CSI frames │
|
||||
│ ▼ │
|
||||
│ ┌────────────────────┐ │
|
||||
│ │ sensing-server │──→ HDMI out │
|
||||
│ │ (inference + │ │
|
||||
│ │ dashboard) │ │
|
||||
│ └────────────────────┘ │
|
||||
│ │
|
||||
│ Single device: $25 │
|
||||
│ No ESP32 nodes needed │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **10x cost reduction** for aggregator: $25 TV box vs $300+ laptop/PC
|
||||
- **Always-on deployment**: Mains-powered, fanless, designed for 24/7 operation
|
||||
- **HDMI output**: Direct connection to TV/monitor for wall-mounted dashboards
|
||||
- **Familiar hardware**: Available globally, no specialized ordering required
|
||||
- **Armbian ecosystem**: Mature Debian-based distro with package management, systemd, SSH
|
||||
- **Path to standalone**: Custom WiFi firmware could eliminate ESP32 dependency entirely
|
||||
- **PWA for mobile**: No native app development needed for mobile monitoring
|
||||
- **Multi-room scaling**: One TV box per room, each self-contained
|
||||
|
||||
### Negative
|
||||
|
||||
- **ARM cross-compilation**: Adds CI complexity; `candle`/ONNX Runtime ARM builds need testing
|
||||
- **Armbian compatibility**: Not all TV boxes are well-supported; need a tested device list
|
||||
- **Performance uncertainty**: ARM A53 cores are ~3-5x slower than x86 for NN inference; may need model quantization (INT8) for real-time operation
|
||||
- **Phase 2 risk**: Custom WiFi firmware is chipset-specific, may require kernel patches per driver version, and CSI quality varies by chipset
|
||||
- **Support burden**: Different hardware = more configurations to support
|
||||
- **No GPU**: TV boxes lack discrete GPU; inference is CPU-only (but our models are small enough)
|
||||
|
||||
### Neutral
|
||||
|
||||
- **No changes to existing ESP32 firmware** — TV box receives the same UDP frames
|
||||
- **No changes to sensing server protocol** — Phase 2 CSI output uses same binary format
|
||||
- **Existing web UI works as-is** — Chromium kiosk mode or any browser on the LAN
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1 (2-3 weeks)
|
||||
|
||||
1. Add `aarch64-unknown-linux-gnu` cross-compilation target using `cross`
|
||||
2. Build and test sensing-server binary on reference TV box (T95 Max+ / S905X3)
|
||||
3. Create systemd service + Armbian deployment script
|
||||
4. Benchmark: measure inference latency, memory usage, thermal throttling
|
||||
5. Create `docs/deployment/armbian-tv-box.md` setup guide
|
||||
6. Add HDMI kiosk mode configuration (Chromium autostart)
|
||||
|
||||
### Phase 2 (4-8 weeks, R&D)
|
||||
|
||||
1. Acquire TV box with BCM43455 (proven Nexmon CSI support)
|
||||
2. Build Armbian with Nexmon CSI patches for BCM43455
|
||||
3. Write userspace CSI reader → ESP32 binary protocol converter
|
||||
4. Test CSI quality comparison: ESP32 vs BCM43455
|
||||
5. If viable: add RTL8822CS CSI extraction via rtw88 driver modification
|
||||
|
||||
### Phase 3 (1 week)
|
||||
|
||||
1. Add PWA manifest to sensing server web UI
|
||||
2. Test on Android Chrome, iOS Safari
|
||||
3. Add service worker for offline dashboard caching
|
||||
|
||||
## References
|
||||
|
||||
- [Nexmon CSI](https://github.com/seemoo-lab/nexmon_csi) — Broadcom WiFi CSI extraction (BCM43455, BCM4339, BCM4358)
|
||||
- [Armbian](https://www.armbian.com/) — Debian/Ubuntu for ARM SBCs and TV boxes
|
||||
- [rtw88 driver](https://github.com/torvalds/linux/tree/master/drivers/net/wireless/realtek/rtw88) — Mainline Linux driver for Realtek 802.11ac chips
|
||||
- [mt76 driver](https://github.com/torvalds/linux/tree/master/drivers/net/wireless/mediatek/mt76) — Mainline Linux driver for MediaTek WiFi chips
|
||||
- [cross](https://github.com/cross-rs/cross) — Zero-setup Rust cross-compilation
|
||||
- [ADR-018: ESP32 CSI Binary Protocol](ADR-018-dev-implementation.md) — Binary frame format reused for Phase 2 CSI extraction
|
||||
- [ADR-039: Edge Intelligence](ADR-039-esp32-edge-intelligence.md) — On-device processing tiers
|
||||
- [ADR-043: Sensing Server](ADR-043-sensing-server-ui-api-completion.md) — Single-binary deployment target
|
||||
@@ -0,0 +1,152 @@
|
||||
# ADR-047: RuView Observatory — Immersive Three.js WiFi Sensing Visualization
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (Implemented)
|
||||
|
||||
## Date
|
||||
|
||||
2026-03-04
|
||||
|
||||
## Context
|
||||
|
||||
The project has a functional tabbed dashboard UI (`ui/index.html`) with existing Three.js components (body model, gaussian splats, signal visualization, environment). While effective for monitoring, it lacks a cinematic, immersive visualization suitable for demonstrations and stakeholder presentations.
|
||||
|
||||
We need an immersive Three.js room-based visualization with practical WiFi sensing data overlays — human wireframe pose, dot-matrix body mass, vital signs HUD, signal field heatmap — powered by ESP32 CSI data (demo mode with live WebSocket path).
|
||||
|
||||
## Decision
|
||||
|
||||
### Standalone Page Architecture
|
||||
|
||||
`ui/observatory.html` is a standalone full-screen entry point, separate from the tabbed dashboard. Linked via "Observatory" nav tab in `ui/index.html`. No build step — vanilla JS modules with Three.js r160 via CDN importmap.
|
||||
|
||||
### Room-Based Visualization
|
||||
|
||||
Instead of abstract holographic panels, the observatory renders a practical room scene with:
|
||||
|
||||
| Element | Implementation | Data Source |
|
||||
|---------|---------------|-------------|
|
||||
| Human wireframe | COCO 17-keypoint skeleton, CylinderGeometry tube bones, SphereGeometry joints with glow halos | `persons[].position`, `vital_signs.breathing_rate_bpm` |
|
||||
| Dot-matrix mist | 800 Points with per-particle alpha ShaderMaterial, body-shaped distribution | `persons[].position`, `persons[].motion_score` |
|
||||
| Particle trail | 200 Points with age-based fade, emitted from moving person | `persons[].position`, `persons[].motion_score` |
|
||||
| Signal field | 400 floor-level Points with green→amber color ramp | `signal_field.values` (20×20 grid) |
|
||||
| WiFi waves | 5 wireframe SphereGeometry shells, AdditiveBlending, pulsing outward | Always-on animation from router position |
|
||||
| Router | BoxGeometry body, 3 CylinderGeometry antennas, pulsing LED, PointLight | Static scene element |
|
||||
| Room | GridHelper floor, BoxGeometry wireframe boundary, reflective MeshStandardMaterial floor, furniture (table, bed) | Static scene element |
|
||||
|
||||
### HUD Overlay
|
||||
|
||||
Glass-morphism HTML panels overlaid on the 3D canvas:
|
||||
|
||||
- **Left panel (Vital Signs):** Heart rate (BPM), respiration (RPM), confidence (%) with animated bars
|
||||
- **Right panel (WiFi Signal):** RSSI, variance, motion power, person count, 2D RSSI sparkline, presence state badge, fall alert
|
||||
- **Top-right:** Data source badge (DEMO/LIVE), scenario badge, FPS counter, settings gear
|
||||
- **Bottom:** Capability bar (Pose Estimation, Vital Monitoring, Presence Detection)
|
||||
- **Bottom-right:** Keyboard shortcut hints
|
||||
|
||||
### Settings Dialog (4 Tabs)
|
||||
|
||||
Full customization with localStorage persistence and JSON export:
|
||||
|
||||
| Tab | Controls |
|
||||
|-----|----------|
|
||||
| **Rendering** | Bloom strength/radius/threshold, exposure, vignette, film grain, chromatic aberration |
|
||||
| **Wireframe** | Bone thickness, joint size, glow intensity, particle trail, wireframe color, joint color, aura opacity |
|
||||
| **Scene** | Signal field opacity, WiFi wave intensity, room brightness, floor reflection, FOV, orbit speed, grid toggle, room boundary toggle |
|
||||
| **Data** | Scenario selector (auto-cycle or fixed), cycle speed, data source (demo/WebSocket), WS URL, reset camera, export settings |
|
||||
|
||||
### Demo-First with Live Data Path
|
||||
|
||||
Four auto-cycling scenarios (30s default, configurable) with 2s cosine crossfade:
|
||||
|
||||
| Scenario | Description |
|
||||
|----------|-------------|
|
||||
| `empty_room` | Low variance, no presence, flat amplitude, stable RSSI -45dBm |
|
||||
| `single_breathing` | 1 person, breathing 16 BPM, HR 72 BPM, sinusoidal subcarrier modulation |
|
||||
| `two_walking` | 2 persons, high motion, Doppler-like shifts, moving signal field peaks |
|
||||
| `fall_event` | 2s variance spike at t=5s, then stillness, fall flag, confidence drop |
|
||||
|
||||
Data contract matches `SensingUpdate` struct from the Rust sensing server. Live WebSocket connection configurable in settings dialog.
|
||||
|
||||
### Post-Processing Pipeline
|
||||
|
||||
EffectComposer chain: RenderPass → UnrealBloomPass → custom VignetteShader
|
||||
|
||||
- **UnrealBloom:** strength 1.0, radius 0.5, threshold 0.25 (configurable)
|
||||
- **VignetteShader:** warm shadow shift, edge chromatic aberration, film grain
|
||||
- **Adaptive quality:** Auto-degrades when FPS < 25, restores when FPS > 55
|
||||
|
||||
### RuView Foundation Color Palette
|
||||
|
||||
| Role | Color | Hex |
|
||||
|------|-------|-----|
|
||||
| Background | Deep dark | `#080c14` |
|
||||
| Primary wireframe | Green glow | `#00d878` |
|
||||
| Warm accent | Amber | `#ffb020` |
|
||||
| Signal | Blue | `#2090ff` |
|
||||
| Heart / joints | Red | `#ff4060` |
|
||||
| Alert | Crimson | `#ff3040` |
|
||||
|
||||
### Technology Choices
|
||||
|
||||
| Decision | Rationale |
|
||||
|----------|-----------|
|
||||
| Standalone page vs tab | Full-screen immersion, independent loading |
|
||||
| Room-based vs abstract panels | Practical spatial context for WiFi sensing data |
|
||||
| Vanilla JS + CDN, no build step | Matches existing `ui/` pattern, served as static files by Axum |
|
||||
| Custom ShaderMaterial for mist | Per-particle alpha, body-shaped distribution, AdditiveBlending |
|
||||
| CylinderGeometry tube bones | Visible at any zoom vs thin Line geometry |
|
||||
| COCO 17-keypoint skeleton | Standard pose format, 16 bone connections |
|
||||
| localStorage settings | Persistent customization without server round-trip |
|
||||
| Adaptive quality | 3 levels, auto-switches based on FPS measurement |
|
||||
|
||||
### Keyboard Shortcuts
|
||||
|
||||
| Key | Action |
|
||||
|-----|--------|
|
||||
| `A` | Toggle autopilot orbit |
|
||||
| `D` | Cycle demo scenario |
|
||||
| `F` | Toggle FPS counter |
|
||||
| `S` | Open/close settings |
|
||||
| `Space` | Pause/resume data |
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `ui/observatory.html` | Full-screen entry point with HUD overlay + settings dialog |
|
||||
| `ui/observatory/js/main.js` | Scene orchestrator (~1,100 lines): room, wireframe, mist, trails, settings, HUD, animation loop |
|
||||
| `ui/observatory/js/demo-data.js` | 4 scenarios with cosine crossfade, setScenario/setCycleDuration API |
|
||||
| `ui/observatory/js/nebula-background.js` | Procedural fBM nebula + star field background sphere |
|
||||
| `ui/observatory/js/post-processing.js` | EffectComposer: UnrealBloom + VignetteShader (chromatic, grain, warmth) |
|
||||
| `ui/observatory/css/observatory.css` | Foundation color scheme, glass-morphism panels, settings dialog, responsive |
|
||||
| `ui/index.html` | Modified: added Observatory nav link |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Standalone page does not affect existing dashboard stability
|
||||
- Demo-first allows offline presentations without hardware
|
||||
- Same `SensingUpdate` contract enables seamless live WebSocket switch
|
||||
- Room-based visualization provides intuitive spatial context for WiFi sensing
|
||||
- Dot-matrix mist gives visual body mass without occluding wireframe
|
||||
- Full settings customization without code changes (localStorage + JSON export)
|
||||
- Adaptive quality ensures usability on weaker hardware
|
||||
- ~20 draw calls keeps performance well within budget
|
||||
|
||||
### Negative
|
||||
- Additional static files served by Axum (minimal overhead)
|
||||
- Three.js r160 loaded from CDN (no build step, matches existing pattern)
|
||||
- Settings persistence is per-browser (localStorage, not synced)
|
||||
|
||||
### Risks
|
||||
- CDN dependency for Three.js (mitigated: can vendor locally if needed)
|
||||
- Post-processing may not work on very old GPUs (mitigated: adaptive quality disables bloom)
|
||||
|
||||
## References
|
||||
|
||||
- ADR-045: AMOLED display support
|
||||
- ADR-046: Android TV / Armbian deployment
|
||||
- Existing `ui/components/scene.js` — Three.js scene pattern
|
||||
- Existing `ui/components/gaussian-splats.js` — ShaderMaterial pattern
|
||||
- Existing `ui/services/sensing.service.js` — WebSocket data contract
|
||||
@@ -0,0 +1,140 @@
|
||||
# ADR-048: Adaptive CSI Activity Classifier
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-03-05 |
|
||||
| Deciders | ruv |
|
||||
| Depends on | ADR-024 (AETHER Embeddings), ADR-039 (Edge Processing), ADR-045 (AMOLED Display) |
|
||||
|
||||
## Context
|
||||
|
||||
WiFi-based activity classification using ESP32 Channel State Information (CSI) relies on hand-tuned thresholds to distinguish between activity states (absent, present_still, present_moving, active). These static thresholds are brittle — they don't account for:
|
||||
|
||||
- **Environment-specific signal patterns**: Room geometry, furniture, wall materials, and ESP32 placement all affect how CSI signals respond to human activity.
|
||||
- **Temporal noise characteristics**: Real ESP32 CSI data at ~10 FPS has significant frame-to-frame jitter that causes classification to jump between states.
|
||||
- **Vital signs estimation noise**: Heart rate and breathing rate estimates from Goertzel filter banks produce large swings (50+ BPM frame-to-frame) at low confidence levels.
|
||||
|
||||
The existing threshold-based approach produces noisy, unstable classifications that degrade the user experience in the Observatory visualization and the main dashboard.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Three-Stage Signal Smoothing Pipeline
|
||||
|
||||
All CSI-derived metrics pass through a three-stage pipeline before reaching the UI:
|
||||
|
||||
#### Stage 1: Adaptive Baseline Subtraction
|
||||
- EMA with α=0.003 (~30s time constant) tracks the "quiet room" noise floor
|
||||
- Only updates during low-motion periods to avoid inflating baseline during activity
|
||||
- 50-frame warm-up period for initial baseline learning
|
||||
- Subtracts 70% of baseline from raw motion score to remove environmental drift
|
||||
|
||||
#### Stage 2: EMA + Median Filtering
|
||||
- **Motion score**: Blended from 4 signals (temporal diff 40%, variance 20%, motion band power 25%, change points 15%), then EMA-smoothed with α=0.15
|
||||
- **Vital signs**: 21-frame sliding window → trimmed mean (drop top/bottom 25%) → EMA with α=0.02 (~5s time constant)
|
||||
- **Dead-band**: HR won't update unless trimmed mean differs by >2 BPM; BR needs >0.5 BPM
|
||||
- **Outlier rejection**: HR jumps >8 BPM/frame and BR jumps >2 BPM/frame are discarded
|
||||
|
||||
#### Stage 3: Hysteresis Debounce
|
||||
- Activity state transitions require 4 consecutive frames (~0.4s) of agreement before committing
|
||||
- Prevents rapid flickering between states
|
||||
- Independent candidate tracking resets on new direction changes
|
||||
|
||||
### 2. Adaptive Classifier Module (`adaptive_classifier.rs`)
|
||||
|
||||
A Rust-native environment-tuned classifier that learns from labeled JSONL recordings:
|
||||
|
||||
#### Feature Extraction (15 features)
|
||||
| # | Feature | Source | Discriminative Power |
|
||||
|---|---------|--------|---------------------|
|
||||
| 0 | variance | Server | Medium — temporal CSI spread |
|
||||
| 1 | motion_band_power | Server | Medium — high-frequency subcarrier energy |
|
||||
| 2 | breathing_band_power | Server | Low — respiratory band energy |
|
||||
| 3 | spectral_power | Server | Low — mean squared amplitude |
|
||||
| 4 | dominant_freq_hz | Server | Low — peak subcarrier index |
|
||||
| 5 | change_points | Server | Medium — threshold crossing count |
|
||||
| 6 | mean_rssi | Server | Low — received signal strength |
|
||||
| 7 | amp_mean | Subcarrier | Medium — mean amplitude across 56 subcarriers |
|
||||
| 8 | amp_std | Subcarrier | **High** — amplitude spread (motion increases spread) |
|
||||
| 9 | amp_skew | Subcarrier | Medium — asymmetry of amplitude distribution |
|
||||
| 10 | amp_kurt | Subcarrier | **High** — peakedness (presence creates peaks) |
|
||||
| 11 | amp_iqr | Subcarrier | Medium — inter-quartile range |
|
||||
| 12 | amp_entropy | Subcarrier | **High** — spectral entropy (motion increases disorder) |
|
||||
| 13 | amp_max | Subcarrier | Medium — peak amplitude value |
|
||||
| 14 | amp_range | Subcarrier | Medium — amplitude dynamic range |
|
||||
|
||||
#### Training Algorithm
|
||||
- **Multiclass logistic regression** with softmax output
|
||||
- **Mini-batch SGD** (batch size 32, 200 epochs, linear learning rate decay)
|
||||
- **Z-score normalisation** using global mean/stddev computed from all training data
|
||||
- Per-class statistics (mean, stddev) stored for Mahalanobis distance fallback
|
||||
- Deterministic shuffling (LCG PRNG, seed 42) for reproducible results
|
||||
|
||||
#### Training Data Pipeline
|
||||
1. Record labeled CSI sessions via `POST /api/v1/recording/start {"id":"train_<label>"}`
|
||||
2. Filename-based label assignment: `*empty*`→absent, `*still*`→present_still, `*walking*`→present_moving, `*active*`→active
|
||||
3. Train via `POST /api/v1/adaptive/train`
|
||||
4. Model saved to `data/adaptive_model.json`, auto-loaded on server restart
|
||||
|
||||
#### Inference Pipeline
|
||||
1. Extract 15-feature vector from current CSI frame
|
||||
2. Z-score normalise using stored global mean/stddev
|
||||
3. Compute softmax probabilities across 4 classes
|
||||
4. Blend adaptive model confidence (70%) with smoothed threshold confidence (30%)
|
||||
5. Override classification only when adaptive model is loaded
|
||||
|
||||
### 3. API Endpoints
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/api/v1/adaptive/train` | Train classifier from `train_*` recordings |
|
||||
| GET | `/api/v1/adaptive/status` | Check model status, accuracy, class stats |
|
||||
| POST | `/api/v1/adaptive/unload` | Revert to threshold-based classification |
|
||||
| POST | `/api/v1/recording/start` | Start recording CSI frames (JSONL) |
|
||||
| POST | `/api/v1/recording/stop` | Stop recording |
|
||||
| GET | `/api/v1/recording/list` | List available recordings |
|
||||
|
||||
### 4. Vital Signs Smoothing
|
||||
|
||||
| Parameter | Value | Rationale |
|
||||
|-----------|-------|-----------|
|
||||
| Median window | 21 frames | ~2s of history, robust to transients |
|
||||
| Aggregation | Trimmed mean (middle 50%) | More stable than pure median, less noisy than raw mean |
|
||||
| EMA alpha | 0.02 | ~5s time constant — readings change very slowly |
|
||||
| HR dead-band | ±2 BPM | Prevents display creep from micro-fluctuations |
|
||||
| BR dead-band | ±0.5 BPM | Same for breathing rate |
|
||||
| HR max jump | 8 BPM/frame | Outlier rejection threshold |
|
||||
| BR max jump | 2 BPM/frame | Outlier rejection threshold |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Benefits
|
||||
- **Stable UI**: Vital signs readings hold steady for 5-10+ seconds instead of jumping every frame
|
||||
- **Environment adaptation**: Classifier learns the specific room's signal characteristics
|
||||
- **Graceful fallback**: If no adaptive model is loaded, threshold-based classification with smoothing still works
|
||||
- **No external dependencies**: Pure Rust implementation, no Python/ML frameworks needed
|
||||
- **Fast training**: 3,000+ frames train in <1 second on commodity hardware
|
||||
- **Portable model**: JSON serialisation, loadable on any platform
|
||||
|
||||
### Limitations
|
||||
- **Single-link**: With one ESP32, the feature space is limited. Multi-AP setups (ADR-029) would dramatically improve separability.
|
||||
- **No temporal features**: Current frame-level classification doesn't use sequence models (LSTM/Transformer). Could be added later.
|
||||
- **Label quality**: Training accuracy depends heavily on recording quality (distinct activities, actual room vacancy for "empty").
|
||||
- **Linear classifier**: Logistic regression may underfit non-linear decision boundaries. Could upgrade to 2-layer MLP if needed.
|
||||
|
||||
### Future Work
|
||||
- **Online learning**: Continuously update model weights from user corrections
|
||||
- **Sequence models**: Use sliding window of N frames as input for temporal pattern recognition
|
||||
- **Contrastive pretraining**: Leverage ADR-024 AETHER embeddings for self-supervised feature learning
|
||||
- **Multi-AP fusion**: Use ADR-029 multistatic sensing for richer feature space
|
||||
- **Edge deployment**: Export learned thresholds to ESP32 firmware (ADR-039 Tier 2) for on-device classification
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `crates/wifi-densepose-sensing-server/src/adaptive_classifier.rs` | Adaptive classifier module (feature extraction, training, inference) |
|
||||
| `crates/wifi-densepose-sensing-server/src/main.rs` | Smoothing pipeline, API endpoints, integration |
|
||||
| `ui/observatory/js/hud-controller.js` | UI-side lerp smoothing (4% per frame) |
|
||||
| `data/adaptive_model.json` | Trained model (auto-created by training endpoint) |
|
||||
| `data/recordings/train_*.jsonl` | Labeled training recordings |
|
||||
@@ -0,0 +1,122 @@
|
||||
# ADR-049: Cross-Platform WiFi Interface Detection and Graceful Degradation
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Status | Proposed |
|
||||
| Date | 2026-03-06 |
|
||||
| Deciders | ruv |
|
||||
| Depends on | ADR-013 (Feature-Level Sensing), ADR-025 (macOS CoreWLAN) |
|
||||
| Issue | [#148](https://github.com/ruvnet/wifi-densepose/issues/148) |
|
||||
|
||||
## Context
|
||||
|
||||
Users report `RuntimeError: Cannot read /proc/net/wireless` when running WiFi DensePose in environments where the Linux wireless proc filesystem is unavailable:
|
||||
|
||||
- **Docker containers** on macOS/Windows (Linux kernel detected, but no wireless subsystem)
|
||||
- **WSL2** without USB WiFi passthrough
|
||||
- **Headless Linux servers** without WiFi hardware
|
||||
- **Embedded Linux** boards without wireless-extensions support
|
||||
|
||||
The current architecture has two layers of defense:
|
||||
|
||||
1. **`ws_server.py`** (line 345-355) checks `os.path.exists("/proc/net/wireless")` before instantiating `LinuxWifiCollector` and falls back to `SimulatedCollector` if missing.
|
||||
2. **`rssi_collector.py`** `LinuxWifiCollector._validate_interface()` (line 178-196) raises a hard `RuntimeError` if `/proc/net/wireless` is missing or the interface isn't listed.
|
||||
|
||||
However, there are gaps:
|
||||
|
||||
- **Direct usage**: Any code that instantiates `LinuxWifiCollector` directly (outside `ws_server.py`) hits the unguarded `RuntimeError` with no fallback.
|
||||
- **Error message**: The RuntimeError message tells users to "use SimulatedCollector instead" but doesn't explain how.
|
||||
- **No auto-detection**: The collector selection logic is duplicated between `ws_server.py` and `install.sh` with no shared platform-detection utility.
|
||||
- **Partial `/proc/net/wireless`**: The file may exist (e.g., kernel module loaded) but contain no interfaces, producing a confusing "interface not found" error instead of a clean fallback.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Platform-Aware Collector Factory
|
||||
|
||||
Introduce a `create_collector()` factory function in `rssi_collector.py` that encapsulates the platform detection and fallback chain:
|
||||
|
||||
```python
|
||||
def create_collector(
|
||||
preferred: str = "auto",
|
||||
interface: str = "wlan0",
|
||||
sample_rate_hz: float = 10.0,
|
||||
) -> BaseCollector:
|
||||
"""
|
||||
Create the best available WiFi collector for the current platform.
|
||||
|
||||
Resolution order (when preferred="auto"):
|
||||
1. ESP32 CSI (if UDP port 5005 is receiving frames)
|
||||
2. Platform-native WiFi:
|
||||
- Linux: LinuxWifiCollector (requires /proc/net/wireless + active interface)
|
||||
- Windows: WindowsWifiCollector (netsh wlan)
|
||||
- macOS: MacosWifiCollector (CoreWLAN)
|
||||
3. SimulatedCollector (always available)
|
||||
|
||||
Raises nothing — always returns a usable collector.
|
||||
"""
|
||||
```
|
||||
|
||||
### 2. Soft Validation in LinuxWifiCollector
|
||||
|
||||
Replace the hard `RuntimeError` in `_validate_interface()` with a class method that returns availability status without raising:
|
||||
|
||||
```python
|
||||
@classmethod
|
||||
def is_available(cls, interface: str = "wlan0") -> tuple[bool, str]:
|
||||
"""Check if Linux WiFi collection is possible. Returns (available, reason)."""
|
||||
if not os.path.exists("/proc/net/wireless"):
|
||||
return False, "/proc/net/wireless not found (Docker, WSL, or no wireless subsystem)"
|
||||
with open("/proc/net/wireless") as f:
|
||||
content = f.read()
|
||||
if interface not in content:
|
||||
names = cls._parse_interface_names(content)
|
||||
return False, f"Interface '{interface}' not in /proc/net/wireless. Available: {names}"
|
||||
return True, "ok"
|
||||
```
|
||||
|
||||
The existing `_validate_interface()` continues to raise `RuntimeError` for direct callers who need fail-fast behavior, but `create_collector()` uses `is_available()` to probe without exceptions.
|
||||
|
||||
### 3. Structured Fallback Logging
|
||||
|
||||
When auto-detection skips a collector, log at `WARNING` level with actionable context:
|
||||
|
||||
```
|
||||
WiFi collector: LinuxWifiCollector unavailable (/proc/net/wireless not found — likely Docker/WSL).
|
||||
WiFi collector: Falling back to SimulatedCollector. For real sensing, connect ESP32 nodes via UDP:5005.
|
||||
```
|
||||
|
||||
### 4. Consolidate Platform Detection
|
||||
|
||||
Remove duplicated platform-detection logic from `ws_server.py` and `install.sh`. Both should use `create_collector()` (Python) or a shared `detect_wifi_platform()` shell function.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Zero-crash startup**: `create_collector("auto")` never raises — Docker, WSL, and headless users get `SimulatedCollector` automatically with a clear log message.
|
||||
- **Single detection path**: Platform logic lives in one place (`rssi_collector.py`), reducing drift between `ws_server.py`, `install.sh`, and future entry points.
|
||||
- **Better DX**: Error messages explain *why* a collector is unavailable and *what to do* (connect ESP32, install WiFi driver, etc.).
|
||||
|
||||
### Negative
|
||||
|
||||
- **SimulatedCollector may mask hardware issues**: Users with real WiFi hardware that fails detection might unknowingly run on simulated data. Mitigated by the `WARNING`-level log.
|
||||
- **Breaking change for direct `LinuxWifiCollector` callers**: Code that catches `RuntimeError` from `_validate_interface()` as a signal needs to migrate to `is_available()` or `create_collector()`. This is a minor change — there are no known external consumers.
|
||||
|
||||
### Neutral
|
||||
|
||||
- `_validate_interface()` behavior is unchanged for existing direct callers — this is additive.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
1. Add `create_collector()` and `BaseCollector.is_available()` to `v1/src/sensing/rssi_collector.py`
|
||||
2. Refactor `ws_server.py` `_init_collector()` to call `create_collector()`
|
||||
3. Update `install.sh` `detect_wifi_hardware()` to use shared detection logic
|
||||
4. Add unit tests for each platform path (mock `/proc/net/wireless` presence/absence)
|
||||
5. Comment on issue #148 with the fix
|
||||
|
||||
## References
|
||||
|
||||
- Issue #148: RuntimeError: Cannot read /proc/net/wireless
|
||||
- ADR-013: Feature-Level Sensing on Commodity Gear
|
||||
- ADR-025: macOS CoreWLAN WiFi Sensing
|
||||
- [Linux /proc/net/wireless documentation](https://www.kernel.org/doc/html/latest/networking/statistics.html)
|
||||
@@ -0,0 +1,100 @@
|
||||
# ADR-050: Quality Engineering Response — Security Hardening & Code Quality
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-03-06 |
|
||||
| Deciders | ruv |
|
||||
| Depends on | ADR-032 (Multistatic Mesh Security) |
|
||||
| Issue | [#170](https://github.com/ruvnet/wifi-densepose/issues/170) |
|
||||
|
||||
## Context
|
||||
|
||||
An independent quality engineering analysis ([issue #170](https://github.com/ruvnet/wifi-densepose/issues/170)) identified 7 critical findings across the Rust codebase. After verification against the source code, the following findings are confirmed and require action:
|
||||
|
||||
### Confirmed Critical Findings
|
||||
|
||||
| # | Finding | Location | Verified |
|
||||
|---|---------|----------|----------|
|
||||
| 1 | Fake HMAC in `secure_tdm.rs` — XOR fold with hardcoded key | `hardware/src/esp32/secure_tdm.rs:253` | YES — comments say "sufficient for testing" |
|
||||
| 2 | `sensing-server/main.rs` is 3,741 lines — CC=65, god object | `sensing-server/src/main.rs` | YES — confirmed 3,741 lines |
|
||||
| 3 | WebSocket server has zero authentication | Rust WS codebase | YES — no auth/token checks found |
|
||||
| 4 | Zero security tests in Rust codebase | Entire workspace | YES — no auth/injection/tampering tests |
|
||||
| 5 | 54K fps claim has no supporting benchmark | No criterion benchmarks | YES — no benchmarks exist |
|
||||
|
||||
### Findings Requiring Further Investigation
|
||||
|
||||
| # | Finding | Status |
|
||||
|---|---------|--------|
|
||||
| 6 | Unauthenticated OTA firmware endpoint | Not found in Rust code — may be ESP32 C firmware level |
|
||||
| 7 | WASM upload without mandatory signatures | Needs review of WASM loader |
|
||||
| 8 | O(n^2) autocorrelation in heart rate detection | Needs profiling to confirm impact |
|
||||
|
||||
## Decision
|
||||
|
||||
Address findings in 3 priority sprints as recommended by the report.
|
||||
|
||||
### Sprint 1: Security (Blocks Deployment)
|
||||
|
||||
1. **Replace fake HMAC with real HMAC-SHA256** in `secure_tdm.rs`
|
||||
- Use the `hmac` + `sha2` crates (already in `Cargo.lock`)
|
||||
- Remove XOR fold implementation
|
||||
- Add key derivation (no more hardcoded keys)
|
||||
|
||||
2. **Add WebSocket authentication**
|
||||
- Token-based auth on WS upgrade handshake
|
||||
- Optional API key for local-network deployments
|
||||
- Configurable via environment variable
|
||||
|
||||
3. **Add security test suite**
|
||||
- Auth bypass attempts
|
||||
- Malformed CSI frame injection
|
||||
- Protocol tampering (TDM beacon replay, nonce reuse)
|
||||
|
||||
### Sprint 2: Code Quality & Testability
|
||||
|
||||
4. **Decompose `main.rs`** (3,741 lines -> ~14 focused modules)
|
||||
- Extract HTTP routes, WebSocket handler, CSI pipeline, config, state
|
||||
- Target: no file over 500 lines
|
||||
|
||||
5. **Add criterion benchmarks**
|
||||
- CSI frame parsing throughput
|
||||
- Signal processing pipeline latency
|
||||
- WebSocket broadcast fanout
|
||||
|
||||
### Sprint 3: Functional Verification
|
||||
|
||||
6. **Vital sign accuracy verification**
|
||||
- Reference signal tests with known BPM
|
||||
- False-negative rate measurement
|
||||
|
||||
7. **Fix O(n^2) autocorrelation** (if confirmed by profiling)
|
||||
- Replace brute-force lag with FFT-based autocorrelation
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Addresses all critical security findings before any production deployment
|
||||
- `main.rs` decomposition enables unit testing of server components
|
||||
- Criterion benchmarks provide verifiable performance claims
|
||||
- Security test suite prevents regression
|
||||
|
||||
### Negative
|
||||
|
||||
- Sprint 1 security changes are breaking for any existing TDM mesh deployments (fake HMAC -> real HMAC requires firmware update)
|
||||
- `main.rs` decomposition is a large refactor with merge conflict risk
|
||||
|
||||
### Neutral
|
||||
|
||||
- The report correctly identifies that life-safety claims (disaster detection, vital signs) require rigorous verification — this is an ongoing process, not a single sprint
|
||||
|
||||
## Acknowledgment
|
||||
|
||||
Thanks to [@proffesor-for-testing](https://github.com/proffesor-for-testing) for the thorough 10-report analysis. The full report is archived at the [original gist](https://gist.github.com/proffesor-for-testing/02321e3f272720aa94484fffec6ab19b).
|
||||
|
||||
## References
|
||||
|
||||
- Issue #170: Quality Engineering Analysis
|
||||
- ADR-032: Multistatic Mesh Security Hardening
|
||||
- ADR-028: ESP32 Capability Audit
|
||||
@@ -0,0 +1,621 @@
|
||||
# ADR-052 Appendix: DDD Bounded Contexts — Tauri Desktop Frontend
|
||||
|
||||
This document maps out the domain model for the RuView Tauri desktop application
|
||||
described in ADR-052. It defines bounded contexts, their aggregates, entities,
|
||||
value objects, and the domain events flowing between them.
|
||||
|
||||
## Context Map
|
||||
|
||||
```
|
||||
+-------------------+ +---------------------+ +--------------------+
|
||||
| | | | | |
|
||||
| Device Discovery |------>| Firmware Management |------>| Configuration / |
|
||||
| | | | | Provisioning |
|
||||
+-------------------+ +---------------------+ +--------------------+
|
||||
| | |
|
||||
| | |
|
||||
v v v
|
||||
+-------------------+ +---------------------+ +--------------------+
|
||||
| | | | | |
|
||||
| Sensing Pipeline |<------| Edge Module | | Visualization |
|
||||
| | | (WASM) | | |
|
||||
+-------------------+ +---------------------+ +--------------------+
|
||||
|
||||
Relationship types:
|
||||
-----> Upstream/Downstream (upstream publishes events, downstream consumes)
|
||||
<----- Conformist (downstream conforms to upstream's model)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Device Discovery Context
|
||||
|
||||
**Purpose**: Find, identify, and monitor ESP32 CSI nodes on the local network.
|
||||
|
||||
**Upstream of**: Firmware Management, Configuration, Sensing Pipeline, Visualization
|
||||
|
||||
### Aggregates
|
||||
|
||||
#### `NodeRegistry` (Aggregate Root)
|
||||
|
||||
Maintains the authoritative list of all known nodes. Merges discovery results
|
||||
from multiple strategies (mDNS, UDP probe, HTTP sweep) and deduplicates by MAC
|
||||
address.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `nodes` | `Map<MacAddress, Node>` | All discovered nodes keyed by MAC |
|
||||
| `scan_state` | `ScanState` | Idle, Scanning, Error |
|
||||
| `last_scan` | `DateTime<Utc>` | Timestamp of last completed scan |
|
||||
|
||||
**Invariant**: No two nodes may share the same MAC address. If a node is
|
||||
discovered via multiple strategies, the most recent data wins.
|
||||
|
||||
**Persistence**: The registry is persisted to `~/.ruview/nodes.db` (SQLite via
|
||||
`rusqlite`). On startup, all previously known nodes are loaded as `Offline` and
|
||||
reconciled against a fresh discovery scan. This means the app **remembers the
|
||||
mesh** across restarts — critical for field deployments where nodes may be
|
||||
temporarily powered off.
|
||||
|
||||
#### `Node` (Entity)
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `mac` | `MacAddress` (VO) | IEEE 802.11 MAC address (unique identity) |
|
||||
| `ip` | `IpAddr` | Current IP address (may change on DHCP renewal) |
|
||||
| `hostname` | `Option<String>` | mDNS hostname |
|
||||
| `node_id` | `u8` | NVS-provisioned node ID |
|
||||
| `firmware_version` | `Option<SemVer>` | Firmware version string |
|
||||
| `health` | `HealthStatus` (VO) | Online / Offline / Degraded |
|
||||
| `discovery_method` | `DiscoveryMethod` (VO) | How this node was found |
|
||||
| `last_seen` | `DateTime<Utc>` | Last successful contact |
|
||||
| `tdm_config` | `Option<TdmConfig>` (VO) | TDM slot assignment |
|
||||
| `edge_tier` | `Option<u8>` | Edge processing tier (0/1/2) |
|
||||
|
||||
### Value Objects
|
||||
|
||||
- `MacAddress` — 6-byte hardware address, formatted as `AA:BB:CC:DD:EE:FF`
|
||||
- `HealthStatus` — enum: `Online`, `Offline`, `Degraded(reason: String)`
|
||||
- `DiscoveryMethod` — enum: `Mdns`, `UdpProbe`, `HttpSweep`, `Manual`
|
||||
- `TdmConfig` — `{ slot_index: u8, total_nodes: u8 }`
|
||||
- `SemVer` — semantic version `major.minor.patch`
|
||||
|
||||
### Domain Events
|
||||
|
||||
| Event | Payload | Consumers |
|
||||
|-------|---------|-----------|
|
||||
| `NodeDiscovered` | `{ node: Node }` | Firmware Mgmt (check for updates), Visualization (add to mesh graph) |
|
||||
| `NodeWentOffline` | `{ mac: MacAddress, last_seen: DateTime }` | Visualization (gray out node), Sensing Pipeline (remove from active set) |
|
||||
| `NodeCameOnline` | `{ node: Node }` | Visualization (restore node), Sensing Pipeline (re-add) |
|
||||
| `NodeHealthChanged` | `{ mac: MacAddress, old: HealthStatus, new: HealthStatus }` | Visualization (update indicator) |
|
||||
| `ScanCompleted` | `{ found: usize, new: usize, lost: usize }` | Dashboard (update summary) |
|
||||
|
||||
### Anti-Corruption Layer
|
||||
|
||||
When receiving data from the ESP32 OTA status endpoint (`GET /ota/status`), the
|
||||
response format is owned by the firmware and may change across firmware versions.
|
||||
The ACL translates the raw JSON response into `Node` entity fields:
|
||||
|
||||
```rust
|
||||
/// ACL: Translate ESP32 OTA status response to Node fields.
|
||||
fn translate_ota_status(raw: &serde_json::Value) -> Result<NodePatch, AclError> {
|
||||
NodePatch {
|
||||
firmware_version: raw["version"].as_str().map(SemVer::parse).transpose()?,
|
||||
uptime_secs: raw["uptime_s"].as_u64(),
|
||||
free_heap: raw["free_heap"].as_u64(),
|
||||
// Firmware may add fields in future versions — unknown fields are ignored
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Firmware Management Context
|
||||
|
||||
**Purpose**: Flash, update, and verify firmware on ESP32 nodes.
|
||||
|
||||
**Upstream of**: Configuration (a fresh flash triggers provisioning)
|
||||
**Downstream of**: Device Discovery (needs node list and serial port info)
|
||||
|
||||
### Aggregates
|
||||
|
||||
#### `FlashSession` (Aggregate Root)
|
||||
|
||||
Represents a single firmware flashing operation from start to completion. Each
|
||||
session has a lifecycle: Created -> Connecting -> Erasing -> Writing -> Verifying ->
|
||||
Completed | Failed.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | `Uuid` | Session identifier |
|
||||
| `port` | `SerialPort` (VO) | Target serial port |
|
||||
| `firmware` | `FirmwareBinary` (Entity) | The binary being flashed |
|
||||
| `chip` | `ChipType` (VO) | Target chip (ESP32, ESP32-S3, ESP32-C3) |
|
||||
| `phase` | `FlashPhase` (VO) | Current phase of the flash operation |
|
||||
| `progress` | `Progress` (VO) | Bytes written / total, speed |
|
||||
| `started_at` | `DateTime<Utc>` | When the session started |
|
||||
| `error` | `Option<String>` | Error message if failed |
|
||||
|
||||
**Invariant**: Only one `FlashSession` may be active per serial port at a time.
|
||||
|
||||
#### `FirmwareBinary` (Entity)
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `path` | `PathBuf` | Filesystem path to the `.bin` file |
|
||||
| `size_bytes` | `u64` | Binary size |
|
||||
| `version` | `Option<SemVer>` | Extracted from ESP32 image header |
|
||||
| `chip_type` | `Option<ChipType>` | Detected from image magic bytes |
|
||||
| `checksum` | `Sha256Hash` (VO) | SHA-256 of the binary |
|
||||
|
||||
#### `OtaSession` (Aggregate Root)
|
||||
|
||||
Represents an over-the-air firmware update to a running node.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | `Uuid` | Session identifier |
|
||||
| `target_node` | `MacAddress` | Target node MAC |
|
||||
| `target_ip` | `IpAddr` | Target node IP |
|
||||
| `firmware` | `FirmwareBinary` | The binary being pushed |
|
||||
| `psk` | `Option<SecureString>` | PSK for authentication (ADR-050) |
|
||||
| `phase` | `OtaPhase` | Uploading / Rebooting / Verifying / Done / Failed |
|
||||
| `progress` | `Progress` | Upload progress |
|
||||
|
||||
#### `BatchOtaSession` (Aggregate Root)
|
||||
|
||||
Coordinates rolling firmware updates across multiple mesh nodes. Prevents all
|
||||
nodes from rebooting simultaneously, which would collapse the sensing network.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | `Uuid` | Batch session identifier |
|
||||
| `firmware` | `FirmwareBinary` | The binary being deployed |
|
||||
| `strategy` | `OtaStrategy` | `Sequential`, `TdmSafe`, `Parallel` |
|
||||
| `max_concurrent` | `usize` | Max nodes updating at once |
|
||||
| `batch_delay_secs` | `u64` | Delay between batches |
|
||||
| `fail_fast` | `bool` | Abort remaining on first failure |
|
||||
| `node_states` | `Map<MacAddress, BatchNodeState>` | Per-node progress |
|
||||
|
||||
**Invariant**: In `TdmSafe` mode, adjacent TDM slots are never updated
|
||||
concurrently. Even-slot nodes update first, then odd-slot nodes.
|
||||
|
||||
**Lifecycle**: `Planning → InProgress → Completed | PartialFailure | Aborted`
|
||||
|
||||
- `BatchNodeState` — enum: `Queued`, `Uploading(Progress)`, `Rebooting`, `Verifying`, `Done`, `Failed(String)`, `Skipped`
|
||||
- `OtaStrategy` — enum:
|
||||
- `Sequential` — one node at a time, wait for rejoin
|
||||
- `TdmSafe` — update non-adjacent slots to maintain sensing coverage
|
||||
- `Parallel` — all at once (development only)
|
||||
|
||||
### Value Objects
|
||||
|
||||
- `SerialPort` — `{ name: String, vid: u16, pid: u16, manufacturer: Option<String> }`
|
||||
- `ChipType` — enum: `Esp32`, `Esp32s3`, `Esp32c3`
|
||||
- `FlashPhase` — enum: `Connecting`, `Erasing`, `Writing`, `Verifying`, `Completed`, `Failed`
|
||||
- `OtaPhase` — enum: `Uploading`, `Rebooting`, `Verifying`, `Completed`, `Failed`
|
||||
- `Progress` — `{ bytes_done: u64, bytes_total: u64, speed_bps: u64 }`
|
||||
- `Sha256Hash` — 32-byte hash
|
||||
- `SecureString` — zeroized-on-drop string for PSK tokens
|
||||
|
||||
### Domain Events
|
||||
|
||||
| Event | Payload | Consumers |
|
||||
|-------|---------|-----------|
|
||||
| `FlashStarted` | `{ session_id, port, firmware_version }` | UI (show progress) |
|
||||
| `FlashProgress` | `{ session_id, phase, progress }` | UI (update progress bar) |
|
||||
| `FlashCompleted` | `{ session_id, duration_secs }` | Configuration (trigger provisioning prompt) |
|
||||
| `FlashFailed` | `{ session_id, error }` | UI (show error) |
|
||||
| `OtaStarted` | `{ session_id, target_mac, firmware_version }` | Discovery (mark node as updating) |
|
||||
| `OtaCompleted` | `{ session_id, target_mac, new_version }` | Discovery (refresh node info) |
|
||||
| `OtaFailed` | `{ session_id, target_mac, error }` | UI (show error) |
|
||||
| `BatchOtaStarted` | `{ batch_id, strategy, node_count }` | UI (show batch progress) |
|
||||
| `BatchNodeUpdated` | `{ batch_id, mac, state }` | UI (update per-node status), Discovery (refresh) |
|
||||
| `BatchOtaCompleted` | `{ batch_id, succeeded, failed, skipped }` | UI (show summary), Discovery (full rescan) |
|
||||
|
||||
### Anti-Corruption Layer
|
||||
|
||||
The `espflash` crate has its own error types and progress reporting model. The
|
||||
ACL translates these into domain events:
|
||||
|
||||
```rust
|
||||
/// ACL: Translate espflash progress callbacks to domain FlashProgress events.
|
||||
impl From<espflash::ProgressCallbackMessage> for FlashProgress {
|
||||
fn from(msg: espflash::ProgressCallbackMessage) -> Self {
|
||||
match msg {
|
||||
espflash::ProgressCallbackMessage::Connecting => FlashProgress {
|
||||
phase: FlashPhase::Connecting,
|
||||
progress: Progress::indeterminate(),
|
||||
},
|
||||
espflash::ProgressCallbackMessage::Erasing { addr, total } => FlashProgress {
|
||||
phase: FlashPhase::Erasing,
|
||||
progress: Progress::new(addr as u64, total as u64),
|
||||
},
|
||||
// ... etc
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Configuration / Provisioning Context
|
||||
|
||||
**Purpose**: Manage NVS configuration for ESP32 nodes — WiFi credentials, network
|
||||
targets, TDM mesh settings, edge intelligence parameters, WASM security keys.
|
||||
|
||||
**Downstream of**: Device Discovery (needs serial port), Firmware Management (post-flash provisioning)
|
||||
|
||||
### Aggregates
|
||||
|
||||
#### `ProvisioningSession` (Aggregate Root)
|
||||
|
||||
Represents a single NVS write or read operation on a connected ESP32.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | `Uuid` | Session identifier |
|
||||
| `port` | `SerialPort` (VO) | Target serial port |
|
||||
| `config` | `NodeConfig` (Entity) | Configuration to write |
|
||||
| `direction` | `Direction` | Read or Write |
|
||||
| `phase` | `ProvisionPhase` | Generating / Flashing / Verifying / Done |
|
||||
|
||||
#### `NodeConfig` (Entity)
|
||||
|
||||
The full set of NVS key-value pairs for a single node. Maps directly to the
|
||||
firmware's `nvs_config_t` struct (see `firmware/esp32-csi-node/main/nvs_config.h`).
|
||||
|
||||
| Field | Type | NVS Key | Description |
|
||||
|-------|------|---------|-------------|
|
||||
| `wifi_ssid` | `Option<String>` | `ssid` | WiFi SSID |
|
||||
| `wifi_password` | `Option<SecureString>` | `password` | WiFi password |
|
||||
| `target_ip` | `Option<IpAddr>` | `target_ip` | Aggregator IP |
|
||||
| `target_port` | `Option<u16>` | `target_port` | Aggregator UDP port |
|
||||
| `node_id` | `Option<u8>` | `node_id` | Node identifier |
|
||||
| `tdm_slot` | `Option<u8>` | `tdm_slot` | TDM slot index |
|
||||
| `tdm_total` | `Option<u8>` | `tdm_nodes` | Total TDM nodes |
|
||||
| `edge_tier` | `Option<u8>` | `edge_tier` | Processing tier |
|
||||
| `hop_count` | `Option<u8>` | `hop_count` | Channel hop count |
|
||||
| `channel_list` | `Option<Vec<u8>>` | `chan_list` | Channel sequence |
|
||||
| `dwell_ms` | `Option<u32>` | `dwell_ms` | Hop dwell time |
|
||||
| `power_duty` | `Option<u8>` | `power_duty` | Power duty cycle |
|
||||
| `presence_thresh` | `Option<u16>` | `pres_thresh` | Presence threshold |
|
||||
| `fall_thresh` | `Option<u16>` | `fall_thresh` | Fall detection threshold |
|
||||
| `vital_window` | `Option<u16>` | `vital_win` | Vital sign window |
|
||||
| `vital_interval_ms` | `Option<u16>` | `vital_int` | Vital sign interval |
|
||||
| `top_k_count` | `Option<u8>` | `subk_count` | Top-K subcarriers |
|
||||
| `wasm_max_modules` | `Option<u8>` | `wasm_max` | Max WASM modules |
|
||||
| `wasm_verify` | `Option<bool>` | `wasm_verify` | Require WASM signature |
|
||||
| `wasm_pubkey` | `Option<[u8; 32]>` | `wasm_pubkey` | Ed25519 public key |
|
||||
| `ota_psk` | `Option<SecureString>` | `ota_psk` | OTA pre-shared key |
|
||||
|
||||
**Invariant**: `tdm_slot < tdm_total` when both are set.
|
||||
**Invariant**: `channel_list.len() == hop_count` when both are set.
|
||||
**Invariant**: `10 <= power_duty <= 100`.
|
||||
|
||||
#### `MeshConfig` (Entity)
|
||||
|
||||
A mesh-level configuration that generates per-node `NodeConfig` instances.
|
||||
Corresponds to ADR-044 Phase 2 (config file provisioning).
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `common` | `NodeConfig` | Shared settings (WiFi, target IP, edge tier) |
|
||||
| `nodes` | `Vec<MeshNodeEntry>` | Per-node overrides (port, node_id, tdm_slot) |
|
||||
|
||||
```rust
|
||||
pub struct MeshNodeEntry {
|
||||
pub port: String,
|
||||
pub node_id: u8,
|
||||
pub tdm_slot: u8,
|
||||
// All other fields inherited from common
|
||||
}
|
||||
```
|
||||
|
||||
**Invariant**: `tdm_total` is automatically computed as `nodes.len()`.
|
||||
|
||||
### Value Objects
|
||||
|
||||
- `ProvisionPhase` — enum: `Generating`, `Flashing`, `Verifying`, `Completed`, `Failed`
|
||||
- `Direction` — enum: `Read`, `Write`
|
||||
- `Preset` — enum: `Basic`, `Vitals`, `Mesh3`, `Mesh6Vitals` (ADR-044 Phase 3)
|
||||
|
||||
### Domain Events
|
||||
|
||||
| Event | Payload | Consumers |
|
||||
|-------|---------|-----------|
|
||||
| `NodeProvisioned` | `{ port, node_id, config_summary }` | Discovery (trigger re-scan), UI (show success) |
|
||||
| `NvsReadCompleted` | `{ port, config: NodeConfig }` | UI (populate form) |
|
||||
| `ProvisionFailed` | `{ port, error }` | UI (show error) |
|
||||
| `MeshProvisionStarted` | `{ node_count }` | UI (show batch progress) |
|
||||
| `MeshProvisionCompleted` | `{ success_count, fail_count }` | UI (show summary) |
|
||||
|
||||
---
|
||||
|
||||
## 4. Sensing Pipeline Context
|
||||
|
||||
**Purpose**: Control the sensing server process, receive real-time CSI data, and
|
||||
manage the signal processing pipeline.
|
||||
|
||||
**Downstream of**: Device Discovery (needs node IPs for data attribution)
|
||||
|
||||
### Aggregates
|
||||
|
||||
#### `SensingServer` (Aggregate Root)
|
||||
|
||||
Represents the managed sensing server child process.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `state` | `ServerState` (VO) | Stopped / Starting / Running / Stopping / Crashed |
|
||||
| `config` | `ServerConfig` (VO) | Port configuration, log level, model paths |
|
||||
| `pid` | `Option<u32>` | OS process ID when running |
|
||||
| `started_at` | `Option<DateTime<Utc>>` | Start timestamp |
|
||||
| `log_buffer` | `RingBuffer<LogEntry>` | Last N log lines |
|
||||
| `ws_url` | `Option<Url>` | WebSocket URL for live data |
|
||||
|
||||
**Invariant**: Only one `SensingServer` process may be managed at a time.
|
||||
|
||||
#### `SensingSession` (Entity)
|
||||
|
||||
An active connection to the sensing server's WebSocket for receiving real-time data.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `connection_state` | `WsState` | Connecting / Connected / Disconnected |
|
||||
| `frames_received` | `u64` | Total CSI frames received this session |
|
||||
| `last_frame_at` | `Option<DateTime<Utc>>` | Timestamp of last received frame |
|
||||
| `subscriptions` | `HashSet<DataChannel>` | Which data streams are active |
|
||||
|
||||
### Value Objects
|
||||
|
||||
- `ServerState` — enum: `Stopped`, `Starting`, `Running`, `Stopping`, `Crashed(exit_code: i32)`
|
||||
- `ServerConfig` — `{ http_port: u16, ws_port: u16, udp_port: u16, model_dir: PathBuf, log_level: Level }`
|
||||
- `LogEntry` — `{ timestamp: DateTime, level: Level, target: String, message: String }`
|
||||
- `DataChannel` — enum: `CsiFrames`, `PoseUpdates`, `VitalSigns`, `ActivityClassification`
|
||||
- `WsState` — enum: `Connecting`, `Connected`, `Disconnected(reason: String)`
|
||||
|
||||
### Domain Events
|
||||
|
||||
| Event | Payload | Consumers |
|
||||
|-------|---------|-----------|
|
||||
| `ServerStarted` | `{ pid, ports: ServerConfig }` | UI (enable sensing view), Discovery (start health polling via WS) |
|
||||
| `ServerStopped` | `{ exit_code, uptime_secs }` | UI (disable sensing view) |
|
||||
| `ServerCrashed` | `{ exit_code, last_log_lines }` | UI (show crash report) |
|
||||
| `CsiFrameReceived` | `{ node_id, timestamp, subcarrier_count }` | Visualization (update charts) |
|
||||
| `PoseUpdated` | `{ persons: Vec<PersonPose> }` | Visualization (draw skeletons) |
|
||||
| `VitalSignUpdate` | `{ node_id, bpm, breath_rate }` | Visualization (update vitals chart) |
|
||||
| `ActivityDetected` | `{ label, confidence }` | Visualization (show activity) |
|
||||
|
||||
---
|
||||
|
||||
## 5. Edge Module (WASM) Context
|
||||
|
||||
**Purpose**: Upload, manage, and monitor WASM edge processing modules running
|
||||
on ESP32 nodes.
|
||||
|
||||
**Downstream of**: Device Discovery (needs node IPs and WASM capability info)
|
||||
**Upstream of**: Sensing Pipeline (WASM modules emit edge-processed events)
|
||||
|
||||
### Aggregates
|
||||
|
||||
#### `ModuleRegistry` (Aggregate Root)
|
||||
|
||||
Tracks all WASM modules across all nodes.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `modules` | `Map<(MacAddress, ModuleId), WasmModule>` | Per-node module inventory |
|
||||
|
||||
#### `WasmModule` (Entity)
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | `ModuleId` (VO) | Node-assigned module identifier |
|
||||
| `name` | `String` | Filename of the uploaded `.wasm` |
|
||||
| `size_bytes` | `u64` | Module size |
|
||||
| `status` | `ModuleStatus` (VO) | Loaded / Running / Stopped / Error |
|
||||
| `node_mac` | `MacAddress` | Which node this module runs on |
|
||||
| `uploaded_at` | `DateTime<Utc>` | Upload timestamp |
|
||||
| `signed` | `bool` | Whether the module has an Ed25519 signature |
|
||||
|
||||
### Value Objects
|
||||
|
||||
- `ModuleId` — string identifier assigned by the node firmware
|
||||
- `ModuleStatus` — enum: `Loaded`, `Running`, `Stopped`, `Error(String)`
|
||||
|
||||
### Domain Events
|
||||
|
||||
| Event | Payload | Consumers |
|
||||
|-------|---------|-----------|
|
||||
| `ModuleUploaded` | `{ node_mac, module_id, name, size }` | UI (refresh list) |
|
||||
| `ModuleStarted` | `{ node_mac, module_id }` | UI (update status) |
|
||||
| `ModuleStopped` | `{ node_mac, module_id }` | UI (update status) |
|
||||
| `ModuleUnloaded` | `{ node_mac, module_id }` | UI (remove from list) |
|
||||
| `ModuleError` | `{ node_mac, module_id, error }` | UI (show error) |
|
||||
|
||||
### Anti-Corruption Layer
|
||||
|
||||
The ESP32 WASM management HTTP API (`/wasm/*` on port 8032) returns raw JSON
|
||||
with firmware-specific field names. The ACL normalizes these:
|
||||
|
||||
```rust
|
||||
/// ACL: Translate ESP32 WASM list response to domain WasmModule entities.
|
||||
fn translate_wasm_list(raw: &[serde_json::Value]) -> Vec<WasmModule> {
|
||||
raw.iter().filter_map(|entry| {
|
||||
Some(WasmModule {
|
||||
id: ModuleId(entry["id"].as_str()?.to_string()),
|
||||
name: entry["name"].as_str().unwrap_or("unknown").to_string(),
|
||||
size_bytes: entry["size"].as_u64().unwrap_or(0),
|
||||
status: match entry["state"].as_str() {
|
||||
Some("running") => ModuleStatus::Running,
|
||||
Some("stopped") => ModuleStatus::Stopped,
|
||||
Some("loaded") => ModuleStatus::Loaded,
|
||||
other => ModuleStatus::Error(
|
||||
format!("Unknown state: {:?}", other)
|
||||
),
|
||||
},
|
||||
// ...
|
||||
})
|
||||
}).collect()
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Visualization Context
|
||||
|
||||
**Purpose**: Render real-time and historical sensing data — CSI heatmaps, pose
|
||||
skeletons, vital sign charts, mesh topology graphs.
|
||||
|
||||
**Downstream of**: Sensing Pipeline (receives data events), Device Discovery (needs
|
||||
node metadata for labeling)
|
||||
|
||||
This context is **purely presentational** and contains no domain logic. It
|
||||
transforms domain events from other contexts into visual representations.
|
||||
|
||||
### Aggregates
|
||||
|
||||
None — this context is a **Query Model** (CQRS read side). It subscribes to
|
||||
domain events and projects them into view models.
|
||||
|
||||
### View Models
|
||||
|
||||
#### `DashboardView`
|
||||
|
||||
| Field | Source Context | Description |
|
||||
|-------|---------------|-------------|
|
||||
| `nodes` | Device Discovery | Node cards with health, version, signal quality |
|
||||
| `server` | Sensing Pipeline | Server status, uptime, port info |
|
||||
| `recent_activity` | All contexts | Timeline of recent events |
|
||||
|
||||
#### `SignalView`
|
||||
|
||||
| Field | Source Context | Description |
|
||||
|-------|---------------|-------------|
|
||||
| `csi_heatmap` | Sensing Pipeline | Subcarrier amplitude x time matrix |
|
||||
| `signal_field` | Sensing Pipeline | 2D signal strength grid |
|
||||
| `activity_label` | Sensing Pipeline | Current classification |
|
||||
| `confidence` | Sensing Pipeline | Classification confidence |
|
||||
|
||||
#### `PoseView`
|
||||
|
||||
| Field | Source Context | Description |
|
||||
|-------|---------------|-------------|
|
||||
| `persons` | Sensing Pipeline | Array of detected person skeletons |
|
||||
| `zones` | Sensing Pipeline | Active zones in the sensing area |
|
||||
|
||||
#### `VitalsView`
|
||||
|
||||
| Field | Source Context | Description |
|
||||
|-------|---------------|-------------|
|
||||
| `breathing_rate_bpm` | Sensing Pipeline | Per-node breathing rate time series |
|
||||
| `heart_rate_bpm` | Sensing Pipeline | Per-node heart rate time series |
|
||||
|
||||
#### `MeshView`
|
||||
|
||||
| Field | Source Context | Description |
|
||||
|-------|---------------|-------------|
|
||||
| `nodes` | Device Discovery | Positioned nodes for graph layout |
|
||||
| `edges` | Device Discovery | Inter-node visibility/connectivity |
|
||||
| `tdm_timeline` | Device Discovery | TDM slot schedule visualization |
|
||||
| `sync_status` | Sensing Pipeline | Per-node sync status with server |
|
||||
|
||||
---
|
||||
|
||||
## Cross-Context Event Flow
|
||||
|
||||
```
|
||||
NodeDiscovered
|
||||
Device Discovery ─────────────────────────────────> Firmware Management
|
||||
│ │
|
||||
│ NodeDiscovered │ FlashCompleted
|
||||
│ NodeHealthChanged │
|
||||
├──────────────────> Visualization v
|
||||
│ Configuration
|
||||
│ NodeDiscovered │
|
||||
├──────────────────> Sensing Pipeline │ NodeProvisioned
|
||||
│ │
|
||||
│ v
|
||||
│ Device Discovery
|
||||
│ (re-scan triggered)
|
||||
│
|
||||
│ NodeDiscovered
|
||||
└──────────────────> Edge Module (WASM)
|
||||
│
|
||||
│ ModuleUploaded, ModuleStarted
|
||||
│
|
||||
v
|
||||
Sensing Pipeline
|
||||
│
|
||||
│ CsiFrameReceived, PoseUpdated, VitalSignUpdate
|
||||
│
|
||||
v
|
||||
Visualization
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
1. **Event Bus**: Domain events are dispatched via Tauri's event system
|
||||
(`app_handle.emit("event-name", payload)`). The frontend subscribes using
|
||||
`listen("event-name", callback)`. This provides natural cross-context
|
||||
communication without coupling contexts directly.
|
||||
|
||||
2. **State Isolation**: Each bounded context maintains its own `State<'_, T>`
|
||||
managed by Tauri. Contexts do not share mutable state directly — they
|
||||
communicate exclusively through events.
|
||||
|
||||
3. **Module Organization**: Each bounded context maps to a Rust module under
|
||||
`src/commands/` and `src/domain/`:
|
||||
|
||||
```
|
||||
src/
|
||||
commands/ # Tauri command handlers (application layer)
|
||||
discovery.rs # Device Discovery context commands
|
||||
flash.rs # Firmware Management context commands
|
||||
ota.rs # Firmware Management context commands
|
||||
provision.rs # Configuration context commands
|
||||
server.rs # Sensing Pipeline context commands
|
||||
wasm.rs # Edge Module context commands
|
||||
domain/ # Domain models (pure Rust, no Tauri dependency)
|
||||
discovery/
|
||||
mod.rs
|
||||
node.rs # Node entity, MacAddress VO
|
||||
registry.rs # NodeRegistry aggregate
|
||||
events.rs # Discovery domain events
|
||||
firmware/
|
||||
mod.rs
|
||||
binary.rs # FirmwareBinary entity
|
||||
flash.rs # FlashSession aggregate
|
||||
ota.rs # OtaSession aggregate
|
||||
events.rs
|
||||
config/
|
||||
mod.rs
|
||||
nvs.rs # NodeConfig entity
|
||||
mesh.rs # MeshConfig entity
|
||||
provision.rs # ProvisioningSession aggregate
|
||||
events.rs
|
||||
sensing/
|
||||
mod.rs
|
||||
server.rs # SensingServer aggregate
|
||||
session.rs # SensingSession entity
|
||||
events.rs
|
||||
wasm/
|
||||
mod.rs
|
||||
module.rs # WasmModule entity
|
||||
registry.rs # ModuleRegistry aggregate
|
||||
events.rs
|
||||
acl/ # Anti-corruption layers
|
||||
ota_status.rs # ESP32 OTA status response translator
|
||||
wasm_api.rs # ESP32 WASM API response translator
|
||||
espflash.rs # espflash crate adapter
|
||||
```
|
||||
|
||||
4. **Testing Strategy**: Domain modules under `src/domain/` have no Tauri
|
||||
dependency and can be tested with standard `cargo test`. Command handlers
|
||||
under `src/commands/` require Tauri test utilities for integration testing.
|
||||
|
||||
5. **Shared Kernel**: The `MacAddress`, `SemVer`, and `SecureString` value objects
|
||||
are shared across contexts. They live in a `src/domain/shared.rs` module.
|
||||
This is acceptable because they are immutable value objects with no behavior
|
||||
beyond validation and formatting.
|
||||
@@ -0,0 +1,810 @@
|
||||
# ADR-052: Tauri Desktop Frontend — RuView Hardware Management & Visualization
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Status | Proposed |
|
||||
| Date | 2026-03-06 |
|
||||
| Deciders | ruv |
|
||||
| Depends on | ADR-012 (ESP32 CSI Mesh), ADR-039 (Edge Intelligence), ADR-040 (WASM Programmable Sensing), ADR-044 (Provisioning Enhancements), ADR-050 (Security Hardening), ADR-051 (Server Decomposition) |
|
||||
| Issue | [#177](https://github.com/ruvnet/RuView/issues/177) |
|
||||
|
||||
## Context
|
||||
|
||||
RuView currently requires users to interact with multiple disconnected tools to manage a WiFi DensePose deployment:
|
||||
|
||||
| Task | Current Tool | Pain Point |
|
||||
|------|-------------|------------|
|
||||
| Flash firmware | `esptool.py` CLI | Requires Python, pip, correct chip/baud flags |
|
||||
| Provision NVS | `provision.py` CLI | 13+ flags, no GUI, no read-back |
|
||||
| OTA update | `curl POST :8032/ota` | Manual HTTP, PSK header construction |
|
||||
| WASM modules | `curl` to `:8032/wasm/*` | No visibility into module state |
|
||||
| Start sensing server | `cargo run` or binary | Manual port configuration, no log viewer |
|
||||
| View sensing data | Browser at `localhost:8080` | Separate window, no hardware context |
|
||||
| Mesh topology | Mental model | No visualization of TDM slots, sync, health |
|
||||
| Node discovery | Manual IP tracking | No mDNS/UDP broadcast discovery |
|
||||
|
||||
There is no single tool that provides a unified view of the entire deployment — from ESP32 hardware through the sensing pipeline to pose visualization. Field operators deploying multi-node meshes must context-switch between terminals, browsers, and serial monitors.
|
||||
|
||||
### Why a Desktop App
|
||||
|
||||
A browser-based UI cannot access serial ports (for flashing), raw UDP sockets (for node discovery), or the local filesystem (for firmware binaries). A desktop application is required for hardware management. Tauri v2 is the natural choice because:
|
||||
|
||||
1. **Rust backend** — integrates directly with the existing Rust workspace (`wifi-densepose-rs`). Crates like `wifi-densepose-hardware` (serial port parsing), `wifi-densepose-config`, and `wifi-densepose-sensing-server` can be linked as library dependencies.
|
||||
2. **Small binary** — Tauri bundles the system webview rather than shipping Chromium (~150 MB savings vs Electron).
|
||||
3. **Cross-platform** — Windows, macOS, Linux from the same codebase.
|
||||
4. **Security model** — Tauri's capability-based permissions system restricts frontend access to explicitly allowed Rust commands.
|
||||
|
||||
### Why Not Electron / Flutter / Native
|
||||
|
||||
| Option | Rejected Because |
|
||||
|--------|-----------------|
|
||||
| Electron | 150+ MB bundle, no Rust integration, duplicates webview |
|
||||
| Flutter | No serial port plugins, Dart FFI to Rust is awkward |
|
||||
| Native (GTK/Qt) | Platform-specific UI code, no web component reuse |
|
||||
| Web-only (PWA) | Cannot access serial ports or raw UDP |
|
||||
|
||||
## Decision
|
||||
|
||||
Build a Tauri v2 desktop application as a new crate in the Rust workspace. The frontend uses TypeScript with React and Vite. The Rust backend exposes Tauri commands that bridge the frontend to serial ports, UDP sockets, HTTP management endpoints, and the sensing server process.
|
||||
|
||||
### 1. Workspace Integration
|
||||
|
||||
Add a new crate to the workspace:
|
||||
|
||||
```
|
||||
rust-port/wifi-densepose-rs/
|
||||
Cargo.toml # Add "crates/wifi-densepose-desktop" to members
|
||||
crates/
|
||||
wifi-densepose-desktop/ # NEW — Tauri app crate
|
||||
Cargo.toml
|
||||
tauri.conf.json
|
||||
capabilities/
|
||||
default.json # Tauri v2 capability permissions
|
||||
icons/ # App icons (all platforms)
|
||||
src/
|
||||
main.rs # Tauri entry point
|
||||
lib.rs # Command module re-exports
|
||||
commands/
|
||||
mod.rs
|
||||
discovery.rs # Node discovery commands
|
||||
flash.rs # Firmware flashing commands
|
||||
ota.rs # OTA update commands
|
||||
wasm.rs # WASM module management commands
|
||||
server.rs # Sensing server lifecycle commands
|
||||
provision.rs # NVS provisioning commands
|
||||
serial.rs # Serial port enumeration
|
||||
state.rs # Tauri managed state
|
||||
discovery/
|
||||
mod.rs
|
||||
mdns.rs # mDNS service discovery
|
||||
udp_broadcast.rs # UDP broadcast probe
|
||||
flash/
|
||||
mod.rs
|
||||
espflash.rs # Rust-native ESP32 flashing (via espflash crate)
|
||||
esptool.rs # Fallback: bundled esptool.py wrapper
|
||||
frontend/
|
||||
package.json
|
||||
tsconfig.json
|
||||
vite.config.ts
|
||||
index.html
|
||||
src/
|
||||
main.tsx
|
||||
App.tsx
|
||||
routes.tsx
|
||||
hooks/
|
||||
useNodes.ts # Node discovery and status polling
|
||||
useServer.ts # Sensing server state
|
||||
useWebSocket.ts # WS connection to sensing server
|
||||
stores/
|
||||
nodeStore.ts # Zustand store for discovered nodes
|
||||
serverStore.ts # Sensing server process state
|
||||
settingsStore.ts # User preferences (dark mode, ports)
|
||||
pages/
|
||||
Dashboard.tsx # Hardware management overview
|
||||
NodeDetail.tsx # Single node detail + config
|
||||
FlashFirmware.tsx # Firmware flashing wizard
|
||||
WasmModules.tsx # WASM module manager
|
||||
SensingView.tsx # Live sensing data visualization
|
||||
MeshTopology.tsx # Multi-node mesh topology view
|
||||
Settings.tsx # App settings and preferences
|
||||
components/
|
||||
NodeCard.tsx # Node status card (health, version, signal)
|
||||
NodeList.tsx # Discovered node list
|
||||
FirmwareProgress.tsx # Flash/OTA progress indicator
|
||||
LogViewer.tsx # Scrolling log output
|
||||
SignalChart.tsx # Real-time CSI signal chart
|
||||
PoseOverlay.tsx # Pose skeleton overlay
|
||||
MeshGraph.tsx # D3/force-graph mesh topology
|
||||
SerialPortSelect.tsx # Serial port dropdown
|
||||
ProvisionForm.tsx # NVS provisioning form
|
||||
lib/
|
||||
tauri.ts # Typed Tauri invoke wrappers
|
||||
types.ts # Shared TypeScript types
|
||||
```
|
||||
|
||||
### 2. Rust Backend — Tauri Commands
|
||||
|
||||
#### 2.1 Node Discovery
|
||||
|
||||
```rust
|
||||
// commands/discovery.rs
|
||||
|
||||
/// Discover ESP32 CSI nodes on the local network.
|
||||
/// Strategy 1: mDNS — nodes announce _ruview._tcp service
|
||||
/// Strategy 2: UDP broadcast probe on port 5005 (CSI aggregator port)
|
||||
/// Strategy 3: HTTP health check sweep on port 8032 (OTA server)
|
||||
#[tauri::command]
|
||||
async fn discover_nodes(timeout_ms: u64) -> Result<Vec<DiscoveredNode>, String>;
|
||||
|
||||
/// Get detailed status from a specific node via HTTP.
|
||||
/// Calls GET /ota/status on port 8032.
|
||||
#[tauri::command]
|
||||
async fn get_node_status(ip: String) -> Result<NodeStatus, String>;
|
||||
|
||||
/// Subscribe to node health updates (periodic polling).
|
||||
#[tauri::command]
|
||||
async fn watch_nodes(interval_ms: u64, state: State<'_, AppState>) -> Result<(), String>;
|
||||
```
|
||||
|
||||
The `DiscoveredNode` struct:
|
||||
|
||||
```rust
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct DiscoveredNode {
|
||||
pub ip: String,
|
||||
pub mac: Option<String>,
|
||||
pub hostname: Option<String>,
|
||||
pub node_id: u8,
|
||||
pub firmware_version: Option<String>,
|
||||
pub tdm_slot: Option<u8>,
|
||||
pub tdm_total: Option<u8>,
|
||||
pub edge_tier: Option<u8>,
|
||||
pub uptime_secs: Option<u64>,
|
||||
pub discovery_method: DiscoveryMethod, // Mdns | UdpProbe | HttpSweep
|
||||
pub last_seen: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
```
|
||||
|
||||
#### 2.2 Firmware Flashing
|
||||
|
||||
```rust
|
||||
// commands/flash.rs
|
||||
|
||||
/// List available serial ports with chip detection.
|
||||
#[tauri::command]
|
||||
async fn list_serial_ports() -> Result<Vec<SerialPortInfo>, String>;
|
||||
|
||||
/// Flash firmware binary to an ESP32 via serial port.
|
||||
/// Uses the `espflash` crate for Rust-native flashing (no Python dependency).
|
||||
/// Falls back to bundled esptool.py if espflash fails.
|
||||
/// Emits progress events via Tauri event system.
|
||||
#[tauri::command]
|
||||
async fn flash_firmware(
|
||||
port: String,
|
||||
firmware_path: String,
|
||||
chip: Chip, // Esp32, Esp32s3, Esp32c3
|
||||
baud: Option<u32>,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<FlashResult, String>;
|
||||
|
||||
/// Read firmware info from a connected ESP32 (chip type, flash size, MAC).
|
||||
#[tauri::command]
|
||||
async fn read_chip_info(port: String) -> Result<ChipInfo, String>;
|
||||
```
|
||||
|
||||
Flash progress is emitted as Tauri events:
|
||||
|
||||
```rust
|
||||
#[derive(Serialize, Clone)]
|
||||
pub struct FlashProgress {
|
||||
pub phase: FlashPhase, // Connecting | Erasing | Writing | Verifying
|
||||
pub progress_pct: f32, // 0.0 - 100.0
|
||||
pub bytes_written: u64,
|
||||
pub bytes_total: u64,
|
||||
pub speed_bps: u64,
|
||||
}
|
||||
```
|
||||
|
||||
#### 2.3 OTA Updates
|
||||
|
||||
```rust
|
||||
// commands/ota.rs
|
||||
|
||||
/// Push firmware to a node via HTTP OTA (port 8032).
|
||||
/// Includes PSK authentication per ADR-050.
|
||||
#[tauri::command]
|
||||
async fn ota_update(
|
||||
node_ip: String,
|
||||
firmware_path: String,
|
||||
psk: Option<String>,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<OtaResult, String>;
|
||||
|
||||
/// Get OTA status from a node (current version, partition info).
|
||||
#[tauri::command]
|
||||
async fn ota_status(node_ip: String, psk: Option<String>) -> Result<OtaStatus, String>;
|
||||
|
||||
/// Batch OTA update — push firmware to multiple nodes sequentially.
|
||||
/// Skips nodes already running the target version.
|
||||
#[tauri::command]
|
||||
async fn ota_batch_update(
|
||||
nodes: Vec<String>, // IPs
|
||||
firmware_path: String,
|
||||
psk: Option<String>,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<Vec<OtaResult>, String>;
|
||||
```
|
||||
|
||||
#### 2.4 WASM Module Management
|
||||
|
||||
```rust
|
||||
// commands/wasm.rs
|
||||
|
||||
/// List WASM modules loaded on a node.
|
||||
/// Calls GET /wasm/list on port 8032.
|
||||
#[tauri::command]
|
||||
async fn wasm_list(node_ip: String) -> Result<Vec<WasmModule>, String>;
|
||||
|
||||
/// Upload a WASM module to a node.
|
||||
/// Calls POST /wasm/upload on port 8032 with binary payload.
|
||||
#[tauri::command]
|
||||
async fn wasm_upload(
|
||||
node_ip: String,
|
||||
wasm_path: String,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<WasmUploadResult, String>;
|
||||
|
||||
/// Start/stop a WASM module on a node.
|
||||
#[tauri::command]
|
||||
async fn wasm_control(
|
||||
node_ip: String,
|
||||
module_id: String,
|
||||
action: WasmAction, // Start | Stop | Unload
|
||||
) -> Result<(), String>;
|
||||
```
|
||||
|
||||
#### 2.5 Sensing Server Lifecycle
|
||||
|
||||
```rust
|
||||
// commands/server.rs
|
||||
|
||||
/// Start the sensing server as a managed child process.
|
||||
/// The server binary is either bundled with the Tauri app (sidecar)
|
||||
/// or discovered on PATH.
|
||||
#[tauri::command]
|
||||
async fn start_server(
|
||||
config: ServerConfig,
|
||||
state: State<'_, AppState>,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<(), String>;
|
||||
|
||||
/// Stop the managed sensing server process.
|
||||
#[tauri::command]
|
||||
async fn stop_server(state: State<'_, AppState>) -> Result<(), String>;
|
||||
|
||||
/// Get sensing server status (running/stopped, PID, ports, uptime).
|
||||
#[tauri::command]
|
||||
async fn server_status(state: State<'_, AppState>) -> Result<ServerStatus, String>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct ServerConfig {
|
||||
pub http_port: u16, // Default: 8080
|
||||
pub ws_port: u16, // Default: 8765
|
||||
pub udp_port: u16, // Default: 5005
|
||||
pub static_dir: Option<String>, // Path to UI static files
|
||||
pub model_dir: Option<String>, // Path to ML models
|
||||
pub log_level: String, // trace, debug, info, warn, error
|
||||
}
|
||||
```
|
||||
|
||||
The sensing server is bundled as a Tauri sidecar binary. Tauri v2 supports sidecar binaries via `externalBin` in `tauri.conf.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"bundle": {
|
||||
"externalBin": ["sensing-server"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### 2.6 NVS Provisioning
|
||||
|
||||
```rust
|
||||
// commands/provision.rs
|
||||
|
||||
/// Provision NVS configuration to an ESP32 via serial port.
|
||||
/// Replaces the Python provision.py script with a Rust-native implementation.
|
||||
/// Generates NVS partition binary and flashes it to the NVS partition offset.
|
||||
#[tauri::command]
|
||||
async fn provision_node(
|
||||
port: String,
|
||||
config: NvsConfig,
|
||||
app_handle: AppHandle,
|
||||
) -> Result<ProvisionResult, String>;
|
||||
|
||||
/// Read current NVS configuration from a connected ESP32.
|
||||
/// Reads the NVS partition and parses key-value pairs.
|
||||
#[tauri::command]
|
||||
async fn read_nvs(port: String) -> Result<NvsConfig, String>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct NvsConfig {
|
||||
pub wifi_ssid: Option<String>,
|
||||
pub wifi_password: Option<String>,
|
||||
pub target_ip: Option<String>,
|
||||
pub target_port: Option<u16>,
|
||||
pub node_id: Option<u8>,
|
||||
pub tdm_slot: Option<u8>,
|
||||
pub tdm_total: Option<u8>,
|
||||
pub edge_tier: Option<u8>,
|
||||
pub presence_thresh: Option<u16>,
|
||||
pub fall_thresh: Option<u16>,
|
||||
pub vital_window: Option<u16>,
|
||||
pub vital_interval_ms: Option<u16>,
|
||||
pub top_k_count: Option<u8>,
|
||||
pub hop_count: Option<u8>,
|
||||
pub channel_list: Option<Vec<u8>>,
|
||||
pub dwell_ms: Option<u32>,
|
||||
pub power_duty: Option<u8>,
|
||||
pub wasm_max_modules: Option<u8>,
|
||||
pub wasm_verify: Option<bool>,
|
||||
pub wasm_pubkey: Option<Vec<u8>>,
|
||||
pub ota_psk: Option<String>,
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Frontend Architecture
|
||||
|
||||
#### 3.1 Tech Stack
|
||||
|
||||
| Layer | Choice | Rationale |
|
||||
|-------|--------|-----------|
|
||||
| Framework | React 19 | Component model, ecosystem, team familiarity |
|
||||
| Build | Vite 6 | Fast HMR, Tauri plugin support |
|
||||
| State | Zustand | Lightweight, no boilerplate, works with Tauri events |
|
||||
| Routing | React Router v7 | File-based routes, type-safe |
|
||||
| UI Components | shadcn/ui + Tailwind CSS | Accessible, customizable, no runtime CSS-in-JS |
|
||||
| Charts | Recharts or visx | Real-time signal visualization |
|
||||
| Topology Graph | D3 force-directed | Mesh network visualization |
|
||||
| Serial UI | Custom | Tauri command integration |
|
||||
| Icons | Lucide React | Consistent, tree-shakeable |
|
||||
|
||||
#### 3.2 Page Layout
|
||||
|
||||
```
|
||||
+------------------------------------------+
|
||||
| RuView [Settings] [?] |
|
||||
+-------+----------------------------------+
|
||||
| | |
|
||||
| Nav | Dashboard / Active Page |
|
||||
| | |
|
||||
| [D] | +--------+ +--------+ +------+ |
|
||||
| [F] | | Node 1 | | Node 2 | | +Add | |
|
||||
| [W] | +--------+ +--------+ +------+ |
|
||||
| [S] | |
|
||||
| [M] | Server Status: Running |
|
||||
| [T] | +--------------------------+ |
|
||||
| | | Live Signal / Pose View | |
|
||||
| | +--------------------------+ |
|
||||
+-------+----------------------------------+
|
||||
| Status Bar: 3 nodes | Server: :8080 |
|
||||
+------------------------------------------+
|
||||
|
||||
Nav items:
|
||||
[D] Dashboard — overview of all nodes and server
|
||||
[F] Flash — firmware flashing wizard
|
||||
[W] WASM — edge module management
|
||||
[S] Sensing — live sensing data view
|
||||
[M] Mesh — topology visualization
|
||||
[T] Settings — ports, paths, preferences
|
||||
```
|
||||
|
||||
#### 3.3 Dashboard Page
|
||||
|
||||
The dashboard is the primary landing page showing:
|
||||
|
||||
1. **Node Grid** — cards for each discovered ESP32 node showing:
|
||||
- IP address and hostname
|
||||
- Firmware version (with update indicator if newer available)
|
||||
- Node ID and TDM slot assignment
|
||||
- Edge processing tier (raw / stats / vitals)
|
||||
- Signal quality indicator (last CSI frame age)
|
||||
- Health status (online/offline/degraded)
|
||||
- Quick actions: OTA update, configure, view logs
|
||||
|
||||
2. **Sensing Server Panel** — start/stop button, port configuration, log tail
|
||||
|
||||
3. **Discovery Controls** — scan button, auto-discovery toggle, network range filter
|
||||
|
||||
#### 3.4 Flash Firmware Page
|
||||
|
||||
A wizard-style flow:
|
||||
|
||||
1. **Select Port** — dropdown of detected serial ports with chip info
|
||||
2. **Select Firmware** — file picker for `.bin` files, or select from bundled builds
|
||||
3. **Configure** — chip type, baud rate, flash mode
|
||||
4. **Flash** — progress bar with phase indicators (connecting, erasing, writing, verifying)
|
||||
5. **Provision** — optional NVS provisioning form (WiFi, target IP, TDM, edge tier)
|
||||
6. **Verify** — serial monitor showing boot log, success/fail indicator
|
||||
|
||||
#### 3.5 WASM Module Manager Page
|
||||
|
||||
| Column | Content |
|
||||
|--------|---------|
|
||||
| Module ID | Auto-assigned by node |
|
||||
| Name | Filename of uploaded `.wasm` |
|
||||
| Size | Module size in KB |
|
||||
| Status | Running / Stopped / Error |
|
||||
| Node | Which ESP32 node it runs on |
|
||||
| Actions | Start / Stop / Unload / View Logs |
|
||||
|
||||
Upload panel: drag-and-drop `.wasm` file, select target node(s), upload button.
|
||||
|
||||
#### 3.6 Sensing View Page
|
||||
|
||||
Embeds the existing web UI (`ui/`) via an iframe pointing at the sensing server's static file route, or builds native React components that connect to the same WebSocket API. The native approach is preferred because it allows:
|
||||
|
||||
- Tighter integration with the node status sidebar
|
||||
- Shared state between hardware management and visualization
|
||||
- Offline access to recorded data
|
||||
|
||||
Key visualization components:
|
||||
- **CSI Heatmap** — subcarrier amplitude over time
|
||||
- **Signal Field** — 2D signal strength visualization
|
||||
- **Pose Skeleton** — detected body keypoints and connections
|
||||
- **Vital Signs** — real-time breathing rate and heart rate charts
|
||||
- **Activity Classification** — current activity label with confidence
|
||||
|
||||
#### 3.7 Mesh Topology Page
|
||||
|
||||
A force-directed graph showing:
|
||||
- Nodes as circles (color = health status, size = edge tier)
|
||||
- Edges between nodes that can see each other
|
||||
- TDM slot labels on each node
|
||||
- Sync status indicators (in-sync / drifting / lost)
|
||||
- Click a node to navigate to its detail page
|
||||
|
||||
### 4. Platform-Specific Considerations
|
||||
|
||||
#### 4.1 macOS
|
||||
|
||||
- **Serial driver signing**: CP210x and CH340 drivers require user approval in System Preferences > Security
|
||||
- **App signing**: Tauri apps must be signed and notarized for distribution outside the App Store
|
||||
- **USB permissions**: No special permissions needed beyond driver installation
|
||||
- **CoreWLAN**: The sensing server can use CoreWLAN for WiFi scanning (ADR-025); the desktop app inherits this capability
|
||||
|
||||
#### 4.2 Windows
|
||||
|
||||
- **COM port access**: Windows assigns COM port numbers; the app lists them via the Windows Registry or `SetupDi` API
|
||||
- **Driver installation**: USB-to-serial drivers (CP210x, CH340, FTDI) must be installed; the app can detect missing drivers and link to downloads
|
||||
- **Firewall**: The sensing server's UDP listener may trigger Windows Firewall prompts; the app should pre-configure rules or guide the user
|
||||
- **Code signing**: EV certificate required for SmartScreen trust; unsigned apps trigger warnings
|
||||
|
||||
#### 4.3 Linux
|
||||
|
||||
- **udev rules**: ESP32 serial ports (`/dev/ttyUSB*`, `/dev/ttyACM*`) require udev rules for non-root access. The app bundles a `99-ruview-esp32.rules` file and offers to install it:
|
||||
```
|
||||
SUBSYSTEM=="tty", ATTRS{idVendor}=="10c4", MODE="0666" # CP210x
|
||||
SUBSYSTEM=="tty", ATTRS{idVendor}=="1a86", MODE="0666" # CH340
|
||||
```
|
||||
- **AppImage/deb/rpm**: Tauri supports all three packaging formats
|
||||
- **Wayland vs X11**: Tauri uses webkit2gtk which works on both
|
||||
|
||||
### 5. Cargo.toml for the Desktop Crate
|
||||
|
||||
```toml
|
||||
[package]
|
||||
name = "wifi-densepose-desktop"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
description = "Tauri desktop frontend for RuView WiFi DensePose"
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "wifi_densepose_desktop"
|
||||
crate-type = ["staticlib", "cdylib", "rlib"]
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2", features = [] }
|
||||
|
||||
[dependencies]
|
||||
tauri = { version = "2", features = [] }
|
||||
tauri-plugin-shell = "2" # Sidecar process management
|
||||
tauri-plugin-dialog = "2" # File picker dialogs
|
||||
tauri-plugin-fs = "2" # Filesystem access
|
||||
tauri-plugin-process = "2" # Process management
|
||||
tauri-plugin-notification = "2" # Desktop notifications
|
||||
|
||||
# Workspace crates
|
||||
wifi-densepose-hardware = { workspace = true }
|
||||
wifi-densepose-config = { workspace = true }
|
||||
wifi-densepose-core = { workspace = true }
|
||||
|
||||
# Serial port access
|
||||
serialport = { workspace = true }
|
||||
|
||||
# ESP32 flashing (Rust-native, replaces esptool.py)
|
||||
espflash = "3"
|
||||
|
||||
# Network discovery
|
||||
mdns-sd = "0.11" # mDNS/DNS-SD service discovery
|
||||
|
||||
# HTTP client for OTA and WASM management
|
||||
reqwest = { version = "0.12", features = ["json", "multipart", "stream"] }
|
||||
|
||||
# Async runtime
|
||||
tokio = { workspace = true }
|
||||
|
||||
# Serialization
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
# Logging
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
# Time
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
```
|
||||
|
||||
### 6. Tauri Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/tauri-apps/tauri/dev/crates/tauri-config-schema/schema.json",
|
||||
"productName": "RuView",
|
||||
"version": "0.3.0",
|
||||
"identifier": "net.ruv.ruview",
|
||||
"build": {
|
||||
"frontendDist": "../frontend/dist",
|
||||
"devUrl": "http://localhost:5173",
|
||||
"beforeDevCommand": "cd frontend && npm run dev",
|
||||
"beforeBuildCommand": "cd frontend && npm run build"
|
||||
},
|
||||
"app": {
|
||||
"windows": [
|
||||
{
|
||||
"title": "RuView - WiFi DensePose",
|
||||
"width": 1280,
|
||||
"height": 800,
|
||||
"minWidth": 900,
|
||||
"minHeight": 600
|
||||
}
|
||||
]
|
||||
},
|
||||
"bundle": {
|
||||
"active": true,
|
||||
"targets": "all",
|
||||
"icon": [
|
||||
"icons/32x32.png",
|
||||
"icons/128x128.png",
|
||||
"icons/128x128@2x.png",
|
||||
"icons/icon.icns",
|
||||
"icons/icon.ico"
|
||||
],
|
||||
"externalBin": ["sensing-server"],
|
||||
"linux": {
|
||||
"deb": { "depends": ["libwebkit2gtk-4.1-0"] },
|
||||
"appimage": { "bundleMediaFramework": true }
|
||||
},
|
||||
"windows": {
|
||||
"wix": { "language": "en-US" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 7. Tauri v2 Capabilities (Permissions)
|
||||
|
||||
```json
|
||||
{
|
||||
"identifier": "default",
|
||||
"description": "RuView default capability set",
|
||||
"windows": ["main"],
|
||||
"permissions": [
|
||||
"core:default",
|
||||
"shell:allow-execute",
|
||||
"shell:allow-open",
|
||||
"dialog:allow-open",
|
||||
"dialog:allow-save",
|
||||
"fs:allow-read",
|
||||
"fs:allow-write",
|
||||
"process:allow-exit",
|
||||
"notification:default"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 8. Development Workflow
|
||||
|
||||
```bash
|
||||
# Prerequisites
|
||||
cargo install tauri-cli@^2
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop/frontend
|
||||
npm install
|
||||
|
||||
# Development (hot-reload frontend + Rust rebuild)
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-desktop
|
||||
cargo tauri dev
|
||||
|
||||
# Production build
|
||||
cargo tauri build
|
||||
|
||||
# Build sensing-server sidecar (must be done before tauri build)
|
||||
cargo build --release -p wifi-densepose-sensing-server
|
||||
# Copy to sidecar location:
|
||||
# target/release/sensing-server -> crates/wifi-densepose-desktop/binaries/sensing-server-{arch}
|
||||
```
|
||||
|
||||
### 9. Persistent Node Registry
|
||||
|
||||
Discovery alone is transient — nodes appear when they broadcast, disappear when they don't. A persistent local registry transforms discovery into **reconciliation**.
|
||||
|
||||
```
|
||||
~/.ruview/nodes.db (SQLite via rusqlite)
|
||||
```
|
||||
|
||||
**Schema:**
|
||||
|
||||
```sql
|
||||
CREATE TABLE nodes (
|
||||
mac TEXT PRIMARY KEY, -- e.g. "AA:BB:CC:DD:EE:FF"
|
||||
last_ip TEXT, -- last known IP
|
||||
last_seen INTEGER NOT NULL, -- Unix timestamp
|
||||
firmware TEXT, -- e.g. "0.3.1"
|
||||
chip TEXT DEFAULT 'esp32s3', -- esp32, esp32s3, esp32c3
|
||||
mesh_role TEXT DEFAULT 'node', -- 'coordinator' | 'node' | 'aggregator'
|
||||
tdm_slot INTEGER, -- assigned TDM slot index
|
||||
capabilities TEXT, -- JSON: {"wasm": true, "ota": true, "csi": true}
|
||||
friendly_name TEXT, -- user-assigned label
|
||||
notes TEXT -- free-form notes
|
||||
);
|
||||
```
|
||||
|
||||
**Behavior:**
|
||||
|
||||
- On discovery broadcast, upsert into registry (update `last_ip`, `last_seen`, `firmware`)
|
||||
- Dashboard shows **all registered nodes**, dimming those not seen recently
|
||||
- User can manually add nodes by MAC/IP (for networks without mDNS)
|
||||
- Export/import registry as JSON for fleet management across machines
|
||||
- Node health history (uptime, last OTA, error count) tracked over time
|
||||
|
||||
This means the desktop app **remembers the mesh** across restarts, which is critical for field deployments where nodes may be offline temporarily.
|
||||
|
||||
### 10. OTA Safety Gate — Rolling Updates
|
||||
|
||||
Mesh deployments cannot tolerate all nodes rebooting simultaneously. The OTA subsystem includes a **rolling update mode** that preserves sensing continuity:
|
||||
|
||||
```rust
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct BatchOtaConfig {
|
||||
/// Update strategy
|
||||
pub strategy: OtaStrategy,
|
||||
/// Max nodes updating concurrently
|
||||
pub max_concurrent: usize,
|
||||
/// Delay between batches (seconds)
|
||||
pub batch_delay_secs: u64,
|
||||
/// Abort if any node fails
|
||||
pub fail_fast: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub enum OtaStrategy {
|
||||
/// Update one node at a time, wait for it to rejoin mesh
|
||||
Sequential,
|
||||
/// Update non-adjacent TDM slots to maintain coverage
|
||||
TdmSafe,
|
||||
/// Update all nodes simultaneously (development only)
|
||||
Parallel,
|
||||
}
|
||||
```
|
||||
|
||||
**`TdmSafe` strategy:**
|
||||
|
||||
1. Sort nodes by TDM slot index
|
||||
2. Update even-slot nodes first (slots 0, 2, 4...)
|
||||
3. Wait for each to reboot and rejoin mesh (verified via beacon)
|
||||
4. Then update odd-slot nodes (slots 1, 3, 5...)
|
||||
5. At no point are adjacent nodes offline simultaneously
|
||||
|
||||
**UI flow:**
|
||||
|
||||
- User selects target firmware + target nodes
|
||||
- App shows pre-update diff (current vs new version per node)
|
||||
- Progress bar per node with states: `queued → uploading → rebooting → verifying → done`
|
||||
- Abort button halts remaining updates without rolling back completed ones
|
||||
- Post-update health check confirms all nodes are sensing
|
||||
|
||||
### 11. Plugin Architecture (Future)
|
||||
|
||||
This desktop tool is quietly becoming the **control plane for RuView**. Once it manages discovery, firmware, OTA, WASM, sensing, and mesh topology, plugin extensibility becomes inevitable:
|
||||
|
||||
- **Firmware management** today → **swarm orchestration** tomorrow
|
||||
- **WASM upload** today → **edge module marketplace** tomorrow
|
||||
- **Sensing view** today → **activity classification dashboard** tomorrow
|
||||
|
||||
The Tauri command surface should be designed with this trajectory in mind:
|
||||
|
||||
- Commands are grouped by bounded context (already done)
|
||||
- Each context can be extended by loading additional Tauri plugins
|
||||
- The node registry becomes the source of truth for all plugins
|
||||
- Event bus (Tauri's `emit`/`listen`) provides cross-plugin communication
|
||||
|
||||
This does NOT mean building a plugin system in Phase 1. It means keeping the architecture open to it: no hardcoded views, state flows through the registry, commands are typed and versioned.
|
||||
|
||||
### 12. Security Considerations
|
||||
|
||||
1. **PSK Storage**: OTA PSK tokens are stored in the OS keychain via `tauri-plugin-stronghold` or the platform's native credential store, never in plaintext config files.
|
||||
|
||||
2. **Serial Port Access**: Tauri's capability system restricts which commands the frontend can invoke. Serial port access is only available through the typed `flash_firmware` and `provision_node` commands, not raw serial I/O.
|
||||
|
||||
3. **Network Requests**: OTA and WASM management commands only communicate with nodes on the local network. The app does not make external network requests except for update checks (opt-in).
|
||||
|
||||
4. **Firmware Validation**: Before flashing, the app validates the firmware binary header (ESP32 image magic bytes, partition table offset) to prevent bricking.
|
||||
|
||||
5. **WASM Signature Verification**: The desktop app can sign WASM modules before upload using a locally stored Ed25519 key pair, complementing the node-side verification (ADR-040).
|
||||
|
||||
### 13. Implementation Phases
|
||||
|
||||
| Phase | Scope | Effort | Priority |
|
||||
|-------|-------|--------|----------|
|
||||
| **Phase 1: Skeleton** | Tauri project scaffolding, workspace integration, basic window with React | 1 week | P0 |
|
||||
| **Phase 2: Discovery** | Serial port listing, UDP/mDNS node discovery, dashboard with node cards | 1 week | P0 |
|
||||
| **Phase 3: Flash** | espflash integration, firmware flashing wizard with progress events | 1 week | P0 |
|
||||
| **Phase 4: Server** | Sidecar sensing server start/stop, log viewer, status panel | 1 week | P1 |
|
||||
| **Phase 5: OTA** | HTTP OTA with PSK auth, batch update, version comparison | 1 week | P1 |
|
||||
| **Phase 6: Provisioning** | NVS read/write via serial, provisioning form, mesh config file | 1 week | P1 |
|
||||
| **Phase 7: WASM** | Module upload/list/start/stop, drag-and-drop, per-module logs | 1 week | P2 |
|
||||
| **Phase 8: Sensing** | WebSocket integration, live signal charts, pose overlay | 2 weeks | P2 |
|
||||
| **Phase 9: Mesh View** | Force-directed topology graph, TDM slot visualization, sync status | 1 week | P2 |
|
||||
| **Phase 10: Polish** | App signing, auto-update, udev rules installer, onboarding wizard | 1 week | P3 |
|
||||
|
||||
Total estimated effort: ~11 weeks for a single developer.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Single pane of glass** — all hardware management, sensing, and visualization in one app
|
||||
- **No Python dependency** — Rust-native `espflash` replaces `esptool.py` for firmware flashing
|
||||
- **Replaces 6+ CLI tools** — flash, provision, OTA, WASM management, server control, visualization
|
||||
- **Accessible to non-developers** — GUI replaces CLI flags and curl commands
|
||||
- **Cross-platform** — one codebase for Windows, macOS, Linux
|
||||
- **Workspace integration** — shares types, config, and hardware crates with sensing server
|
||||
- **Small binary** — ~15-20 MB vs ~150 MB for Electron equivalent
|
||||
|
||||
### Negative
|
||||
|
||||
- **New frontend dependency** — introduces Node.js/npm build step into the Rust workspace
|
||||
- **Tauri version churn** — Tauri v2 is recent; API stability is not yet proven at scale
|
||||
- **webkit2gtk on Linux** — depends on system webview version; old distros may have stale webkit
|
||||
- **espflash limitations** — the `espflash` crate may not support all chip variants or flash modes that `esptool.py` handles; fallback to bundled Python is needed
|
||||
- **Maintenance surface** — adds ~5,000 lines of TypeScript and ~2,000 lines of Rust
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| espflash cannot flash all ESP32 variants | Medium | High | Bundle esptool.py as fallback sidecar |
|
||||
| Tauri v2 breaking changes | Low | Medium | Pin to specific Tauri version; update in dedicated PRs |
|
||||
| Serial port access fails on macOS Sequoia+ | Medium | Medium | Test on latest macOS; document driver requirements |
|
||||
| webkit2gtk version mismatch on Linux | Medium | Low | Set minimum version in deb/rpm dependencies |
|
||||
| Sidecar sensing server fails to start | Low | Medium | Detect failure and show manual start instructions |
|
||||
|
||||
## References
|
||||
|
||||
- Tauri v2 documentation: https://v2.tauri.app/
|
||||
- espflash crate: https://crates.io/crates/espflash
|
||||
- mdns-sd crate: https://crates.io/crates/mdns-sd
|
||||
- ADR-012: ESP32 CSI Sensor Mesh
|
||||
- ADR-039: ESP32 Edge Intelligence
|
||||
- ADR-040: WASM Programmable Sensing
|
||||
- ADR-044: Provisioning Tool Enhancements
|
||||
- ADR-050: Quality Engineering — Security Hardening
|
||||
- ADR-051: Sensing Server Decomposition
|
||||
- `firmware/esp32-csi-node/` — ESP32 firmware source
|
||||
- `firmware/esp32-csi-node/provision.py` — Current provisioning script
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-sensing-server/` — Sensing server
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-hardware/` — Hardware crate
|
||||
- `ui/` — Existing web UI
|
||||
@@ -0,0 +1,274 @@
|
||||
# ADR-053: UI Design System — Dark Professional + Unity-Inspired Interface
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-03-06 |
|
||||
| Deciders | ruv |
|
||||
| Depends on | ADR-052 (Tauri Desktop Frontend) |
|
||||
|
||||
## Context
|
||||
|
||||
RuView Desktop (ADR-052) needs a UI design system that communicates precision and control — befitting a hardware management control plane for embedded sensing infrastructure. The interface must handle dense data (CSI heatmaps, node registries, log streams, mesh topologies) without feeling overwhelming, while remaining usable by both engineers and field operators.
|
||||
|
||||
Two design inspirations:
|
||||
|
||||
1. **Data-first professional tools** — Dense information displays where data speaks for itself. Clean typography, structured layouts, and deliberate use of color for status. The interface shows what matters and hides what doesn't. Think: network monitoring dashboards, embedded systems IDEs, infrastructure control panels.
|
||||
|
||||
2. **Unity Editor** — Dockable panel system, inspector/hierarchy/scene separation, property grids, dark professional theme, and dense-but-organized data display. Unity's UI is purpose-built for managing complex real-time systems — exactly what RuView needs.
|
||||
|
||||
The combination yields a professional control panel for WiFi sensing infrastructure. Data is organized into scannable panels with clear hierarchy. Status is communicated through consistent color coding. The layout adapts from high-level overview down to individual node details through progressive disclosure.
|
||||
|
||||
## Decision
|
||||
|
||||
### Design Principles
|
||||
|
||||
1. **Data is the interface** — The system reveals patterns through visualization, not through explanation. Every pixel earns its place.
|
||||
2. **Precision typography** — Typography is clean and authoritative. Technical values are displayed without ambiguity. Labels are concise.
|
||||
3. **Panel-based layout** — Dockable regions inspired by Unity's panel system. The operator can see the entire mesh at a glance, then drill into any node.
|
||||
4. **Status through color** — Deliberate color coding: green (online), amber (degraded), red (offline/failed), blue (scanning/new). No gratuitous color.
|
||||
5. **Progressive disclosure** — Dashboard shows the overview. Clicking a node reveals its details. Summary first, detail on interaction.
|
||||
6. **Dual typography** — Monospace for all technical values (MAC addresses, firmware versions, CSI amplitudes). Sans-serif for labels and descriptions. The contrast signals "data vs. context."
|
||||
7. **Powered by rUv** — Subtle branding: footer tagline, about dialog, splash screen.
|
||||
|
||||
### Color System
|
||||
|
||||
```css
|
||||
:root {
|
||||
/* Background layers */
|
||||
--bg-base: #0d1117; /* App background */
|
||||
--bg-surface: #161b22; /* Panel backgrounds */
|
||||
--bg-elevated: #1c2333; /* Cards, modals, dropdowns */
|
||||
--bg-hover: #242d3d; /* Hover state */
|
||||
--bg-active: #2d3748; /* Active/selected state */
|
||||
|
||||
/* Text hierarchy */
|
||||
--text-primary: #e6edf3; /* Headings, primary content */
|
||||
--text-secondary: #8b949e; /* Labels, descriptions */
|
||||
--text-muted: #484f58; /* Disabled, hints, placeholders */
|
||||
|
||||
/* Status indicators */
|
||||
--status-online: #3fb950; /* Node online, healthy */
|
||||
--status-warning: #d29922; /* Degraded, needs attention */
|
||||
--status-error: #f85149; /* Offline, failed, critical */
|
||||
--status-info: #58a6ff; /* Scanning, discovering, info */
|
||||
|
||||
/* Accent */
|
||||
--accent: #7c3aed; /* rUv purple — primary actions */
|
||||
--accent-hover: #6d28d9;
|
||||
|
||||
/* Borders */
|
||||
--border: #30363d;
|
||||
--border-active: #58a6ff;
|
||||
|
||||
/* Data display */
|
||||
--font-mono: 'JetBrains Mono', 'Fira Code', 'Consolas', monospace;
|
||||
--font-sans: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
|
||||
}
|
||||
```
|
||||
|
||||
### Typography Scale
|
||||
|
||||
```css
|
||||
/* Typographic hierarchy */
|
||||
.heading-xl { font: 600 28px/1.2 var(--font-sans); } /* Page titles */
|
||||
.heading-lg { font: 600 20px/1.3 var(--font-sans); } /* Section titles */
|
||||
.heading-md { font: 600 16px/1.4 var(--font-sans); } /* Card titles */
|
||||
.heading-sm { font: 600 13px/1.4 var(--font-sans); } /* Panel labels */
|
||||
.body { font: 400 14px/1.6 var(--font-sans); } /* Body text */
|
||||
.body-sm { font: 400 12px/1.5 var(--font-sans); } /* Captions */
|
||||
.data { font: 400 13px/1.4 var(--font-mono); } /* Technical values */
|
||||
.data-lg { font: 500 18px/1.2 var(--font-mono); } /* Key metrics */
|
||||
```
|
||||
|
||||
### Layout System
|
||||
|
||||
Three-region layout: navigation sidebar, node list, and detail inspector. Unity's docking system provides the mechanical framework.
|
||||
|
||||
```
|
||||
+--[ Sidebar ]--+--[ Main ]-------------------------------------+
|
||||
| | |
|
||||
| [Nav Items] | +--[ Command Bar ]---------------------------+ |
|
||||
| | | Breadcrumb | Actions | Search | |
|
||||
| Dashboard | +-------+-----------------------------------+ |
|
||||
| Nodes | | | | |
|
||||
| Flash | | Node | Detail Inspector | |
|
||||
| OTA | | List | (selected node properties) | |
|
||||
| Edge Modules | | | | |
|
||||
| Sensing | | | [Property Grid] | |
|
||||
| Mesh View | | | [Status Indicators] | |
|
||||
| Settings | | | [Action Buttons] | |
|
||||
| | | | | |
|
||||
+-[ Status Bar ]+--+-------+-----------------------------------+ |
|
||||
| rUv | 3 nodes online | Server: running | Port: 8080 |
|
||||
+---------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Panel behaviors:**
|
||||
- Sidebar collapses to icon-only on narrow windows
|
||||
- Node List / Inspector split is resizable via drag handle
|
||||
- Inspector scrolls independently — drill into any node without losing the list
|
||||
- Status Bar shows global system state at a glance (node count, server status, port)
|
||||
|
||||
### Component Library
|
||||
|
||||
#### 1. NodeCard
|
||||
|
||||
```
|
||||
+-- NodeCard -----------------------------------------------+
|
||||
| [●] ESP32-S3 Node #2 firmware: 0.3.1 |
|
||||
| MAC: AA:BB:CC:DD:EE:FF TDM Slot: 2/4 |
|
||||
| IP: 192.168.1.42 Edge Tier: 1 |
|
||||
| Last seen: 3s ago [Flash] [OTA] [···] |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
Status dot uses `--status-online/warning/error`. Card background shifts on hover.
|
||||
|
||||
#### 2. FlashProgress
|
||||
|
||||
```
|
||||
+-- Flash Progress -----------------------------------------+
|
||||
| Flashing firmware to COM3 (ESP32-S3) |
|
||||
| |
|
||||
| Phase: Writing |
|
||||
| [████████████████████░░░░░░░░░░] 67.3% |
|
||||
| 412 KB / 612 KB • 38.2 KB/s • ~5s remaining |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
Progress bar uses `--accent` fill with subtle pulse animation during active writes.
|
||||
|
||||
#### 3. Mesh Topology View (Three.js)
|
||||
|
||||
Interactive 3D visualization of the sensing network. Each node is a sphere. Edges are lines representing signal paths. The coordinator node is visually distinct (larger, outlined ring). Built with **Three.js**, consistent with the existing visualization stack in `ui/observatory/js/` and `ui/components/`.
|
||||
|
||||
```
|
||||
+-- Mesh Topology ------------------------------------------+
|
||||
| |
|
||||
| [Node 0]----[Node 1] |
|
||||
| | \ / | |
|
||||
| | [Coordinator] | Coordinator = TDM master |
|
||||
| | / \ | |
|
||||
| [Node 2]----[Node 3] |
|
||||
| |
|
||||
| Drift: ±0.3ms | Cycle: 50ms | 4/4 nodes online |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Three.js implementation details:**
|
||||
- Force-directed layout computed on CPU, rendered as `THREE.Group` with `THREE.Mesh` (spheres) and `THREE.Line` (edges)
|
||||
- Node spheres use `THREE.MeshPhongMaterial` with emissive color matching `--status-online/warning/error`
|
||||
- Edge lines use `THREE.LineBasicMaterial` with opacity mapped to signal strength
|
||||
- Coordinator node rendered with `THREE.RingGeometry` outline
|
||||
- Camera: `OrbitControls` for pan/zoom/rotate, reset button returns to default view
|
||||
- Follows existing patterns: `BufferGeometry` + `BufferAttribute` for dynamic updates (see `ui/observatory/js/subcarrier-manifold.js`)
|
||||
- Raycasting for node click → opens detail in Inspector panel
|
||||
- Real-time updates as nodes join, leave, or change status — geometry attributes updated per frame
|
||||
|
||||
#### 4. PropertyGrid (Unity Inspector-style)
|
||||
|
||||
```
|
||||
+-- Node Inspector -----------------------------------------+
|
||||
| General [▼] |
|
||||
| MAC Address AA:BB:CC:DD:EE:FF |
|
||||
| IP Address 192.168.1.42 |
|
||||
| Firmware 0.3.1 |
|
||||
| Chip ESP32-S3 |
|
||||
| TDM Configuration [▼] |
|
||||
| Slot Index 2 |
|
||||
| Total Nodes 4 |
|
||||
| Cycle Period 50 ms |
|
||||
| Sync Drift +0.12 ms |
|
||||
| WASM Modules [▼] |
|
||||
| [0] activity_detect running 12.4 KB 83 us/f |
|
||||
| [1] vital_monitor stopped 8.1 KB — us/f |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
Collapsible sections with alternating row backgrounds for scanability.
|
||||
|
||||
#### 5. StatusBadge
|
||||
|
||||
```
|
||||
[● Online] [◐ Degraded] [○ Offline] [↻ Updating]
|
||||
```
|
||||
|
||||
Small inline badges with status dot, label, and optional tooltip.
|
||||
|
||||
#### 6. LogViewer
|
||||
|
||||
```
|
||||
+-- Server Log (auto-scroll) -----------[ Clear ] [ ⏸ ]---+
|
||||
| 19:42:01.234 INFO sensing-server HTTP on 127.0.0.1:8080|
|
||||
| 19:42:01.235 INFO sensing-server WS on 127.0.0.1:8765 |
|
||||
| 19:42:01.890 INFO udp_receiver CSI frame from .42 |
|
||||
| 19:42:02.003 WARN vital_signs Low signal quality |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
Monospace, color-coded by log level (INFO=text, WARN=amber, ERROR=red). Virtual scrolling for performance.
|
||||
|
||||
### Spacing and Grid
|
||||
|
||||
```css
|
||||
/* 4px base grid */
|
||||
--space-1: 4px; /* Tight spacing (within components) */
|
||||
--space-2: 8px; /* Component internal padding */
|
||||
--space-3: 12px; /* Between related elements */
|
||||
--space-4: 16px; /* Card padding, section gaps */
|
||||
--space-5: 24px; /* Between sections */
|
||||
--space-6: 32px; /* Page-level spacing */
|
||||
--space-8: 48px; /* Major section breaks */
|
||||
|
||||
/* Panel dimensions */
|
||||
--sidebar-width: 220px;
|
||||
--sidebar-collapsed: 52px;
|
||||
--statusbar-height: 28px;
|
||||
--toolbar-height: 44px;
|
||||
```
|
||||
|
||||
### Animations
|
||||
|
||||
Minimal and purposeful:
|
||||
- Panel collapse/expand: 200ms ease-out
|
||||
- Node card health transition: 300ms (color fade, not flash)
|
||||
- Progress bar fill: smooth 60fps CSS transition
|
||||
- Mesh graph: Three.js render loop at 60fps, force simulation on requestAnimationFrame
|
||||
- No loading spinners — use skeleton placeholders instead
|
||||
|
||||
### Branding
|
||||
|
||||
- **Splash screen**: rUv logo + "RuView Desktop" + version, 1.5s duration
|
||||
- **Status bar**: "Powered by rUv" in `--text-muted`, left-aligned
|
||||
- **About dialog**: rUv logo, version, license, links to GitHub and docs
|
||||
- **App icon**: Stylized WiFi signal + human silhouette in rUv purple (#7c3aed)
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Professional, data-dense UI suitable for hardware management
|
||||
- Consistent design language across all 7 pages
|
||||
- Dual typography (mono + sans-serif) ensures readability at all information densities
|
||||
- Unity-inspired panels feel natural to engineers familiar with IDE/editor tools
|
||||
- Dark theme reduces eye strain for extended monitoring sessions
|
||||
|
||||
### Negative
|
||||
|
||||
- Custom design system means no off-the-shelf component library (shadcn/ui partially usable)
|
||||
- Dockable panels add complexity to the layout system
|
||||
- Dark-only theme may not suit all users (could add light mode later)
|
||||
|
||||
### Neutral
|
||||
|
||||
- The design system is CSS-only with React components — no heavy UI framework dependency
|
||||
- Component library can be extracted as a separate package if other rUv projects need it
|
||||
|
||||
## References
|
||||
|
||||
- ADR-052: Tauri Desktop Frontend
|
||||
- Unity Editor UI Guidelines: https://docs.unity3d.com/Manual/UIE-USS.html
|
||||
- Three.js (existing project dependency): `ui/observatory/js/`, `ui/components/`
|
||||
- Inter font: https://rsms.me/inter/
|
||||
- JetBrains Mono: https://www.jetbrains.com/lp/mono/
|
||||
@@ -0,0 +1,699 @@
|
||||
# ADR-054: RuView Desktop Full Implementation
|
||||
|
||||
## Status
|
||||
**Accepted** — Implementation in progress
|
||||
|
||||
## Context
|
||||
|
||||
RuView Desktop v0.3.0 shipped with a complete React/TypeScript frontend but stub-only Rust backend commands. Users report:
|
||||
- Settings cannot be saved (#206) ✅ Fixed in PR #209
|
||||
- Flash firmware does nothing
|
||||
- OTA updates are non-functional
|
||||
- Node discovery returns hardcoded data
|
||||
- Server start/stop is cosmetic only
|
||||
|
||||
This ADR defines the complete implementation plan to make all desktop features production-ready with proper security, optimization, and error handling.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement all 14 Tauri commands with full functionality, security hardening, and performance optimization.
|
||||
|
||||
---
|
||||
|
||||
## 1. Command Implementation Matrix
|
||||
|
||||
| Module | Command | Current | Target | Priority | Security |
|
||||
|--------|---------|---------|--------|----------|----------|
|
||||
| **Settings** | `get_settings` | ✅ Done | ✅ Done | P0 | File permissions |
|
||||
| | `save_settings` | ✅ Done | ✅ Done | P0 | Input validation |
|
||||
| **Discovery** | `discover_nodes` | Stub | Full mDNS + UDP | P1 | Network boundary |
|
||||
| | `list_serial_ports` | Stub | Real enumeration | P1 | USB device access |
|
||||
| **Flash** | `flash_firmware` | Stub | espflash integration | P1 | Binary validation |
|
||||
| | `flash_progress` | Stub | Event streaming | P1 | Progress channel |
|
||||
| **OTA** | `ota_update` | Stub | HTTP multipart + PSK | P1 | TLS + PSK auth |
|
||||
| | `batch_ota_update` | Stub | Parallel with backoff | P2 | Rate limiting |
|
||||
| **WASM** | `wasm_list` | Stub | HTTP GET /api/wasm | P2 | Response validation |
|
||||
| | `wasm_upload` | Stub | HTTP POST multipart | P2 | Size limits, signing |
|
||||
| | `wasm_control` | Stub | HTTP POST commands | P2 | Action whitelist |
|
||||
| **Server** | `start_server` | Partial | Child process spawn | P1 | Port validation |
|
||||
| | `stop_server` | Partial | Graceful shutdown | P1 | PID verification |
|
||||
| | `server_status` | Partial | Health check | P1 | Timeout handling |
|
||||
| **Provision** | `provision_node` | Stub | NVS binary write | P2 | Serial validation |
|
||||
| | `read_nvs` | Stub | NVS binary read | P2 | Parse validation |
|
||||
|
||||
---
|
||||
|
||||
## 2. Implementation Details
|
||||
|
||||
### 2.1 Discovery Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
mdns-sd = "0.11"
|
||||
serialport = "4.6"
|
||||
tokio = { version = "1", features = ["net", "time"] }
|
||||
```
|
||||
|
||||
**discover_nodes Implementation:**
|
||||
```rust
|
||||
pub async fn discover_nodes(timeout_ms: Option<u64>) -> Result<Vec<DiscoveredNode>, String> {
|
||||
let timeout = Duration::from_millis(timeout_ms.unwrap_or(3000));
|
||||
let mut nodes = Vec::new();
|
||||
|
||||
// 1. mDNS discovery (_ruview._tcp.local)
|
||||
let mdns = ServiceDaemon::new()?;
|
||||
let receiver = mdns.browse("_ruview._tcp.local.")?;
|
||||
|
||||
// 2. UDP broadcast probe (port 5005)
|
||||
let socket = UdpSocket::bind("0.0.0.0:0").await?;
|
||||
socket.set_broadcast(true)?;
|
||||
socket.send_to(b"RUVIEW_DISCOVER", "255.255.255.255:5005").await?;
|
||||
|
||||
// 3. Collect responses with timeout
|
||||
tokio::select! {
|
||||
_ = collect_mdns(&receiver, &mut nodes) => {},
|
||||
_ = collect_udp(&socket, &mut nodes) => {},
|
||||
_ = tokio::time::sleep(timeout) => {},
|
||||
}
|
||||
|
||||
Ok(nodes)
|
||||
}
|
||||
```
|
||||
|
||||
**list_serial_ports Implementation:**
|
||||
```rust
|
||||
pub async fn list_serial_ports() -> Result<Vec<SerialPortInfo>, String> {
|
||||
let ports = serialport::available_ports()
|
||||
.map_err(|e| format!("Failed to enumerate ports: {}", e))?;
|
||||
|
||||
Ok(ports.into_iter().map(|p| SerialPortInfo {
|
||||
name: p.port_name,
|
||||
vid: extract_vid(&p.port_type),
|
||||
pid: extract_pid(&p.port_type),
|
||||
manufacturer: extract_manufacturer(&p.port_type),
|
||||
chip: detect_esp_chip(&p.port_type),
|
||||
}).collect())
|
||||
}
|
||||
```
|
||||
|
||||
### 2.2 Flash Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
espflash = "4.0"
|
||||
tokio = { version = "1", features = ["sync"] }
|
||||
```
|
||||
|
||||
**flash_firmware Implementation:**
|
||||
```rust
|
||||
pub async fn flash_firmware(
|
||||
port: String,
|
||||
firmware_path: String,
|
||||
chip: Option<String>,
|
||||
baud: Option<u32>,
|
||||
app: AppHandle,
|
||||
) -> Result<FlashResult, String> {
|
||||
// 1. Validate firmware binary
|
||||
let firmware = std::fs::read(&firmware_path)
|
||||
.map_err(|e| format!("Cannot read firmware: {}", e))?;
|
||||
validate_esp_binary(&firmware)?;
|
||||
|
||||
// 2. Open serial connection
|
||||
let serial = serialport::new(&port, baud.unwrap_or(460800))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.open()
|
||||
.map_err(|e| format!("Cannot open {}: {}", port, e))?;
|
||||
|
||||
// 3. Connect to ESP bootloader
|
||||
let mut flasher = Flasher::connect(serial, None, None)?;
|
||||
|
||||
// 4. Flash with progress callback
|
||||
let start = Instant::now();
|
||||
flasher.write_bin_to_flash(
|
||||
0x0,
|
||||
&firmware,
|
||||
Some(&mut |current, total| {
|
||||
let _ = app.emit("flash_progress", FlashProgress {
|
||||
phase: "writing".into(),
|
||||
progress_pct: (current as f32 / total as f32) * 100.0,
|
||||
bytes_written: current as u64,
|
||||
bytes_total: total as u64,
|
||||
});
|
||||
}),
|
||||
)?;
|
||||
|
||||
Ok(FlashResult {
|
||||
success: true,
|
||||
message: "Flash complete".into(),
|
||||
duration_secs: start.elapsed().as_secs_f64(),
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
### 2.3 OTA Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
reqwest = { version = "0.12", features = ["multipart", "rustls-tls"] }
|
||||
sha2 = "0.10"
|
||||
```
|
||||
|
||||
**ota_update Implementation:**
|
||||
```rust
|
||||
pub async fn ota_update(
|
||||
node_ip: String,
|
||||
firmware_path: String,
|
||||
psk: Option<String>,
|
||||
) -> Result<OtaResult, String> {
|
||||
// 1. Validate IP format
|
||||
let ip: IpAddr = node_ip.parse()
|
||||
.map_err(|_| "Invalid IP address")?;
|
||||
|
||||
// 2. Read and hash firmware
|
||||
let firmware = tokio::fs::read(&firmware_path).await
|
||||
.map_err(|e| format!("Cannot read firmware: {}", e))?;
|
||||
let hash = Sha256::digest(&firmware);
|
||||
|
||||
// 3. Build multipart request
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()?;
|
||||
|
||||
let form = multipart::Form::new()
|
||||
.part("firmware", multipart::Part::bytes(firmware)
|
||||
.file_name("firmware.bin")
|
||||
.mime_str("application/octet-stream")?);
|
||||
|
||||
// 4. Send with PSK auth header
|
||||
let mut req = client.post(format!("http://{}:8032/ota", ip))
|
||||
.multipart(form);
|
||||
|
||||
if let Some(key) = psk {
|
||||
req = req.header("X-OTA-PSK", key);
|
||||
}
|
||||
|
||||
let resp = req.send().await
|
||||
.map_err(|e| format!("OTA request failed: {}", e))?;
|
||||
|
||||
if resp.status().is_success() {
|
||||
Ok(OtaResult {
|
||||
success: true,
|
||||
node_ip: node_ip.clone(),
|
||||
message: "OTA update initiated".into(),
|
||||
})
|
||||
} else {
|
||||
Err(format!("OTA failed: {}", resp.status()))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**batch_ota_update Implementation:**
|
||||
```rust
|
||||
pub async fn batch_ota_update(
|
||||
node_ips: Vec<String>,
|
||||
firmware_path: String,
|
||||
psk: Option<String>,
|
||||
strategy: Option<String>,
|
||||
) -> Result<Vec<OtaResult>, String> {
|
||||
let firmware = Arc::new(tokio::fs::read(&firmware_path).await?);
|
||||
let psk = Arc::new(psk);
|
||||
|
||||
let strategy = strategy.unwrap_or("sequential".into());
|
||||
|
||||
match strategy.as_str() {
|
||||
"parallel" => {
|
||||
// All at once (max 4 concurrent)
|
||||
let semaphore = Arc::new(Semaphore::new(4));
|
||||
let handles: Vec<_> = node_ips.into_iter().map(|ip| {
|
||||
let fw = firmware.clone();
|
||||
let key = psk.clone();
|
||||
let sem = semaphore.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = sem.acquire().await;
|
||||
ota_single(&ip, &fw, key.as_ref().as_ref()).await
|
||||
})
|
||||
}).collect();
|
||||
|
||||
let results = futures::future::join_all(handles).await;
|
||||
Ok(results.into_iter().filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
"tdm_safe" => {
|
||||
// One per TDM slot group with delays
|
||||
let mut results = Vec::new();
|
||||
for ip in node_ips {
|
||||
results.push(ota_single(&ip, &firmware, psk.as_ref().as_ref()).await);
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
_ => {
|
||||
// Sequential (default)
|
||||
let mut results = Vec::new();
|
||||
for ip in node_ips {
|
||||
results.push(ota_single(&ip, &firmware, psk.as_ref().as_ref()).await);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.4 Server Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
tokio = { version = "1", features = ["process"] }
|
||||
sysinfo = "0.32"
|
||||
```
|
||||
|
||||
**start_server Implementation:**
|
||||
```rust
|
||||
pub async fn start_server(
|
||||
config: ServerConfig,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<(), String> {
|
||||
// 1. Check if already running
|
||||
{
|
||||
let srv = state.server.lock().map_err(|e| e.to_string())?;
|
||||
if srv.running {
|
||||
return Err("Server already running".into());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Validate ports
|
||||
validate_port(config.http_port.unwrap_or(8080))?;
|
||||
validate_port(config.ws_port.unwrap_or(8765))?;
|
||||
|
||||
// 3. Spawn sensing server as child process
|
||||
let child = Command::new("wifi-densepose-sensing-server")
|
||||
.args([
|
||||
"--http-port", &config.http_port.unwrap_or(8080).to_string(),
|
||||
"--ws-port", &config.ws_port.unwrap_or(8765).to_string(),
|
||||
"--udp-port", &config.udp_port.unwrap_or(5005).to_string(),
|
||||
])
|
||||
.spawn()
|
||||
.map_err(|e| format!("Failed to start server: {}", e))?;
|
||||
|
||||
// 4. Update state
|
||||
let mut srv = state.server.lock().map_err(|e| e.to_string())?;
|
||||
srv.running = true;
|
||||
srv.pid = Some(child.id());
|
||||
srv.child = Some(child);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
**stop_server Implementation:**
|
||||
```rust
|
||||
pub async fn stop_server(state: State<'_, AppState>) -> Result<(), String> {
|
||||
let mut srv = state.server.lock().map_err(|e| e.to_string())?;
|
||||
|
||||
if let Some(mut child) = srv.child.take() {
|
||||
// Graceful shutdown via SIGTERM
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use nix::sys::signal::{kill, Signal};
|
||||
use nix::unistd::Pid;
|
||||
let _ = kill(Pid::from_raw(child.id() as i32), Signal::SIGTERM);
|
||||
}
|
||||
|
||||
// Wait up to 5s, then force kill
|
||||
tokio::select! {
|
||||
_ = child.wait() => {},
|
||||
_ = tokio::time::sleep(Duration::from_secs(5)) => {
|
||||
let _ = child.kill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
srv.running = false;
|
||||
srv.pid = None;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### 2.5 WASM Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
reqwest = { version = "0.12", features = ["json", "multipart"] }
|
||||
```
|
||||
|
||||
**wasm_list Implementation:**
|
||||
```rust
|
||||
pub async fn wasm_list(node_ip: String) -> Result<Vec<WasmModuleInfo>, String> {
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client.get(format!("http://{}:8080/api/wasm", node_ip))
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Request failed: {}", e))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("Node returned {}", resp.status()));
|
||||
}
|
||||
|
||||
let modules: Vec<WasmModuleInfo> = resp.json().await
|
||||
.map_err(|e| format!("Invalid response: {}", e))?;
|
||||
|
||||
Ok(modules)
|
||||
}
|
||||
```
|
||||
|
||||
**wasm_upload Implementation:**
|
||||
```rust
|
||||
pub async fn wasm_upload(
|
||||
node_ip: String,
|
||||
wasm_path: String,
|
||||
) -> Result<WasmUploadResult, String> {
|
||||
// 1. Validate WASM binary
|
||||
let wasm = tokio::fs::read(&wasm_path).await
|
||||
.map_err(|e| format!("Cannot read WASM: {}", e))?;
|
||||
|
||||
if wasm.len() > 256 * 1024 {
|
||||
return Err("WASM module exceeds 256KB limit".into());
|
||||
}
|
||||
|
||||
if &wasm[0..4] != b"\0asm" {
|
||||
return Err("Invalid WASM magic bytes".into());
|
||||
}
|
||||
|
||||
// 2. Upload to node
|
||||
let client = reqwest::Client::new();
|
||||
let form = multipart::Form::new()
|
||||
.part("module", multipart::Part::bytes(wasm)
|
||||
.file_name(Path::new(&wasm_path).file_name().unwrap().to_string_lossy())
|
||||
.mime_str("application/wasm")?);
|
||||
|
||||
let resp = client.post(format!("http://{}:8080/api/wasm", node_ip))
|
||||
.multipart(form)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if resp.status().is_success() {
|
||||
let result: WasmUploadResult = resp.json().await?;
|
||||
Ok(result)
|
||||
} else {
|
||||
Err(format!("Upload failed: {}", resp.status()))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.6 Provision Module
|
||||
|
||||
**Dependencies:**
|
||||
```toml
|
||||
nvs-partition-tool = "0.1" # Or implement NVS binary format
|
||||
serialport = "4.6"
|
||||
```
|
||||
|
||||
**provision_node Implementation:**
|
||||
```rust
|
||||
pub async fn provision_node(
|
||||
port: String,
|
||||
config: ProvisioningConfig,
|
||||
) -> Result<ProvisionResult, String> {
|
||||
// 1. Validate config
|
||||
config.validate()?;
|
||||
|
||||
// 2. Build NVS binary blob
|
||||
let nvs_blob = build_nvs_blob(&config)?;
|
||||
|
||||
// 3. Open serial port
|
||||
let mut serial = serialport::new(&port, 115200)
|
||||
.timeout(Duration::from_secs(10))
|
||||
.open()
|
||||
.map_err(|e| format!("Cannot open {}: {}", port, e))?;
|
||||
|
||||
// 4. Enter bootloader mode
|
||||
enter_bootloader(&mut serial)?;
|
||||
|
||||
// 5. Write NVS partition (offset 0x9000, size 0x6000)
|
||||
write_partition(&mut serial, 0x9000, &nvs_blob)?;
|
||||
|
||||
// 6. Reset device
|
||||
reset_device(&mut serial)?;
|
||||
|
||||
Ok(ProvisionResult {
|
||||
success: true,
|
||||
message: "Provisioning complete".into(),
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Security Hardening
|
||||
|
||||
### 3.1 Input Validation
|
||||
|
||||
```rust
|
||||
// All string inputs sanitized
|
||||
fn validate_ip(ip: &str) -> Result<IpAddr, String> {
|
||||
ip.parse::<IpAddr>().map_err(|_| "Invalid IP address".into())
|
||||
}
|
||||
|
||||
fn validate_port(port: u16) -> Result<(), String> {
|
||||
if port < 1024 && port != 0 {
|
||||
return Err("Privileged ports (1-1023) not allowed".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_path(path: &str) -> Result<PathBuf, String> {
|
||||
let path = PathBuf::from(path);
|
||||
if path.components().any(|c| c == std::path::Component::ParentDir) {
|
||||
return Err("Path traversal detected".into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 Network Security
|
||||
|
||||
```rust
|
||||
// OTA PSK validation
|
||||
fn validate_psk(psk: &str) -> Result<(), String> {
|
||||
if psk.len() < 16 {
|
||||
return Err("PSK must be at least 16 characters".into());
|
||||
}
|
||||
if !psk.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
|
||||
return Err("PSK contains invalid characters".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Rate limiting for network operations
|
||||
struct RateLimiter {
|
||||
last_request: Instant,
|
||||
min_interval: Duration,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
fn check(&mut self) -> Result<(), String> {
|
||||
if self.last_request.elapsed() < self.min_interval {
|
||||
return Err("Rate limit exceeded".into());
|
||||
}
|
||||
self.last_request = Instant::now();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.3 Binary Validation
|
||||
|
||||
```rust
|
||||
fn validate_esp_binary(data: &[u8]) -> Result<(), String> {
|
||||
// Check ESP binary magic (0xE9 at offset 0)
|
||||
if data.is_empty() || data[0] != 0xE9 {
|
||||
return Err("Invalid ESP firmware magic byte".into());
|
||||
}
|
||||
|
||||
// Check minimum size (header + some code)
|
||||
if data.len() < 256 {
|
||||
return Err("Firmware too small".into());
|
||||
}
|
||||
|
||||
// Check maximum size (4MB flash)
|
||||
if data.len() > 4 * 1024 * 1024 {
|
||||
return Err("Firmware exceeds flash size".into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Performance Optimization
|
||||
|
||||
### 4.1 Async Everything
|
||||
|
||||
All I/O operations are async with proper timeouts:
|
||||
|
||||
```rust
|
||||
// Timeout wrapper
|
||||
async fn with_timeout<T, F: Future<Output = Result<T, String>>>(
|
||||
future: F,
|
||||
duration: Duration,
|
||||
) -> Result<T, String> {
|
||||
tokio::time::timeout(duration, future)
|
||||
.await
|
||||
.map_err(|_| "Operation timed out".into())?
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 Connection Pooling
|
||||
|
||||
```rust
|
||||
// Reusable HTTP client
|
||||
lazy_static! {
|
||||
static ref HTTP_CLIENT: reqwest::Client = reqwest::Client::builder()
|
||||
.pool_max_idle_per_host(5)
|
||||
.pool_idle_timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.unwrap();
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 Streaming Progress
|
||||
|
||||
Flash and OTA operations stream progress via Tauri events:
|
||||
|
||||
```rust
|
||||
// Real-time progress updates
|
||||
app.emit("flash_progress", FlashProgress { ... })?;
|
||||
app.emit("ota_progress", OtaProgress { ... })?;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Testing Strategy
|
||||
|
||||
### 5.1 Unit Tests
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn test_validate_ip() {
|
||||
assert!(validate_ip("192.168.1.1").is_ok());
|
||||
assert!(validate_ip("invalid").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_esp_binary() {
|
||||
let valid = vec![0xE9; 1024];
|
||||
assert!(validate_esp_binary(&valid).is_ok());
|
||||
|
||||
let invalid = vec![0x00; 1024];
|
||||
assert!(validate_esp_binary(&invalid).is_err());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5.2 Integration Tests
|
||||
|
||||
```rust
|
||||
#[tokio::test]
|
||||
async fn test_discover_nodes_timeout() {
|
||||
let result = discover_nodes(Some(100)).await;
|
||||
assert!(result.is_ok());
|
||||
// Should return empty or cached results within timeout
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 Mock Testing
|
||||
|
||||
```rust
|
||||
// Mock serial port for flash tests
|
||||
struct MockSerial {
|
||||
responses: VecDeque<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Read for MockSerial { ... }
|
||||
impl Write for MockSerial { ... }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Dependencies Update
|
||||
|
||||
**Cargo.toml additions:**
|
||||
```toml
|
||||
[dependencies]
|
||||
# Discovery
|
||||
mdns-sd = "0.11"
|
||||
serialport = "4.6"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", features = ["json", "multipart", "rustls-tls"] }
|
||||
|
||||
# Crypto
|
||||
sha2 = "0.10"
|
||||
|
||||
# Process management
|
||||
sysinfo = "0.32"
|
||||
|
||||
# Async
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
futures = "0.3"
|
||||
|
||||
# Flash
|
||||
espflash = "4.0"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Implementation Timeline
|
||||
|
||||
| Week | Deliverable |
|
||||
|------|-------------|
|
||||
| 1 | Discovery + Serial ports (real enumeration) |
|
||||
| 1 | Server start/stop (child process management) |
|
||||
| 2 | Flash firmware (espflash integration) |
|
||||
| 2 | OTA update (HTTP multipart) |
|
||||
| 3 | Batch OTA (parallel + sequential strategies) |
|
||||
| 3 | WASM management (list/upload/control) |
|
||||
| 4 | Provision NVS (binary format) |
|
||||
| 4 | Security audit + E2E testing |
|
||||
|
||||
---
|
||||
|
||||
## 8. Rollout Plan
|
||||
|
||||
1. **v0.3.1** — Settings fix + Discovery + Server
|
||||
2. **v0.4.0** — Flash + OTA (single node)
|
||||
3. **v0.5.0** — Batch OTA + WASM + Provision
|
||||
4. **v1.0.0** — Full E2E tested, security audited
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Desktop app becomes fully functional
|
||||
- Real device management capabilities
|
||||
- Production-ready security posture
|
||||
- Async performance throughout
|
||||
|
||||
### Negative
|
||||
- Additional dependencies increase binary size
|
||||
- espflash adds ~2MB to binary
|
||||
- Hardware required for full testing
|
||||
|
||||
### Neutral
|
||||
- Feature parity with browser-based UI
|
||||
- Same API contract as sensing server
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- [Tauri v2 Commands](https://v2.tauri.app/develop/commands/)
|
||||
- [espflash Documentation](https://github.com/esp-rs/espflash)
|
||||
- [ESP32 OTA Protocol](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/ota.html)
|
||||
- [mDNS-SD Rust](https://docs.rs/mdns-sd/)
|
||||
@@ -0,0 +1,119 @@
|
||||
# ADR-055: Integrated Sensing Server in Desktop App
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
The RuView Desktop application (ADR-054) requires the WiFi sensing server to provide real-time CSI data, activity detection, and vital signs monitoring. Currently, the sensing server is a separate binary (`wifi-densepose-sensing-server`) that must be installed separately and found in the system PATH.
|
||||
|
||||
This creates several problems:
|
||||
1. **Distribution complexity**: Users must install two binaries
|
||||
2. **Path issues**: Binary may not be in PATH, causing "No such file or directory" errors
|
||||
3. **Version mismatch**: Server and desktop app versions may diverge
|
||||
4. **Poor UX**: Error messages about missing binaries confuse users
|
||||
|
||||
## Decision
|
||||
Bundle the sensing server binary inside the desktop application and provide intelligent binary discovery with clear fallback paths.
|
||||
|
||||
### Binary Discovery Order
|
||||
The desktop app searches for the sensing server in this order:
|
||||
1. **Custom path** from user settings (`server_path`)
|
||||
2. **Bundled resources** (`Contents/Resources/bin/` on macOS)
|
||||
3. **Next to executable** (same directory as the app binary)
|
||||
4. **System PATH** (legacy fallback)
|
||||
|
||||
### Implementation
|
||||
```rust
|
||||
fn find_server_binary(app: &AppHandle, custom_path: Option<&str>) -> Result<String, String> {
|
||||
// 1. Custom path from settings
|
||||
if let Some(path) = custom_path {
|
||||
if std::path::Path::new(path).exists() {
|
||||
return Ok(path.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Bundled in resources
|
||||
if let Ok(resource_dir) = app.path().resource_dir() {
|
||||
let bundled = resource_dir.join("bin").join(DEFAULT_SERVER_BIN);
|
||||
if bundled.exists() {
|
||||
return Ok(bundled.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Next to executable
|
||||
if let Ok(exe_path) = std::env::current_exe() {
|
||||
if let Some(exe_dir) = exe_path.parent() {
|
||||
let sibling = exe_dir.join(DEFAULT_SERVER_BIN);
|
||||
if sibling.exists() {
|
||||
return Ok(sibling.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. System PATH
|
||||
// ... which lookup ...
|
||||
|
||||
Err("Sensing server binary not found")
|
||||
}
|
||||
```
|
||||
|
||||
### Bundle Configuration
|
||||
In `tauri.conf.json`:
|
||||
```json
|
||||
{
|
||||
"bundle": {
|
||||
"resources": [
|
||||
{
|
||||
"src": "../../target/release/wifi-densepose-sensing-server",
|
||||
"target": "bin/wifi-densepose-sensing-server"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- **Single package distribution**: Users download one DMG/MSI/EXE
|
||||
- **Version alignment**: Server and UI always match
|
||||
- **Better UX**: No PATH configuration required
|
||||
- **Offline capable**: Works without network access to download server
|
||||
|
||||
### Negative
|
||||
- **Larger bundle size**: ~10-15MB additional for server binary
|
||||
- **Build complexity**: Must build server before bundling desktop
|
||||
- **Platform-specific**: Need separate server binaries per platform
|
||||
|
||||
### Neutral
|
||||
- CI/CD workflow updated to build server before desktop
|
||||
- GitHub Actions builds all platforms (macOS arm64/x64, Windows x64)
|
||||
|
||||
## WebSocket Integration
|
||||
The Sensing page connects to the bundled server's WebSocket endpoint:
|
||||
- `ws://127.0.0.1:{ws_port}/ws/sensing` - Real-time CSI data stream
|
||||
- `ws://127.0.0.1:{ws_port}/ws/pose` - Pose estimation stream
|
||||
|
||||
Message format:
|
||||
```typescript
|
||||
interface WsSensingUpdate {
|
||||
type: string;
|
||||
timestamp: number;
|
||||
source: string;
|
||||
tick: number;
|
||||
nodes: WsNodeInfo[];
|
||||
classification: { motion_level: string; presence: boolean; confidence: number };
|
||||
vital_signs?: { breathing_rate_hz?: number; heart_rate_bpm?: number };
|
||||
}
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
- Server binary signed with same certificate as desktop app
|
||||
- Communication over localhost only (127.0.0.1)
|
||||
- No external network access by default
|
||||
- Process spawned as child of desktop app (inherits permissions)
|
||||
|
||||
## Related ADRs
|
||||
- ADR-054: Desktop Full Implementation
|
||||
- ADR-053: UI Design System
|
||||
- ADR-052: Tauri Desktop Frontend
|
||||
@@ -0,0 +1,251 @@
|
||||
# ADR-056: RuView Desktop Complete Capabilities Reference
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
RuView Desktop is a comprehensive WiFi-based sensing platform that combines hardware management, real-time signal processing, neural network inference, and intelligent monitoring. This ADR documents all integrated capabilities across the desktop application and underlying crates.
|
||||
|
||||
## Decision
|
||||
The RuView Desktop application consolidates all WiFi-DensePose functionality into a single, unified interface with the following capabilities.
|
||||
|
||||
---
|
||||
|
||||
## 1. Hardware Management
|
||||
|
||||
### 1.1 Node Discovery
|
||||
- **mDNS discovery**: Automatic detection of ESP32 nodes via Bonjour/Avahi
|
||||
- **UDP probe**: Direct UDP broadcast discovery on port 5005
|
||||
- **HTTP sweep**: Sequential IP scanning with health checks
|
||||
- **Manual registration**: User-defined node configuration
|
||||
|
||||
### 1.2 Firmware Flashing
|
||||
- **Serial flashing**: Direct USB flash via espflash integration
|
||||
- **Chip detection**: Automatic ESP32/S2/S3/C3/C6 identification
|
||||
- **Progress monitoring**: Real-time progress with speed metrics
|
||||
- **Verification**: Post-flash integrity verification
|
||||
|
||||
### 1.3 OTA Updates
|
||||
- **Single-node OTA**: HTTP-based firmware push to individual nodes
|
||||
- **Batch OTA**: Coordinated multi-node updates with strategies:
|
||||
- `sequential`: One node at a time
|
||||
- `tdm_safe`: Respects TDM slot timing
|
||||
- `parallel`: Concurrent updates with throttling
|
||||
- **Rollback support**: Automatic rollback on verification failure
|
||||
- **Version tracking**: Pre/post version comparison
|
||||
|
||||
### 1.4 Node Configuration
|
||||
- **NVS provisioning**: WiFi credentials, node ID, TDM slot assignment
|
||||
- **Mesh configuration**: Coordinator/node/aggregator role assignment
|
||||
- **TDM scheduling**: Time-division multiplexing slot allocation
|
||||
|
||||
---
|
||||
|
||||
## 2. Sensing Server
|
||||
|
||||
### 2.1 Data Sources
|
||||
- **ESP32 CSI**: Real UDP frames from ESP32 hardware (port 5005)
|
||||
- **Windows WiFi**: Native Windows RSSI monitoring via netsh
|
||||
- **Simulation**: Synthetic data generation for demo/testing
|
||||
- **Auto**: Automatic source detection based on available hardware
|
||||
|
||||
### 2.2 Real-Time Processing
|
||||
- **CSI pipeline**: 56-subcarrier amplitude/phase extraction
|
||||
- **FFT analysis**: Spectral decomposition for motion detection
|
||||
- **Vital signs**: Breathing rate (0.1-0.5 Hz), heart rate (0.8-2.0 Hz)
|
||||
- **Motion classification**: still/walking/running/exercising
|
||||
- **Presence detection**: Binary presence with confidence score
|
||||
|
||||
### 2.3 WebSocket Streaming
|
||||
- **Sensing endpoint**: `ws://localhost:8765/ws/sensing`
|
||||
- **Pose endpoint**: `ws://localhost:8765/ws/pose`
|
||||
- **Real-time broadcast**: 10-100 Hz update rate
|
||||
- **Multi-client support**: Concurrent WebSocket connections
|
||||
|
||||
### 2.4 REST API
|
||||
- **Health check**: `GET /health`
|
||||
- **Status**: `GET /api/status`
|
||||
- **Recording control**: `POST /api/recording/start|stop`
|
||||
- **Model management**: `GET/POST /api/models`
|
||||
|
||||
---
|
||||
|
||||
## 3. Neural Network Inference
|
||||
|
||||
### 3.1 Model Formats
|
||||
- **RVF (RuVector Format)**: Proprietary binary container with:
|
||||
- Model weights (quantized f32/f16/i8)
|
||||
- Vital sign configuration
|
||||
- SONA environment profiles
|
||||
- Training provenance
|
||||
- Cryptographic attestation
|
||||
|
||||
### 3.2 Inference Capabilities
|
||||
- **Pose estimation**: 17 COCO keypoints from WiFi CSI
|
||||
- **Activity recognition**: Multi-class classification
|
||||
- **Vital signs**: Breathing and heart rate extraction
|
||||
- **Multi-person detection**: Up to 3 simultaneous subjects
|
||||
|
||||
### 3.3 Self-Learning (SONA)
|
||||
- **Environment adaptation**: LoRA-based fine-tuning to room geometry
|
||||
- **Profile switching**: Multiple learned environment profiles
|
||||
- **Online learning**: Continuous adaptation during runtime
|
||||
- **Transfer learning**: Profile export/import between deployments
|
||||
|
||||
---
|
||||
|
||||
## 4. WASM Edge Modules
|
||||
|
||||
### 4.1 Module Management
|
||||
- **Upload**: Deploy WASM modules to ESP32 nodes
|
||||
- **Start/Stop**: Runtime control of edge processing
|
||||
- **Status monitoring**: CPU, memory, execution count
|
||||
- **Hot reload**: Update modules without node reboot
|
||||
|
||||
### 4.2 Supported Operations
|
||||
- **Local filtering**: On-device noise reduction
|
||||
- **Feature extraction**: Pre-compute features at edge
|
||||
- **Compression**: Reduce data before transmission
|
||||
- **Custom logic**: User-defined processing pipelines
|
||||
|
||||
---
|
||||
|
||||
## 5. Mesh Visualization
|
||||
|
||||
### 5.1 Network Topology
|
||||
- **Live mesh view**: Real-time node connectivity graph
|
||||
- **Signal quality**: RSSI/SNR visualization per link
|
||||
- **Latency monitoring**: Round-trip time measurement
|
||||
- **Packet loss**: Delivery success rate tracking
|
||||
|
||||
### 5.2 CSI Visualization
|
||||
- **Amplitude heatmap**: Per-subcarrier amplitude display
|
||||
- **Phase unwrapping**: Continuous phase visualization
|
||||
- **Spectrogram**: Time-frequency representation
|
||||
- **Signal field**: 3D voxel grid of RF perturbations
|
||||
|
||||
---
|
||||
|
||||
## 6. Training & Export
|
||||
|
||||
### 6.1 Dataset Management
|
||||
- **Recording**: Capture CSI frames with annotations
|
||||
- **Labeling**: Activity and pose ground truth
|
||||
- **Augmentation**: Synthetic data generation
|
||||
- **Export**: Standard formats (JSON, CSV, NumPy)
|
||||
|
||||
### 6.2 Training Pipeline (ADR-023)
|
||||
- **Contrastive pretraining**: Self-supervised feature learning
|
||||
- **Supervised fine-tuning**: Labeled pose estimation
|
||||
- **SONA adaptation**: Environment-specific tuning
|
||||
- **Validation**: Cross-environment testing
|
||||
|
||||
### 6.3 Export Formats
|
||||
- **RVF container**: Production deployment format
|
||||
- **ONNX**: Interoperability with external tools
|
||||
- **PyTorch**: Research and experimentation
|
||||
- **Candle**: Rust-native inference
|
||||
|
||||
---
|
||||
|
||||
## 7. Security Features
|
||||
|
||||
### 7.1 Network Security
|
||||
- **OTA PSK**: Pre-shared key for firmware updates
|
||||
- **Node authentication**: MAC-based node verification
|
||||
- **Encrypted transport**: Optional TLS for API endpoints
|
||||
|
||||
### 7.2 Code Signing
|
||||
- **Firmware verification**: Hash-based integrity checks
|
||||
- **WASM attestation**: Module signature validation
|
||||
- **Model provenance**: Training lineage tracking
|
||||
|
||||
---
|
||||
|
||||
## 8. Configuration & Settings
|
||||
|
||||
### 8.1 Server Configuration
|
||||
- **Ports**: HTTP (8080), WebSocket (8765), UDP (5005)
|
||||
- **Bind address**: Localhost or network-wide
|
||||
- **Data source**: auto/wifi/esp32/simulate
|
||||
- **Log level**: debug/info/warn/error
|
||||
|
||||
### 8.2 Application Settings
|
||||
- **Theme**: Dark/light mode
|
||||
- **Auto-discovery**: Periodic node scanning
|
||||
- **Discovery interval**: Configurable scan frequency
|
||||
- **UI customization**: Responsive layout options
|
||||
|
||||
---
|
||||
|
||||
## 9. Crate Architecture
|
||||
|
||||
| Crate | Capabilities |
|
||||
|-------|-------------|
|
||||
| `wifi-densepose-core` | CSI frame primitives, traits, error types |
|
||||
| `wifi-densepose-signal` | FFT, phase unwrapping, vital signs, RuvSense |
|
||||
| `wifi-densepose-nn` | ONNX/PyTorch/Candle inference backends |
|
||||
| `wifi-densepose-train` | Training pipeline, dataset, metrics |
|
||||
| `wifi-densepose-mat` | Mass casualty assessment tool |
|
||||
| `wifi-densepose-hardware` | ESP32 protocol, TDM, channel hopping |
|
||||
| `wifi-densepose-ruvector` | Cross-viewpoint fusion, attention |
|
||||
| `wifi-densepose-api` | REST API (Axum) |
|
||||
| `wifi-densepose-db` | Postgres/SQLite/Redis persistence |
|
||||
| `wifi-densepose-config` | Configuration management |
|
||||
| `wifi-densepose-wasm` | Browser WASM bindings |
|
||||
| `wifi-densepose-cli` | Command-line interface |
|
||||
| `wifi-densepose-sensing-server` | Real-time sensing server |
|
||||
| `wifi-densepose-wifiscan` | Multi-BSSID scanning |
|
||||
| `wifi-densepose-vitals` | Vital sign extraction |
|
||||
| `wifi-densepose-desktop` | Tauri desktop application |
|
||||
|
||||
---
|
||||
|
||||
## 10. UI Design System (ADR-053)
|
||||
|
||||
### 10.1 Pages
|
||||
- **Dashboard**: Overview, node status, quick actions
|
||||
- **Discovery**: Network scanning interface
|
||||
- **Nodes**: Node management and configuration
|
||||
- **Flash**: Serial firmware flashing
|
||||
- **OTA**: Over-the-air update management
|
||||
- **Edge Modules**: WASM deployment
|
||||
- **Sensing**: Real-time monitoring with server control
|
||||
- **Mesh View**: Network topology visualization
|
||||
- **Settings**: Application configuration
|
||||
|
||||
### 10.2 Components
|
||||
- **StatusBadge**: Health indicator
|
||||
- **NodeCard**: Node information display
|
||||
- **LogViewer**: Real-time log streaming
|
||||
- **ActivityFeed**: Sensing data visualization
|
||||
- **ProgressBar**: Operation progress
|
||||
- **ConfigForm**: Settings input
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- **Unified interface**: All capabilities in one application
|
||||
- **Bundled deployment**: Single package with server included
|
||||
- **Real-time feedback**: WebSocket-based live updates
|
||||
- **Cross-platform**: macOS, Windows, Linux support
|
||||
- **Extensible**: WASM modules, custom models, API access
|
||||
|
||||
### Negative
|
||||
- **Larger bundle**: ~6MB app + ~2.6MB server
|
||||
- **Complexity**: Many features require learning curve
|
||||
- **Hardware dependency**: Full functionality requires ESP32 nodes
|
||||
|
||||
### Neutral
|
||||
- Documentation required for all features
|
||||
- Training materials needed for advanced capabilities
|
||||
- Community contributions welcome
|
||||
|
||||
## Related ADRs
|
||||
- ADR-053: UI Design System
|
||||
- ADR-054: Desktop Full Implementation
|
||||
- ADR-055: Integrated Sensing Server
|
||||
- ADR-023: 8-Phase Training Pipeline
|
||||
- ADR-016: RuVector Integration
|
||||
@@ -0,0 +1,82 @@
|
||||
# ADR-057: Firmware CSI Build Guard and sdkconfig.defaults
|
||||
|
||||
| Field | Value |
|
||||
|-------------|---------------------------------------------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-03-12 |
|
||||
| **Authors** | ruv |
|
||||
| **Issues** | #223, #238, #234, #210, #190 |
|
||||
|
||||
## Context
|
||||
|
||||
Multiple GitHub issues (#223, #238, #234, #210, #190) report firmware problems
|
||||
that fall into two categories:
|
||||
|
||||
1. **CSI not enabled at runtime** — The committed `sdkconfig` had
|
||||
`# CONFIG_ESP_WIFI_CSI_ENABLED is not set` (line 1135), meaning users who
|
||||
built from source or used pre-built binaries got the runtime error:
|
||||
`E (6700) wifi:CSI not enabled in menuconfig!`
|
||||
|
||||
Root cause: `sdkconfig.defaults.template` existed with the correct setting
|
||||
(`CONFIG_ESP_WIFI_CSI_ENABLED=y`) but ESP-IDF only reads
|
||||
`sdkconfig.defaults` — not `.template` suffixed files. No `sdkconfig.defaults`
|
||||
file was committed.
|
||||
|
||||
2. **Unsupported ESP32 variants** — Users attempting to use original ESP32
|
||||
(D0WD) and ESP32-C3 boards. The firmware targets ESP32-S3 only
|
||||
(`CONFIG_IDF_TARGET="esp32s3"`, Xtensa architecture) and this was not
|
||||
surfaced clearly enough in documentation or build errors.
|
||||
|
||||
## Decision
|
||||
|
||||
### Fix 1: Commit `sdkconfig.defaults` (not just template)
|
||||
|
||||
Copy `sdkconfig.defaults.template` → `sdkconfig.defaults` so that ESP-IDF
|
||||
applies the correct defaults (including `CONFIG_ESP_WIFI_CSI_ENABLED=y`)
|
||||
automatically when `sdkconfig` is regenerated.
|
||||
|
||||
### Fix 2: `#error` compile-time guard in `csi_collector.c`
|
||||
|
||||
Add a preprocessor guard:
|
||||
|
||||
```c
|
||||
#ifndef CONFIG_ESP_WIFI_CSI_ENABLED
|
||||
#error "CONFIG_ESP_WIFI_CSI_ENABLED must be set in sdkconfig."
|
||||
#endif
|
||||
```
|
||||
|
||||
This turns a confusing runtime crash into a clear compile-time error with
|
||||
instructions on how to fix it.
|
||||
|
||||
### Fix 3: Fix committed `sdkconfig`
|
||||
|
||||
Change line 1135 from `# CONFIG_ESP_WIFI_CSI_ENABLED is not set` to
|
||||
`CONFIG_ESP_WIFI_CSI_ENABLED=y`.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Positive**: New builds will always have CSI enabled. Users building from
|
||||
source will get a clear compile error if CSI is somehow disabled.
|
||||
- **Positive**: Pre-built release binaries will include CSI support.
|
||||
- **Neutral**: Original ESP32 and ESP32-C3 remain unsupported. This is by
|
||||
design — only ESP32-S3 has the CSI API surface we depend on. Future ADRs
|
||||
may address multi-target support if demand warrants it.
|
||||
- **Negative**: None identified.
|
||||
|
||||
## Hardware Support Matrix
|
||||
|
||||
| Variant | CSI Support | Firmware Target | Status |
|
||||
|--------------|-------------|-----------------|---------------|
|
||||
| ESP32-S3 | Yes | Yes | Supported |
|
||||
| ESP32 (orig) | Partial | No | Unsupported |
|
||||
| ESP32-C3 | Yes (IDF 5.1+) | No | Unsupported |
|
||||
| ESP32-C6 | Yes | No | Unsupported |
|
||||
|
||||
## Notes
|
||||
|
||||
- ESP32-C3 and C6 use RISC-V architecture; a separate build target
|
||||
(`idf.py set-target esp32c3`) would be needed.
|
||||
- Original ESP32 has limited CSI (no STBC HT-LTF2, fewer subcarriers).
|
||||
- Users on unsupported hardware can still write custom firmware using the
|
||||
ADR-018 binary frame format (magic `0xC5110001`) for interop with the
|
||||
Rust aggregator.
|
||||
@@ -0,0 +1,392 @@
|
||||
# ADR-058: Dual-Modal WASM Browser Pose Estimation — Live Video + WiFi CSI Fusion
|
||||
|
||||
- **Status**: Proposed
|
||||
- **Date**: 2026-03-12
|
||||
- **Deciders**: ruv
|
||||
- **Tags**: wasm, browser, cnn, pose-estimation, ruvector, video, multimodal, fusion
|
||||
|
||||
## Context
|
||||
|
||||
WiFi-DensePose estimates human poses from WiFi CSI (Channel State Information).
|
||||
The `ruvector-cnn` crate provides a pure Rust CNN (MobileNet-V3) with WASM bindings.
|
||||
Both modalities exist independently — what's missing is **fusing live webcam video
|
||||
with WiFi CSI** in a single browser demo to achieve robust pose estimation that
|
||||
works even when one modality degrades (occlusion, signal noise, poor lighting).
|
||||
|
||||
Existing assets:
|
||||
|
||||
1. **`wifi-densepose-wasm`** — CSI signal processing compiled to WASM
|
||||
2. **`wifi-densepose-sensing-server`** — Axum server streaming live CSI via WebSocket
|
||||
3. **`ruvector-cnn`** — Pure Rust CNN with MobileNet-V3 backbones, SIMD, contrastive learning
|
||||
4. **`ruvector-cnn-wasm`** — wasm-bindgen bindings: `WasmCnnEmbedder`, `SimdOps`, `LayerOps`, contrastive losses
|
||||
5. **`vendor/ruvector/examples/wasm-vanilla/`** — Reference vanilla JS WASM example
|
||||
|
||||
Research shows multi-modal fusion (camera + WiFi) significantly outperforms either alone:
|
||||
- Camera fails under occlusion, poor lighting, privacy constraints
|
||||
- WiFi CSI fails with signal noise, multipath, low spatial resolution
|
||||
- Fusion compensates: WiFi provides through-wall coverage, camera provides fine-grained detail
|
||||
|
||||
## Decision
|
||||
|
||||
Build a **dual-modal browser demo** at `examples/wasm-browser-pose/` that:
|
||||
|
||||
1. Captures **live webcam video** via `getUserMedia` API
|
||||
2. Receives **live WiFi CSI** via WebSocket from the sensing server
|
||||
3. Processes **both streams** through separate CNN pipelines in `ruvector-cnn-wasm`
|
||||
4. **Fuses embeddings** with learned attention weights for combined pose estimation
|
||||
5. Renders **video overlay** with skeleton + WiFi confidence heatmap on Canvas
|
||||
6. Runs entirely in the browser — all inference client-side via WASM
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ Browser │
|
||||
│ │
|
||||
│ ┌────────────┐ ┌────────────────┐ ┌───────────────────┐ │
|
||||
│ │ getUserMedia│───▶│ Video Frame │───▶│ CNN WASM │ │
|
||||
│ │ (Webcam) │ │ Capture │ │ (Visual Embedder) │ │
|
||||
│ └────────────┘ │ 224×224 RGB │ │ → 512-dim │ │
|
||||
│ └────────────────┘ └────────┬──────────┘ │
|
||||
│ │ │
|
||||
│ visual_embedding │
|
||||
│ │ │
|
||||
│ ┌──────▼──────┐ │
|
||||
│ ┌────────────┐ ┌────────────────┐ │ │ │
|
||||
│ │ WebSocket │───▶│ CSI WASM │ │ Attention │ │
|
||||
│ │ Client │ │ (densepose- │ │ Fusion │ │
|
||||
│ │ │ │ wasm) │ │ Module │ │
|
||||
│ └────────────┘ └───────┬────────┘ │ │ │
|
||||
│ │ └──────┬──────┘ │
|
||||
│ ┌───────▼────────┐ │ │
|
||||
│ │ CNN WASM │ fused_embedding │
|
||||
│ │ (CSI Embedder) │ │ │
|
||||
│ │ → 512-dim │ ┌──────▼──────┐ │
|
||||
│ └───────┬────────┘ │ Pose │ │
|
||||
│ │ │ Decoder │ │
|
||||
│ csi_embedding │ → 17 kpts │ │
|
||||
│ │ └──────┬──────┘ │
|
||||
│ └──────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────────┐ ┌─────▼──────┐ │
|
||||
│ │ Video Canvas │◀────────│ Overlay │ │
|
||||
│ │ + Skeleton │ │ Renderer │ │
|
||||
│ │ + Heatmap │ └────────────┘ │
|
||||
│ └──────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
▲ ▲
|
||||
│ getUserMedia │ WebSocket
|
||||
│ (camera) │ (ws://host:3030/ws/csi)
|
||||
│ │
|
||||
┌────┴────┐ ┌───────┴─────────┐
|
||||
│ Webcam │ │ Sensing Server │
|
||||
└─────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Dual Pipeline Design
|
||||
|
||||
Two parallel CNN pipelines run on each frame tick (~30 FPS):
|
||||
|
||||
| Pipeline | Input | Preprocessing | CNN Config | Output |
|
||||
|----------|-------|---------------|------------|--------|
|
||||
| **Visual** | Webcam frame (640×480) | Resize to 224×224 RGB, ImageNet normalize | MobileNet-V3 Small, 512-dim | Visual embedding |
|
||||
| **CSI** | CSI frame (ADR-018 binary) | Amplitude/phase/delta → 224×224 pseudo-RGB | MobileNet-V3 Small, 512-dim | CSI embedding |
|
||||
|
||||
Both use the same `WasmCnnEmbedder` but with separate instances and weight sets.
|
||||
|
||||
### Fusion Strategy
|
||||
|
||||
**Learned attention-weighted fusion** combines the two 512-dim embeddings:
|
||||
|
||||
```javascript
|
||||
// Attention fusion: learn which modality to trust per-dimension
|
||||
// α ∈ [0,1]^512 — attention weights (shipped as JSON, trained offline)
|
||||
// visual_emb, csi_emb ∈ R^512
|
||||
|
||||
function fuseEmbeddings(visual_emb, csi_emb, attention_weights) {
|
||||
const fused = new Float32Array(512);
|
||||
for (let i = 0; i < 512; i++) {
|
||||
const α = attention_weights[i];
|
||||
fused[i] = α * visual_emb[i] + (1 - α) * csi_emb[i];
|
||||
}
|
||||
return fused;
|
||||
}
|
||||
```
|
||||
|
||||
**Dynamic confidence gating** adjusts fusion based on signal quality:
|
||||
|
||||
| Condition | Behavior |
|
||||
|-----------|----------|
|
||||
| Good video + good CSI | Balanced fusion (α ≈ 0.5) |
|
||||
| Poor lighting / occlusion | CSI-dominant (α → 0, WiFi takes over) |
|
||||
| CSI noise / no ESP32 | Video-dominant (α → 1, camera only) |
|
||||
| Video-only mode (no WiFi) | α = 1.0, pure visual CNN pose estimation |
|
||||
| CSI-only mode (no camera) | α = 0.0, pure WiFi pose estimation |
|
||||
|
||||
Quality detection:
|
||||
- **Video quality**: Frame brightness variance (dark = low quality), motion blur score
|
||||
- **CSI quality**: Signal-to-noise ratio from `wifi-densepose-wasm`, coherence gate output
|
||||
|
||||
### CSI-to-Image Encoding
|
||||
|
||||
CSI data encoded as 3-channel pseudo-image for the CSI CNN pipeline:
|
||||
|
||||
| Channel | Data | Normalization |
|
||||
|---------|------|---------------|
|
||||
| R | CSI amplitude (subcarrier × time window) | Min-max to [0, 255] |
|
||||
| G | CSI phase (unwrapped, subcarrier × time window) | Min-max to [0, 255] |
|
||||
| B | Temporal difference (frame-to-frame Δ amplitude) | Abs, min-max to [0, 255] |
|
||||
|
||||
### Video Processing
|
||||
|
||||
Webcam frames processed through standard ImageNet pipeline:
|
||||
|
||||
```javascript
|
||||
// Capture frame from video element
|
||||
const frame = captureVideoFrame(videoElement, 224, 224); // Returns Uint8Array RGB
|
||||
|
||||
// ImageNet normalization happens inside WasmCnnEmbedder.extract()
|
||||
const visual_embedding = visual_embedder.extract(frame, 224, 224);
|
||||
```
|
||||
|
||||
### Pose Keypoint Mapping
|
||||
|
||||
17 COCO-format keypoints decoded from the fused 512-dim embedding:
|
||||
|
||||
```
|
||||
0: nose 1: left_eye 2: right_eye
|
||||
3: left_ear 4: right_ear 5: left_shoulder
|
||||
6: right_shoulder 7: left_elbow 8: right_elbow
|
||||
9: left_wrist 10: right_wrist 11: left_hip
|
||||
12: right_hip 13: left_knee 14: right_knee
|
||||
15: left_ankle 16: right_ankle
|
||||
```
|
||||
|
||||
Each keypoint decoded as (x, y, confidence) = 51 values from the 512-dim embedding
|
||||
via a learned linear projection.
|
||||
|
||||
### Operating Modes
|
||||
|
||||
The demo supports three modes, selectable in the UI:
|
||||
|
||||
| Mode | Video | CSI | Fusion | Use Case |
|
||||
|------|-------|-----|--------|----------|
|
||||
| **Dual (default)** | ✅ | ✅ | Attention-weighted | Best accuracy, full demo |
|
||||
| **Video Only** | ✅ | ❌ | α = 1.0 | No ESP32 available, quick demo |
|
||||
| **CSI Only** | ❌ | ✅ | α = 0.0 | Privacy mode, through-wall sensing |
|
||||
|
||||
**Video Only mode works without any hardware** — just a webcam — making the demo
|
||||
instantly accessible for anyone wanting to try it.
|
||||
|
||||
### File Layout
|
||||
|
||||
```
|
||||
examples/wasm-browser-pose/
|
||||
├── index.html # Single-page app (vanilla JS, no bundler)
|
||||
├── js/
|
||||
│ ├── app.js # Main entry, mode selection, orchestration
|
||||
│ ├── video-capture.js # getUserMedia, frame extraction, quality detection
|
||||
│ ├── csi-processor.js # WebSocket CSI client, frame parsing, pseudo-image encoding
|
||||
│ ├── fusion.js # Attention-weighted embedding fusion, confidence gating
|
||||
│ ├── pose-decoder.js # Fused embedding → 17 keypoints
|
||||
│ └── canvas-renderer.js # Video overlay, skeleton, CSI heatmap, confidence bars
|
||||
├── data/
|
||||
│ ├── visual-weights.json # Visual CNN → embedding projection (placeholder until trained)
|
||||
│ ├── csi-weights.json # CSI CNN → embedding projection (placeholder until trained)
|
||||
│ ├── fusion-weights.json # Attention fusion α weights (512 values)
|
||||
│ └── pose-weights.json # Fused embedding → keypoint projection
|
||||
├── css/
|
||||
│ └── style.css # Dark theme UI styling
|
||||
├── pkg/ # Built WASM packages (gitignored, built by script)
|
||||
│ ├── wifi_densepose_wasm/
|
||||
│ └── ruvector_cnn_wasm/
|
||||
├── build.sh # wasm-pack build script for both packages
|
||||
└── README.md # Setup and usage instructions
|
||||
```
|
||||
|
||||
### Build Pipeline
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# build.sh — builds both WASM packages into pkg/
|
||||
|
||||
set -e
|
||||
|
||||
# Build wifi-densepose-wasm (CSI processing)
|
||||
wasm-pack build ../../rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm \
|
||||
--target web --out-dir "$(pwd)/pkg/wifi_densepose_wasm" --no-typescript
|
||||
|
||||
# Build ruvector-cnn-wasm (CNN inference for both video and CSI)
|
||||
wasm-pack build ../../vendor/ruvector/crates/ruvector-cnn-wasm \
|
||||
--target web --out-dir "$(pwd)/pkg/ruvector_cnn_wasm" --no-typescript
|
||||
|
||||
echo "Build complete. Serve with: python3 -m http.server 8080"
|
||||
```
|
||||
|
||||
### UI Layout
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ WiFi-DensePose — Live Dual-Modal Pose Estimation │
|
||||
│ [Dual Mode ▼] [⚙ Settings] FPS: 28 ◉ Live │
|
||||
├───────────────────────────┬─────────────────────────────┤
|
||||
│ │ │
|
||||
│ ┌───────────────────┐ │ ┌───────────────────┐ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ Video + Skeleton │ │ │ CSI Heatmap │ │
|
||||
│ │ Overlay │ │ │ (amplitude × │ │
|
||||
│ │ (main canvas) │ │ │ subcarrier) │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ └───────────────────┘ │ └───────────────────┘ │
|
||||
│ │ │
|
||||
├───────────────────────────┴─────────────────────────────┤
|
||||
│ Fusion Confidence: ████████░░ 78% │
|
||||
│ Video: ██████████ 95% │ CSI: ██████░░░░ 61% │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ ┌─────────────────────────────────────────────────┐ │
|
||||
│ │ Embedding Space (2D projection) │ │
|
||||
│ │ · · · │ │
|
||||
│ │ · · · · · · (color = pose cluster) │ │
|
||||
│ │ · · · · │ │
|
||||
│ └─────────────────────────────────────────────────┘ │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Latency: Video 12ms │ CSI 8ms │ Fusion 1ms │ Total 21ms│
|
||||
│ [▶ Record] [📷 Snapshot] [Confidence: ████ 0.6] │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### WASM Module Structure
|
||||
|
||||
| Package | Source Crate | Provides | Size (est.) |
|
||||
|---------|-------------|----------|-------------|
|
||||
| `wifi_densepose_wasm` | `wifi-densepose-wasm` | CSI frame parsing, signal processing, feature extraction | ~200KB |
|
||||
| `ruvector_cnn_wasm` | `ruvector-cnn-wasm` | `WasmCnnEmbedder` (×2 instances), `SimdOps`, `LayerOps`, contrastive losses | ~150KB |
|
||||
|
||||
Two `WasmCnnEmbedder` instances are created — one for video frames, one for CSI pseudo-images.
|
||||
They share the same WASM module but have independent state.
|
||||
|
||||
### Browser API Requirements
|
||||
|
||||
| API | Purpose | Required | Fallback |
|
||||
|-----|---------|----------|----------|
|
||||
| `getUserMedia` | Webcam capture | For video mode | CSI-only mode |
|
||||
| WebAssembly | CNN inference | Yes | None (hard requirement) |
|
||||
| WASM SIMD128 | Accelerated inference | No | Scalar fallback (~2× slower) |
|
||||
| WebSocket | CSI data stream | For CSI mode | Video-only mode |
|
||||
| Canvas 2D | Rendering | Yes | None |
|
||||
| `requestAnimationFrame` | Render loop | Yes | `setTimeout` fallback |
|
||||
| ES Modules | Code organization | Yes | None |
|
||||
|
||||
Target: Chrome 89+, Firefox 89+, Safari 15+, Edge 89+
|
||||
|
||||
### Performance Budget
|
||||
|
||||
| Stage | Target Latency | Notes |
|
||||
|-------|---------------|-------|
|
||||
| Video frame capture + resize | <3ms | `drawImage` to offscreen canvas |
|
||||
| Video CNN embedding | <15ms | 224×224 RGB → 512-dim |
|
||||
| CSI receive + parse | <2ms | Binary WebSocket message |
|
||||
| CSI pseudo-image encoding | <3ms | Amplitude/phase/delta channels |
|
||||
| CSI CNN embedding | <15ms | 224×224 pseudo-RGB → 512-dim |
|
||||
| Attention fusion | <1ms | Element-wise weighted sum |
|
||||
| Pose decoding | <1ms | Linear projection |
|
||||
| Canvas overlay render | <3ms | Video + skeleton + heatmap |
|
||||
| **Total (dual mode)** | **<33ms** | **30 FPS capable** |
|
||||
| **Total (video only)** | **<22ms** | **45 FPS capable** |
|
||||
|
||||
Note: Video and CSI CNN pipelines can run in parallel using Web Workers,
|
||||
reducing dual-mode latency to ~max(15, 15) + 5 = ~20ms (50 FPS).
|
||||
|
||||
### Contrastive Learning Integration
|
||||
|
||||
The demo optionally shows real-time contrastive learning in the browser:
|
||||
|
||||
- **InfoNCE loss** (`WasmInfoNCELoss`): Compare video vs CSI embeddings for the same pose — trains cross-modal alignment
|
||||
- **Triplet loss** (`WasmTripletLoss`): Push apart different poses, pull together same pose across modalities
|
||||
- **SimdOps**: Accelerated dot products for real-time similarity computation
|
||||
- **Embedding space panel**: Live 2D projection shows video and CSI embeddings converging when viewing the same person
|
||||
|
||||
### Relationship to Existing Crates
|
||||
|
||||
| Existing Crate | Role in This Demo |
|
||||
|---------------|-------------------|
|
||||
| `ruvector-cnn-wasm` | CNN inference for **both** video frames and CSI pseudo-images |
|
||||
| `wifi-densepose-wasm` | CSI frame parsing and signal processing |
|
||||
| `wifi-densepose-sensing-server` | WebSocket CSI data source |
|
||||
| `wifi-densepose-core` | ADR-018 frame format definitions |
|
||||
| `ruvector-cnn` | Underlying MobileNet-V3, layers, contrastive learning |
|
||||
|
||||
No new Rust crates are needed. The example is pure HTML/JS consuming existing WASM packages.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Instant demo**: Video-only mode works with just a webcam — no ESP32 needed
|
||||
- **Multi-modal showcase**: Demonstrates camera + WiFi fusion, the core innovation of the project
|
||||
- **Graceful degradation**: Works with video-only, CSI-only, or both
|
||||
- **Through-wall capability**: CSI mode shows pose estimation where cameras cannot reach
|
||||
- **Zero-install**: Anyone with a browser can try it
|
||||
- **Training data collection**: Can record paired (video, CSI) data for offline model training
|
||||
- **Reusable**: JS modules embed directly in the Tauri desktop app's webview
|
||||
|
||||
### Negative
|
||||
|
||||
- **Model weights**: Requires offline-trained weights for visual CNN, CSI CNN, fusion, and pose decoder (~200KB total JSON)
|
||||
- **WASM size**: Two WASM modules total ~350KB (acceptable)
|
||||
- **No GPU**: CPU-only WASM inference; adequate at 224×224 but limits resolution scaling
|
||||
- **Camera privacy**: Video mode requires camera permission (mitigated: CSI-only mode available)
|
||||
- **Two CNN instances**: Memory footprint doubles vs single-modal (~10MB total, acceptable for desktop browsers)
|
||||
|
||||
### Risks
|
||||
|
||||
- **Cross-modal alignment**: Video and CSI embeddings must be trained jointly for fusion to work;
|
||||
without proper training, fusion may be worse than either modality alone
|
||||
- **Latency on mobile**: Dual CNN on mobile browsers may exceed 33ms; implement automatic quality reduction
|
||||
- **WebSocket drops**: Network jitter → CSI frame gaps; buffer last 3 frames, interpolate missing data
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
1. **Phase 1 — Scaffold**: File layout, build.sh, index.html shell, mode selector UI
|
||||
2. **Phase 2 — Video pipeline**: getUserMedia → frame capture → CNN embedding → basic pose display
|
||||
3. **Phase 3 — CSI pipeline**: WebSocket client → CSI parsing → pseudo-image → CNN embedding
|
||||
4. **Phase 4 — Fusion**: Attention-weighted combination, confidence gating, mode switching
|
||||
5. **Phase 5 — Pose decoder**: Linear projection with placeholder weights → 17 keypoints
|
||||
6. **Phase 6 — Overlay renderer**: Video canvas with skeleton overlay, CSI heatmap panel
|
||||
7. **Phase 7 — Training**: Use `wifi-densepose-train` to generate real weights for both CNNs + fusion + decoder
|
||||
8. **Phase 8 — Contrastive demo**: Embedding space visualization, cross-modal similarity display
|
||||
9. **Phase 9 — Web Workers**: Move CNN inference to workers for parallel video + CSI processing
|
||||
10. **Phase 10 — Polish**: Recording, snapshots, adaptive quality, mobile optimization
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### 1. CSI-Only (No Video)
|
||||
Rejected: Misses the opportunity to show multi-modal fusion and makes the demo less
|
||||
accessible (requires ESP32 hardware). Video-only mode as a fallback is strictly better.
|
||||
|
||||
### 2. Server-Side Video Inference
|
||||
Rejected: Adds latency, requires webcam stream upload (privacy concern), and defeats
|
||||
the WASM-first architecture. All inference must be client-side.
|
||||
|
||||
### 3. TensorFlow.js for Video, ruvector-cnn-wasm for CSI
|
||||
Rejected: Would require two different ML frameworks. Using `ruvector-cnn-wasm` for both
|
||||
keeps a single WASM module, unified embedding space, and simpler fusion.
|
||||
|
||||
### 4. Pre-recorded Video Demo
|
||||
Rejected: Live webcam input is far more compelling for demonstrations.
|
||||
Pre-recorded mode can be added as a secondary option.
|
||||
|
||||
### 5. React/Vue Framework
|
||||
Rejected: Adds build tooling. Vanilla JS + ES modules keeps the demo self-contained.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR-018: Binary CSI Frame Format](ADR-018-binary-csi-frame-format.md)
|
||||
- [ADR-024: Contrastive CSI Embedding / AETHER](ADR-024-contrastive-csi-embedding.md)
|
||||
- [ADR-055: Integrated Sensing Server](ADR-055-integrated-sensing-server.md)
|
||||
- `vendor/ruvector/crates/ruvector-cnn/src/lib.rs` — CNN embedder implementation
|
||||
- `vendor/ruvector/crates/ruvector-cnn-wasm/src/lib.rs` — WASM bindings
|
||||
- `vendor/ruvector/examples/wasm-vanilla/index.html` — Reference vanilla JS WASM pattern
|
||||
- Person-in-WiFi: Fine-grained Person Perception using WiFi (ICCV 2019) — camera+WiFi fusion precedent
|
||||
- WiPose: Multi-Person WiFi Pose Estimation (TMC 2022) — cross-modal embedding approach
|
||||
@@ -0,0 +1,83 @@
|
||||
# ADR-059: Live ESP32 CSI Pipeline Integration
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Date
|
||||
|
||||
2026-03-12
|
||||
|
||||
## Context
|
||||
|
||||
ADR-058 established a dual-modal browser demo combining webcam video and WiFi CSI for pose estimation. However, it used simulated CSI data. To demonstrate real-world capability, we need an end-to-end pipeline from physical ESP32 hardware through to the browser visualization.
|
||||
|
||||
The ESP32-S3 firmware (`firmware/esp32-csi-node/`) already supports CSI collection and UDP streaming (ADR-018). The sensing server (`wifi-densepose-sensing-server`) already supports UDP ingestion and WebSocket bridging. The missing piece was connecting these components and enabling the browser demo to consume live data.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a complete live CSI pipeline:
|
||||
|
||||
```
|
||||
ESP32-S3 (CSI capture) → UDP:5005 → sensing-server (Rust/Axum) → WS:8765 → browser demo
|
||||
```
|
||||
|
||||
### Components
|
||||
|
||||
1. **ESP32 Firmware** — Rebuilt with native Windows ESP-IDF v5.4.0 toolchain (no Docker). Configured for target network and PC IP via `sdkconfig`. Helper scripts added:
|
||||
- `build_firmware.ps1` — Sets up IDF environment, cleans, builds, and flashes
|
||||
- `read_serial.ps1` — Serial monitor with DTR/RTS reset capability
|
||||
|
||||
2. **Sensing Server** — `wifi-densepose-sensing-server` started with:
|
||||
- `--source esp32` — Expect real ESP32 UDP frames
|
||||
- `--bind-addr 0.0.0.0` — Accept connections from any interface
|
||||
- `--ui-path <path>` — Serve the demo UI via HTTP
|
||||
|
||||
3. **Browser Demo** — `main.js` updated to auto-connect to `ws://localhost:8765/ws/sensing` on page load. Falls back to simulated CSI if the WebSocket is unavailable (GitHub Pages).
|
||||
|
||||
### Network Configuration
|
||||
|
||||
The ESP32 sends UDP packets to a configured target IP. If the PC's IP doesn't match the firmware's compiled target, a secondary IP alias can be added:
|
||||
|
||||
```powershell
|
||||
# PowerShell (Admin)
|
||||
New-NetIPAddress -IPAddress 192.168.1.100 -PrefixLength 24 -InterfaceAlias "Wi-Fi"
|
||||
```
|
||||
|
||||
### Data Flow
|
||||
|
||||
| Stage | Protocol | Format | Rate |
|
||||
|-------|----------|--------|------|
|
||||
| ESP32 → Server | UDP | ADR-018 binary frame (magic `0xC5110001`, I/Q pairs) | ~100 Hz |
|
||||
| Server → Browser | WebSocket | ADR-018 binary frame (forwarded) | ~10 Hz (tick-ms=100) |
|
||||
| Browser decode | JavaScript | Float32 amplitude/phase arrays | Per frame |
|
||||
|
||||
### Build Environment (Windows)
|
||||
|
||||
ESP-IDF v5.4.0 on Windows requires:
|
||||
- IDF_PATH pointing to the ESP-IDF framework
|
||||
- IDF_TOOLS_PATH pointing to toolchain binaries
|
||||
- MSYS/MinGW environment variables removed (ESP-IDF rejects them)
|
||||
- Python venv from ESP-IDF tools for `idf.py` execution
|
||||
|
||||
The `build_firmware.ps1` script handles all of this automatically.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- First end-to-end demonstration of real WiFi CSI → pose estimation in a browser
|
||||
- No Docker required for firmware builds on Windows
|
||||
- Demo gracefully degrades to simulated CSI when no server is available
|
||||
- Same demo works on GitHub Pages (simulated) and locally (live ESP32)
|
||||
|
||||
### Negative
|
||||
- ESP32 target IP is compiled into firmware; changing it requires a rebuild or NVS override
|
||||
- Windows firewall may block UDP:5005; user must allow it
|
||||
- Mixed content restrictions prevent HTTPS pages from connecting to ws:// (local only)
|
||||
|
||||
## Related
|
||||
|
||||
- [ADR-018](ADR-018-esp32-dev-implementation.md) — ESP32 CSI frame format and UDP streaming
|
||||
- [ADR-058](ADR-058-ruvector-wasm-browser-pose-example.md) — Dual-modal WASM browser pose demo
|
||||
- [ADR-039](ADR-039-edge-intelligence-framework.md) — Edge intelligence on ESP32
|
||||
- Issue [#245](https://github.com/ruvnet/RuView/issues/245) — Tracking issue
|
||||
@@ -0,0 +1,59 @@
|
||||
# ADR-060: Provision Channel Override and MAC Address Filtering
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-03-12
|
||||
- **Issues:** [#247](https://github.com/ruvnet/RuView/issues/247), [#229](https://github.com/ruvnet/RuView/issues/229)
|
||||
|
||||
## Context
|
||||
|
||||
Two related provisioning gaps were reported by users:
|
||||
|
||||
1. **Channel mismatch (Issue #247):** The CSI collector initializes on the
|
||||
Kconfig default channel (typically 6), even when the ESP32 connects to an AP
|
||||
on a different channel (e.g. 11). On managed networks where the user cannot
|
||||
change the router channel, this makes nodes undiscoverable. The
|
||||
`provision.py` script has no `--channel` argument.
|
||||
|
||||
2. **Missing MAC filter (Issue #229):** The v0.2.0 release notes documented a
|
||||
`--filter-mac` argument for `provision.py`, but it was never implemented.
|
||||
The firmware's CSI callback accepts frames from all sources, causing signal
|
||||
mixing in multi-AP environments.
|
||||
|
||||
## Decision
|
||||
|
||||
### Channel configuration
|
||||
|
||||
- Add `--channel` argument to `provision.py` that writes a `csi_channel` key
|
||||
(u8) to NVS.
|
||||
- In `nvs_config.c`, read the `csi_channel` key and override
|
||||
`channel_list[0]` when present.
|
||||
- In `csi_collector_init()`, after WiFi connects, auto-detect the AP channel
|
||||
via `esp_wifi_sta_get_ap_info()` and use it as the default CSI channel when
|
||||
no NVS override is set. This ensures the CSI collector always matches the
|
||||
connected AP's channel without requiring manual provisioning.
|
||||
|
||||
### MAC address filtering
|
||||
|
||||
- Add `--filter-mac` argument to `provision.py` that writes a `filter_mac`
|
||||
key (6-byte blob) to NVS.
|
||||
- In `nvs_config.h`, add a `filter_mac[6]` field and `filter_mac_set` flag.
|
||||
- In `nvs_config.c`, read the `filter_mac` blob from NVS.
|
||||
- In the CSI callback (`wifi_csi_callback`), if `filter_mac_set` is true,
|
||||
compare the source MAC from the received frame against the configured MAC
|
||||
and drop non-matching frames.
|
||||
|
||||
### Provisioning flow
|
||||
|
||||
```
|
||||
python provision.py --port COM7 --channel 11
|
||||
python provision.py --port COM7 --filter-mac "AA:BB:CC:DD:EE:FF"
|
||||
python provision.py --port COM7 --channel 11 --filter-mac "AA:BB:CC:DD:EE:FF"
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
- Users on managed networks can force the CSI channel to match their AP
|
||||
- Multi-AP environments can filter CSI to a single source
|
||||
- Auto-channel detection eliminates the most common misconfiguration
|
||||
- Backward compatible: existing provisioned nodes without these keys behave
|
||||
as before (use Kconfig default channel, accept all MACs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,199 @@
|
||||
# ADR-062: QEMU ESP32-S3 Swarm Configurator
|
||||
|
||||
| Field | Value |
|
||||
|-------------|------------------------------------------------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-03-14 |
|
||||
| **Authors** | RuView Team |
|
||||
| **Relates** | ADR-061 (QEMU testing platform), ADR-060 (channel/MAC filter), ADR-018 (binary frame), ADR-039 (edge intel) |
|
||||
|
||||
## Glossary
|
||||
|
||||
| Term | Definition |
|
||||
|------|-----------|
|
||||
| Swarm | A group of N QEMU ESP32-S3 instances running simultaneously |
|
||||
| Topology | How nodes are connected: star, mesh, line, ring |
|
||||
| Role | Node function: `sensor` (collects CSI), `coordinator` (aggregates + forwards), `gateway` (bridges to host) |
|
||||
| Scenario matrix | Cross-product of topology × node count × NVS config × mock scenario |
|
||||
| Health oracle | Python process that monitors all node UART logs and declares swarm health |
|
||||
|
||||
## Context
|
||||
|
||||
ADR-061 Layer 3 provides a basic multi-node mesh test: N identical nodes with sequential TDM slots connected via a Linux bridge. This is useful but limited:
|
||||
|
||||
1. **All nodes are identical** — real deployments have heterogeneous roles (sensor, coordinator, gateway)
|
||||
2. **Single topology** — only fully-connected bridge; no star, line, or ring topologies
|
||||
3. **No scenario variation per node** — all nodes run the same mock CSI scenario
|
||||
4. **Manual configuration** — each test requires hand-editing env vars and arguments
|
||||
5. **No swarm-level health monitoring** — validation checks individual nodes, not collective behavior
|
||||
6. **No cross-node timing validation** — TDM slot ordering and inter-frame gaps aren't verified
|
||||
|
||||
Real WiFi-DensePose deployments use 3-8 ESP32-S3 nodes in various topologies. A single coordinator aggregates CSI from multiple sensors. The firmware must handle TDM conflicts, missing nodes, role-based behavior differences, and network partitions — none of which ADR-061 Layer 3 tests.
|
||||
|
||||
## Decision
|
||||
|
||||
Build a **QEMU Swarm Configurator** — a YAML-driven tool that defines multi-node test scenarios declaratively and orchestrates them under QEMU with swarm-level validation.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ swarm_config.yaml │
|
||||
│ nodes: [{role: sensor, scenario: 2, channel: 6}] │
|
||||
│ topology: star │
|
||||
│ duration: 60s │
|
||||
│ assertions: [all_nodes_boot, tdm_no_collision, ...] │
|
||||
└──────────────────────┬──────────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────┐
|
||||
│ qemu_swarm.py │
|
||||
│ (orchestrator) │
|
||||
└───┬────┬────┬───┬──────┘
|
||||
│ │ │ │
|
||||
┌────▼┐ ┌▼──┐ ▼ ┌▼────┐
|
||||
│Node0│ │N1 │... │N(n-1)│ QEMU instances
|
||||
│sens │ │sen│ │coord │
|
||||
└──┬──┘ └─┬─┘ └──┬───┘
|
||||
│ │ │
|
||||
┌──▼──────▼─────────▼──┐
|
||||
│ Virtual Network │ TAP bridge / SLIRP
|
||||
│ (topology-shaped) │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
┌──────────▼───────────┐
|
||||
│ Aggregator (Rust) │ Collects frames
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
┌──────────▼───────────┐
|
||||
│ Health Oracle │ Swarm-level assertions
|
||||
│ (swarm_health.py) │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
### YAML Configuration Schema
|
||||
|
||||
```yaml
|
||||
# swarm_config.yaml
|
||||
swarm:
|
||||
name: "3-sensor-star"
|
||||
duration_s: 60
|
||||
topology: star # star | mesh | line | ring
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0 # empty room (baseline)
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true # receives aggregated frames
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 2 # walking person
|
||||
channel: 6
|
||||
tdm_slot: 1 # TDM slot index (auto-assigned from node position if omitted)
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 3 # fall event
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- coordinator_receives_from_all
|
||||
- fall_detected_by_node_2
|
||||
- frame_rate_above: 15 # Hz minimum per node
|
||||
- max_boot_time_s: 10
|
||||
```
|
||||
|
||||
### Topologies
|
||||
|
||||
| Topology | Network | Description |
|
||||
|----------|---------|-------------|
|
||||
| `star` | All sensors connect to coordinator; coordinator has TAP to each sensor | Hub-and-spoke, most common |
|
||||
| `mesh` | All nodes on same bridge (existing Layer 3 behavior) | Every node sees every other |
|
||||
| `line` | Node 0 ↔ Node 1 ↔ Node 2 ↔ ... | Linear chain, tests multi-hop |
|
||||
| `ring` | Like line but last connects to first | Circular, tests routing |
|
||||
|
||||
### Node Roles
|
||||
|
||||
| Role | Behavior | NVS Keys |
|
||||
|------|----------|----------|
|
||||
| `sensor` | Runs mock CSI, sends frames to coordinator | `node_id`, `tdm_slot`, `target_ip` |
|
||||
| `coordinator` | Receives frames from sensors, runs edge aggregation | `node_id`, `tdm_slot=0`, `edge_tier=2` |
|
||||
| `gateway` | Like coordinator but also bridges to host UDP | `node_id`, `target_ip=host`, `is_gateway=1` |
|
||||
|
||||
### Assertions (Swarm-Level)
|
||||
|
||||
| Assertion | What It Checks |
|
||||
|-----------|---------------|
|
||||
| `all_nodes_boot` | Every node's UART log shows boot indicators within timeout |
|
||||
| `no_crashes` | No Guru Meditation, assert, panic in any log |
|
||||
| `tdm_no_collision` | No two nodes transmit in the same TDM slot |
|
||||
| `all_nodes_produce_frames` | Every sensor node's log contains CSI frame output |
|
||||
| `coordinator_receives_from_all` | Coordinator log shows frames from each sensor's node_id |
|
||||
| `fall_detected_by_node_N` | Node N's log reports a fall detection event |
|
||||
| `frame_rate_above` | Each node produces at least N frames/second |
|
||||
| `max_boot_time_s` | All nodes boot within N seconds |
|
||||
| `no_heap_errors` | No OOM or heap corruption in any log |
|
||||
| `network_partitioned_recovery` | After deliberate partition, nodes resume communication (future) |
|
||||
|
||||
### Preset Configurations
|
||||
|
||||
| Preset | Nodes | Topology | Purpose |
|
||||
|--------|-------|----------|---------|
|
||||
| `smoke` | 2 | star | Quick CI smoke test (15s) |
|
||||
| `standard` | 3 | star | Default 3-node (sensor + sensor + coordinator) |
|
||||
| `large-mesh` | 6 | mesh | Scale test with 6 fully-connected nodes |
|
||||
| `line-relay` | 4 | line | Multi-hop relay chain |
|
||||
| `ring-fault` | 4 | ring | Ring with fault injection mid-test |
|
||||
| `heterogeneous` | 5 | star | Mixed scenarios: walk, fall, static, channel-sweep, empty |
|
||||
| `ci-matrix` | 3 | star | CI-optimized preset (30s, minimal assertions) |
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
scripts/
|
||||
├── qemu_swarm.py # Main orchestrator (CLI entry point)
|
||||
├── swarm_health.py # Swarm-level health oracle
|
||||
└── swarm_presets/
|
||||
├── smoke.yaml
|
||||
├── standard.yaml
|
||||
├── large_mesh.yaml
|
||||
├── line_relay.yaml
|
||||
├── ring_fault.yaml
|
||||
├── heterogeneous.yaml
|
||||
└── ci_matrix.yaml
|
||||
|
||||
.github/workflows/
|
||||
└── firmware-qemu.yml # MODIFIED: add swarm test job
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Benefits
|
||||
|
||||
1. **Declarative testing** — define swarm topology in YAML, not shell scripts
|
||||
2. **Role-based nodes** — test coordinator/sensor/gateway interactions
|
||||
3. **Topology variety** — star/mesh/line/ring match real deployment patterns
|
||||
4. **Swarm-level assertions** — validate collective behavior, not just individual nodes
|
||||
5. **Preset library** — quick CI smoke tests and thorough manual validation
|
||||
6. **Reproducible** — YAML configs are version-controlled and shareable
|
||||
|
||||
### Limitations
|
||||
|
||||
1. **Still requires root** for TAP bridge topologies (star, line, ring); mesh can use SLIRP
|
||||
2. **QEMU resource usage** — 6+ QEMU instances use ~2GB RAM, may slow CI runners
|
||||
3. **No real RF** — inter-node communication is IP-based, not WiFi CSI multipath
|
||||
|
||||
## References
|
||||
|
||||
- ADR-061: QEMU ESP32-S3 firmware testing platform (Layers 1-9)
|
||||
- ADR-060: Channel override and MAC address filter provisioning
|
||||
- ADR-018: Binary CSI frame format (magic `0xC5110001`)
|
||||
- ADR-039: Edge intelligence pipeline (biquad, vitals, fall detection)
|
||||
@@ -0,0 +1,261 @@
|
||||
# ADR-063: 60 GHz mmWave Sensor Fusion with WiFi CSI
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-03-15
|
||||
**Deciders:** @ruvnet
|
||||
**Related:** ADR-014 (SOTA signal processing), ADR-021 (vital sign extraction), ADR-029 (RuvSense multistatic), ADR-039 (edge intelligence), ADR-042 (CHCI coherent sensing)
|
||||
|
||||
## Context
|
||||
|
||||
RuView currently senses the environment using WiFi CSI — a passive technique that analyzes how WiFi signals are disturbed by human presence and movement. While this works through walls and requires no line of sight, CSI-derived vital signs (breathing rate, heart rate) are inherently noisy because they rely on phase extraction from multipath-rich WiFi channels.
|
||||
|
||||
A complementary sensing modality exists: **60 GHz mmWave radar** modules (e.g., Seeed MR60BHA2) that use active FMCW radar at 60 GHz to measure breathing and heart rate with clinical-grade accuracy. These modules are inexpensive (~$15), run on ESP32-C6/C3, and output structured vital signs over UART.
|
||||
|
||||
**Live hardware capture (COM4, 2026-03-15)** from a Seeed MR60BHA2 on an ESP32-C6 running ESPHome:
|
||||
|
||||
```
|
||||
[D][sensor:093]: 'Real-time respiratory rate': Sending state 22.00000
|
||||
[D][sensor:093]: 'Real-time heart rate': Sending state 92.00000 bpm
|
||||
[D][sensor:093]: 'Distance to detection object': Sending state 0.00000 cm
|
||||
[D][sensor:093]: 'Target Number': Sending state 0.00000
|
||||
[D][binary_sensor:036]: 'Person Information': Sending state OFF
|
||||
[D][sensor:093]: 'Seeed MR60BHA2 Illuminance': Sending state 0.67913 lx
|
||||
```
|
||||
|
||||
### The Opportunity
|
||||
|
||||
Fusing WiFi CSI with mmWave radar creates a sensor system that is greater than the sum of its parts:
|
||||
|
||||
| Capability | WiFi CSI Alone | mmWave Alone | Fused |
|
||||
|-----------|---------------|-------------|-------|
|
||||
| Through-wall sensing | Yes (5m+) | No (LoS only, ~3m) | Yes — CSI for room-scale, mmWave for precision |
|
||||
| Heart rate accuracy | ±5-10 BPM | ±1-2 BPM | ±1-2 BPM (mmWave primary, CSI cross-validates) |
|
||||
| Breathing accuracy | ±2-3 BPM | ±0.5 BPM | ±0.5 BPM |
|
||||
| Presence detection | Good (adaptive threshold) | Excellent (range-gated) | Excellent + through-wall |
|
||||
| Multi-person | Via subcarrier clustering | Via range-Doppler bins | Combined spatial + RF resolution |
|
||||
| Fall detection | Phase acceleration | Range/velocity + micro-Doppler | Dual-confirm reduces false positives to near-zero |
|
||||
| Pose estimation | Via trained model | Not available | CSI provides pose; mmWave provides ground-truth vitals for training |
|
||||
| Coverage | Whole room (passive) | ~120° cone, 3m range | Full room + precision zone |
|
||||
| Cost per node | ~$9 (ESP32-S3) | ~$15 (ESP32-C6 + MR60BHA2) | ~$24 combined |
|
||||
|
||||
### RuVector Integration Points
|
||||
|
||||
The RuVector v2.0.4 stack (already integrated per ADR-016) provides the signal processing backbone:
|
||||
|
||||
| RuVector Component | Role in mmWave Fusion |
|
||||
|-------------------|----------------------|
|
||||
| `ruvector-attention` (`bvp.rs`) | Blood Volume Pulse estimation — mmWave heart rate can calibrate the WiFi CSI BVP phase extraction |
|
||||
| `ruvector-temporal-tensor` (`breathing.rs`) | Breathing rate estimation — mmWave provides ground-truth for adaptive filter tuning |
|
||||
| `ruvector-solver` (`triangulation.rs`) | Multilateration — mmWave range-gated distance + CSI amplitude = 3D position |
|
||||
| `ruvector-attn-mincut` (`spectrogram.rs`) | Time-frequency decomposition — mmWave Doppler complements CSI phase spectrogram |
|
||||
| `ruvector-mincut` (`metrics.rs`, DynamicPersonMatcher) | Multi-person association — mmWave target IDs help disambiguate CSI subcarrier clusters |
|
||||
|
||||
### RuvSense Integration Points
|
||||
|
||||
The RuvSense multistatic sensing pipeline (ADR-029) gains new capabilities:
|
||||
|
||||
| RuvSense Module | mmWave Integration |
|
||||
|----------------|-------------------|
|
||||
| `pose_tracker.rs` (AETHER re-ID) | mmWave distance + velocity as additional re-ID features for Kalman tracker |
|
||||
| `longitudinal.rs` (Welford stats) | mmWave vitals as reference signal for CSI drift detection |
|
||||
| `intention.rs` (pre-movement) | mmWave micro-Doppler detects pre-movement 100-200ms earlier than CSI |
|
||||
| `adversarial.rs` (consistency check) | mmWave provides independent signal to detect CSI spoofing/anomalies |
|
||||
| `coherence_gate.rs` | mmWave presence as additional gate input — if mmWave says "no person", CSI coherence gate rejects |
|
||||
|
||||
### Cross-Viewpoint Fusion Integration
|
||||
|
||||
The viewpoint fusion pipeline (`ruvector/src/viewpoint/`) extends naturally:
|
||||
|
||||
| Viewpoint Module | mmWave Extension |
|
||||
|-----------------|-----------------|
|
||||
| `attention.rs` (CrossViewpointAttention) | mmWave range becomes a new "viewpoint" in the attention mechanism |
|
||||
| `geometry.rs` (GeometricDiversityIndex) | mmWave cone geometry contributes to Fisher Information / Cramer-Rao bounds |
|
||||
| `coherence.rs` (phase phasor) | mmWave phase coherence as validation for WiFi phasor coherence |
|
||||
| `fusion.rs` (MultistaticArray) | mmWave node becomes a member of the multistatic array with its own domain events |
|
||||
|
||||
## Decision
|
||||
|
||||
Add 60 GHz mmWave radar sensor support to the RuView firmware and sensing pipeline with auto-detection and device-specific capabilities.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Sensing Node │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │
|
||||
│ │ ESP32-S3 │ │ ESP32-C6 │ │ Combined │ │
|
||||
│ │ WiFi CSI │ │ + MR60BHA2 │ │ S3 + UART │ │
|
||||
│ │ (COM7) │ │ 60GHz mmWave │ │ mmWave │ │
|
||||
│ │ │ │ (COM4) │ │ │ │
|
||||
│ │ Passive │ │ Active radar │ │ Both modes │ │
|
||||
│ │ Through-wall │ │ LoS, precise │ │ │ │
|
||||
│ └──────┬───────┘ └──────┬───────┘ └─────┬──────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────┬───────────┘ │ │
|
||||
│ ▼ │ │
|
||||
│ ┌────────────────┐ │ │
|
||||
│ │ Fusion Engine │◄──────────────────────┘ │
|
||||
│ │ │ │
|
||||
│ │ • Kalman fuse │ Vitals packet (extended): │
|
||||
│ │ • Cross-validate│ magic 0xC5110004 │
|
||||
│ │ • Ground-truth │ + mmwave_hr, mmwave_br │
|
||||
│ │ calibration │ + mmwave_distance │
|
||||
│ │ • Fall confirm │ + mmwave_target_count │
|
||||
│ └────────────────┘ + confidence scores │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Three Deployment Modes
|
||||
|
||||
**Mode 1: Standalone CSI (existing)** — ESP32-S3 only, WiFi CSI sensing.
|
||||
|
||||
**Mode 2: Standalone mmWave** — ESP32-C6 + MR60BHA2, precise vitals in a single room.
|
||||
|
||||
**Mode 3: Fused (recommended)** — ESP32-S3 + mmWave module on UART, or two separate nodes with server-side fusion.
|
||||
|
||||
### Auto-Detection Protocol
|
||||
|
||||
The firmware will auto-detect connected mmWave modules at boot:
|
||||
|
||||
1. **UART probe** — On configured UART pins, send the MR60BHA2 identification command (`0x01 0x01 0x00 0x01 ...`) and check for valid response header
|
||||
2. **Protocol detection** — Identify the sensor family:
|
||||
- Seeed MR60BHA2 (breathing + heart rate)
|
||||
- Seeed MR60FDA1 (fall detection)
|
||||
- Seeed MR24HPC1 (presence + light sleep/deep sleep)
|
||||
- HLK-LD2410 (presence + distance)
|
||||
- HLK-LD2450 (multi-target tracking)
|
||||
3. **Capability registration** — Register detected sensor capabilities in the edge config:
|
||||
|
||||
```c
|
||||
typedef struct {
|
||||
uint8_t mmwave_detected; /** 1 if mmWave module found on UART */
|
||||
uint8_t mmwave_type; /** Sensor family (MR60BHA2, MR60FDA1, etc.) */
|
||||
uint8_t mmwave_has_hr; /** Heart rate capability */
|
||||
uint8_t mmwave_has_br; /** Breathing rate capability */
|
||||
uint8_t mmwave_has_fall; /** Fall detection capability */
|
||||
uint8_t mmwave_has_presence; /** Presence detection capability */
|
||||
uint8_t mmwave_has_distance; /** Range measurement capability */
|
||||
uint8_t mmwave_has_tracking; /** Multi-target tracking capability */
|
||||
float mmwave_hr_bpm; /** Latest heart rate from mmWave */
|
||||
float mmwave_br_bpm; /** Latest breathing rate from mmWave */
|
||||
float mmwave_distance_cm; /** Distance to nearest target */
|
||||
uint8_t mmwave_target_count; /** Number of detected targets */
|
||||
bool mmwave_person_present;/** mmWave presence state */
|
||||
} mmwave_state_t;
|
||||
```
|
||||
|
||||
### Supported Sensors
|
||||
|
||||
| Sensor | Frequency | Capabilities | UART Protocol | Cost |
|
||||
|--------|-----------|-------------|---------------|------|
|
||||
| **Seeed MR60BHA2** | 60 GHz | HR, BR, presence, illuminance | Seeed proprietary frames | ~$15 |
|
||||
| **Seeed MR60FDA1** | 60 GHz | Fall detection, presence | Seeed proprietary frames | ~$15 |
|
||||
| **Seeed MR24HPC1** | 24 GHz | Presence, sleep stage, distance | Seeed proprietary frames | ~$10 |
|
||||
| **HLK-LD2410** | 24 GHz | Presence, distance (motion + static) | HLK binary protocol | ~$3 |
|
||||
| **HLK-LD2450** | 24 GHz | Multi-target tracking (x,y,speed) | HLK binary protocol | ~$5 |
|
||||
|
||||
### Fusion Algorithms
|
||||
|
||||
**1. Vital Sign Fusion (Kalman filter)**
|
||||
```
|
||||
mmWave HR (high confidence, 1 Hz) ─┐
|
||||
├─► Kalman fuse → fused HR ± confidence
|
||||
CSI-derived HR (lower confidence) ─┘
|
||||
```
|
||||
|
||||
**2. Fall Detection (dual-confirm)**
|
||||
```
|
||||
CSI phase accel > thresh ──────┐
|
||||
├─► AND gate → confirmed fall (near-zero false positives)
|
||||
mmWave range-velocity pattern ─┘
|
||||
```
|
||||
|
||||
**3. Presence Validation**
|
||||
```
|
||||
CSI adaptive threshold ────┐
|
||||
├─► Weighted vote → robust presence
|
||||
mmWave target count > 0 ──┘
|
||||
```
|
||||
|
||||
**4. Training Calibration**
|
||||
```
|
||||
mmWave ground-truth vitals → train CSI BVP extraction model
|
||||
mmWave distance → calibrate CSI triangulation
|
||||
mmWave micro-Doppler → label CSI activity patterns
|
||||
```
|
||||
|
||||
### Vitals Packet Extension
|
||||
|
||||
Extend the existing 32-byte vitals packet (magic `0xC5110002`) with a new 48-byte fused packet:
|
||||
|
||||
```c
|
||||
typedef struct __attribute__((packed)) {
|
||||
/* Existing 32-byte vitals fields */
|
||||
uint32_t magic; /* 0xC5110004 (fused vitals) */
|
||||
uint8_t node_id;
|
||||
uint8_t flags; /* Bit0=presence, Bit1=fall, Bit2=motion, Bit3=mmwave_present */
|
||||
uint16_t breathing_rate; /* Fused BPM * 100 */
|
||||
uint32_t heartrate; /* Fused BPM * 10000 */
|
||||
int8_t rssi;
|
||||
uint8_t n_persons;
|
||||
uint8_t mmwave_type; /* Sensor type enum */
|
||||
uint8_t fusion_confidence;/* 0-100 fusion quality score */
|
||||
float motion_energy;
|
||||
float presence_score;
|
||||
uint32_t timestamp_ms;
|
||||
/* New mmWave fields (16 bytes) */
|
||||
float mmwave_hr_bpm; /* Raw mmWave heart rate */
|
||||
float mmwave_br_bpm; /* Raw mmWave breathing rate */
|
||||
float mmwave_distance; /* Distance to nearest target (cm) */
|
||||
uint8_t mmwave_targets; /* Target count */
|
||||
uint8_t mmwave_confidence;/* mmWave signal quality 0-100 */
|
||||
uint16_t reserved;
|
||||
} edge_fused_vitals_pkt_t;
|
||||
|
||||
_Static_assert(sizeof(edge_fused_vitals_pkt_t) == 48, "fused vitals must be 48 bytes");
|
||||
```
|
||||
|
||||
### NVS Configuration
|
||||
|
||||
New provisioning parameters:
|
||||
|
||||
```bash
|
||||
python provision.py --port COM7 \
|
||||
--mmwave-uart-tx 17 --mmwave-uart-rx 18 \ # UART pins for mmWave module
|
||||
--mmwave-type auto \ # auto-detect, or: mr60bha2, ld2410, etc.
|
||||
--fusion-mode kalman \ # kalman, vote, mmwave-primary, csi-primary
|
||||
--fall-dual-confirm true # require both CSI + mmWave for fall alert
|
||||
```
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
| Phase | Scope | Effort |
|
||||
|-------|-------|--------|
|
||||
| **Phase 1** | UART driver + MR60BHA2 parser + auto-detection | 2 weeks |
|
||||
| **Phase 2** | Fused vitals packet + Kalman vital sign fusion | 1 week |
|
||||
| **Phase 3** | Dual-confirm fall detection + presence voting | 1 week |
|
||||
| **Phase 4** | HLK-LD2410/LD2450 support + multi-target fusion | 2 weeks |
|
||||
| **Phase 5** | RuVector calibration pipeline (mmWave as ground truth) | 3 weeks |
|
||||
| **Phase 6** | Server-side fusion for separate CSI + mmWave nodes | 2 weeks |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Near-zero false positive fall detection (dual-confirm)
|
||||
- Clinical-grade vital signs when mmWave is present, with CSI as fallback
|
||||
- Self-calibrating CSI pipeline using mmWave ground truth
|
||||
- Backward compatible — existing CSI-only nodes work unchanged
|
||||
- Low incremental cost (~$3-15 per mmWave module)
|
||||
- Auto-detection means zero configuration for supported sensors
|
||||
- RuVector attention/solver/temporal-tensor modules gain a high-quality reference signal
|
||||
|
||||
### Negative
|
||||
- Added firmware complexity (~2-3 KB RAM for mmWave state + UART buffer)
|
||||
- mmWave modules require line-of-sight (complementary to CSI, not replacement)
|
||||
- Multiple UART protocols to maintain (Seeed, HLK families)
|
||||
- 48-byte fused packet requires server parser update
|
||||
|
||||
### Neutral
|
||||
- ESP32-C6 cannot run the full CSI pipeline (single-core RISC-V) but can serve as a dedicated mmWave bridge node
|
||||
- mmWave modules add ~15 mA power draw per node
|
||||
@@ -0,0 +1,327 @@
|
||||
# ADR-064: Multimodal Ambient Intelligence — WiFi CSI + mmWave + Environmental Sensors
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-03-15
|
||||
**Deciders:** @ruvnet
|
||||
**Related:** ADR-063 (mmWave fusion), ADR-039 (edge intelligence), ADR-042 (CHCI), ADR-029 (RuvSense multistatic), ADR-024 (AETHER contrastive embeddings)
|
||||
|
||||
## Context
|
||||
|
||||
With ADR-063 we demonstrated real-time fusion of WiFi CSI (ESP32-S3, COM7) and 60 GHz mmWave radar (Seeed MR60BHA2 on ESP32-C6, COM4). The live capture showed:
|
||||
|
||||
- **mmWave**: HR 75 bpm, BR 25/min, presence at 52 cm, 1.4 Hz update
|
||||
- **WiFi CSI**: Channel 5, RSSI -41, 20+ Hz frame rate, through-wall coverage
|
||||
- **BH1750**: Ambient light 0.0-0.7 lux (room darkness level)
|
||||
|
||||
This ADR explores the full spectrum of what becomes possible when these modalities are combined — from immediately practical applications to speculative research directions.
|
||||
|
||||
---
|
||||
|
||||
## Tier 1: Practical (Build Now)
|
||||
|
||||
### 1.1 Intelligent Fall Detection with Zero False Positives
|
||||
|
||||
**Current state:** CSI-only fall detection with 15.0 rad/s² threshold (v0.4.3.1).
|
||||
**With fusion:** mmWave confirms fall via range-velocity signature (sudden height drop + impact deceleration). CSI provides the alert; mmWave provides the confirmation.
|
||||
|
||||
```
|
||||
CSI phase acceleration > 15 rad/s² ─┐
|
||||
├─► AND gate + temporal correlation
|
||||
mmWave: height drop > 50cm in <1s ──┘ → CONFIRMED FALL (call 911)
|
||||
```
|
||||
|
||||
**Impact:** Elderly care facilities spend $34B/year on fall injuries. A $24 sensor node with zero false positives replaces $200/month medical alert wearables that residents forget to wear.
|
||||
|
||||
### 1.2 Sleep Quality Monitoring
|
||||
|
||||
**Sensors used:** mmWave (BR/HR), CSI (bed occupancy, movement), BH1750 (light)
|
||||
|
||||
| Metric | Source | Method |
|
||||
|--------|--------|--------|
|
||||
| Sleep onset | CSI motion → still transition | Phase variance drops below threshold |
|
||||
| Sleep stages | mmWave BR variability | BR 12-20 = light sleep, 6-12 = deep sleep |
|
||||
| REM detection | mmWave HR variability | HR variability increases during REM |
|
||||
| Restlessness | CSI motion energy | Counts of motion episodes per hour |
|
||||
| Room darkness | BH1750 | Correlate light exposure with sleep latency |
|
||||
| Wake events | CSI + mmWave | Motion + HR spike = awakening |
|
||||
|
||||
**Output:** Sleep score (0-100), time in each stage, disturbance log.
|
||||
**No wearable required.** Works through a mattress.
|
||||
|
||||
### 1.3 Occupancy-Aware HVAC and Lighting
|
||||
|
||||
**Sensors:** CSI (room-level presence through walls), mmWave (precise count + distance), BH1750 (ambient light)
|
||||
|
||||
- CSI detects which rooms are occupied (through walls, whole-floor sensing)
|
||||
- mmWave counts exact number of people in the sensor's room
|
||||
- BH1750 measures if lights are on/needed
|
||||
- System sends MQTT/UDP commands to smart home controllers
|
||||
|
||||
**Energy savings:** 20-40% HVAC reduction by not heating/cooling empty rooms.
|
||||
|
||||
### 1.4 Bathroom Safety for Elderly
|
||||
|
||||
**Sensor placement:** One CSI node outside bathroom (through-wall), one mmWave inside.
|
||||
|
||||
- CSI detects person entered bathroom (through-wall)
|
||||
- mmWave monitors vitals while showering (waterproof enclosure)
|
||||
- If no movement for > N minutes AND HR drops: alert
|
||||
- Fall detection in shower (slippery surface = high risk)
|
||||
|
||||
### 1.5 Baby/Infant Breathing Monitor
|
||||
|
||||
**mmWave at crib-side:** Contactless breathing monitoring at 0.5-1m range.
|
||||
- BR < 10 or BR = 0 for > 20s: alarm (apnea detection)
|
||||
- CSI provides room context (parent present? other motion?)
|
||||
- BH1750 tracks night feeding times (light on/off events)
|
||||
|
||||
---
|
||||
|
||||
## Tier 2: Advanced (Research Prototype)
|
||||
|
||||
### 2.1 Gait Analysis and Fall Risk Prediction
|
||||
|
||||
**Method:** CSI tracks walking pattern across the room; mmWave measures stride length and velocity.
|
||||
|
||||
| Feature | Source | Clinical Use |
|
||||
|---------|--------|-------------|
|
||||
| Gait velocity | mmWave Doppler | < 0.8 m/s = fall risk indicator |
|
||||
| Stride variability | CSI phase patterns | High variability = cognitive decline marker |
|
||||
| Turning stability | CSI + mmWave | Difficulty turning = Parkinson's indicator |
|
||||
| Get-up time | mmWave (sit→stand) | Timed Up and Go (TUG) test, contactless |
|
||||
|
||||
**Clinical value:** Gait velocity is called the "sixth vital sign" — it predicts hospitalization, cognitive decline, and mortality. Currently requires a $10,000 GAITRite mat. A $24 sensor node replaces it.
|
||||
|
||||
### 2.2 Emotion and Stress Detection via Micro-Vitals
|
||||
|
||||
**mmWave at desk:** Continuous HR variability (HRV) monitoring during work.
|
||||
|
||||
- **HRV time-domain:** SDNN, RMSSD from beat-to-beat intervals
|
||||
- **HRV frequency-domain:** LF/HF ratio (sympathetic/parasympathetic balance)
|
||||
- Low HF power = stress; high HF = relaxation
|
||||
- CSI detects fidgeting, posture shifts (correlated with stress)
|
||||
- BH1750 correlates lighting with mood/productivity
|
||||
|
||||
**Application:** Smart office that adjusts lighting, temperature, and notification frequency based on detected stress level.
|
||||
|
||||
### 2.3 Gesture Recognition as Room Control
|
||||
|
||||
**CSI:** Already has DTW template matching gesture classifier (`ruvsense/gesture.rs`).
|
||||
**mmWave:** Adds range-Doppler micro-gesture detection (hand wave, swipe, circle).
|
||||
|
||||
- CSI recognizes gross gestures (wave arm, walk pattern)
|
||||
- mmWave recognizes fine hand gestures (swipe left/right, push/pull)
|
||||
- Fused: spatial context (CSI knows where you are) + precise gesture (mmWave knows what your hand did)
|
||||
|
||||
**Use case:** Wave at the sensor to turn off lights. Swipe to change music. No voice assistant, no camera, no wearable.
|
||||
|
||||
### 2.4 Respiratory Disease Screening
|
||||
|
||||
**mmWave BR patterns over days/weeks:**
|
||||
|
||||
| Pattern | Indicator |
|
||||
|---------|-----------|
|
||||
| BR > 20 at rest, trending up | Possible pneumonia/COVID |
|
||||
| Periodic breathing (Cheyne-Stokes) | Heart failure |
|
||||
| Obstructive apnea pattern | Sleep apnea (> 5 events/hour) |
|
||||
| BR variability decrease | COPD exacerbation |
|
||||
|
||||
**CSI adds:** Cough detection (sudden phase disturbance pattern), movement reduction (malaise indicator).
|
||||
|
||||
**Longitudinal tracking** via `ruvsense/longitudinal.rs` (Welford stats, biomechanics drift detection) — the system learns your normal breathing pattern and alerts on deviations.
|
||||
|
||||
### 2.5 Multi-Room Activity Recognition
|
||||
|
||||
**3-6 CSI nodes (through walls) + 1-2 mmWave (key rooms):**
|
||||
|
||||
```
|
||||
Kitchen (CSI): person detected, high motion → cooking
|
||||
Living room (mmWave + CSI): 2 people, low motion, HR stable → watching TV
|
||||
Bedroom (CSI): person detected, minimal motion → sleeping
|
||||
Bathroom (CSI): person entered 3 min ago, still inside → OK
|
||||
Front door (CSI): motion pattern = leaving/arriving
|
||||
```
|
||||
|
||||
**Output:** Activity timeline, daily routine deviation alerts, loneliness detection (no visitors in N days).
|
||||
|
||||
---
|
||||
|
||||
## Tier 3: Speculative (Research Frontier)
|
||||
|
||||
### 3.1 Cardiac Arrhythmia Detection
|
||||
|
||||
**mmWave at < 1m range:** Beat-to-beat interval extraction from chest wall displacement.
|
||||
|
||||
- Atrial fibrillation: irregular R-R intervals (coefficient of variation > 0.1)
|
||||
- Bradycardia/tachycardia: sustained HR < 60 or > 100
|
||||
- Premature ventricular contractions: occasional short-long-short patterns
|
||||
|
||||
**Challenge:** Requires sub-millimeter displacement resolution. The MR60BHA2 may lack the SNR for single-beat extraction, but clinical-grade 60 GHz modules (Infineon BGT60TR13C) can achieve this.
|
||||
|
||||
**CSI role:** Validates that the person is stationary (motion corrupts beat-to-beat analysis).
|
||||
|
||||
### 3.2 Blood Pressure Estimation (Contactless)
|
||||
|
||||
**Theory:** Pulse Transit Time (PTT) between two body points correlates with blood pressure. With two mmWave sensors at different body positions, PTT can be estimated from the phase difference of reflected chest/wrist signals.
|
||||
|
||||
**Feasibility:** Academic papers demonstrate ±10 mmHg accuracy in controlled settings. Far from clinical grade but useful for trending.
|
||||
|
||||
### 3.3 RF Tomography — 3D Occupancy Imaging
|
||||
|
||||
**Method:** Multiple CSI nodes form a tomographic array. Each TX-RX pair measures signal attenuation. Inverse problem (ISTA L1 solver, already in `ruvsense/tomography.rs`) reconstructs a 3D voxel grid of where absorbers (people) are.
|
||||
|
||||
**mmWave adds:** Range-gated targets as sparse priors for the tomographic reconstruction, dramatically reducing the ill-posedness of the inverse problem.
|
||||
|
||||
```
|
||||
CSI tomography (coarse 3D grid, 50cm resolution) ─┐
|
||||
├─► Sparse fusion
|
||||
mmWave targets (precise range, cm resolution) ─────┘ → 10cm 3D occupancy map
|
||||
```
|
||||
|
||||
### 3.4 Sign Language Recognition
|
||||
|
||||
**CSI phase patterns (body/arm movement) + mmWave Doppler (hand micro-movements):**
|
||||
|
||||
- CSI captures the gross arm trajectory of each sign
|
||||
- mmWave captures the finger configuration at the pause point
|
||||
- AETHER contrastive embeddings (`ADR-024`) learn to map (CSI phase sequence, mmWave Doppler) → sign label
|
||||
- No camera required — works in the dark, preserves privacy
|
||||
|
||||
**Training data:** Record CSI + mmWave while performing signs with a camera as ground truth, then deploy camera-free.
|
||||
|
||||
### 3.5 Cognitive Load Estimation
|
||||
|
||||
**Multimodal features:**
|
||||
|
||||
| Feature | Source | Cognitive Load Indicator |
|
||||
|---------|--------|------------------------|
|
||||
| HR increase | mmWave | Sympathetic activation |
|
||||
| BR irregularity | mmWave | Cognitive interference |
|
||||
| Posture stiffness | CSI motion variance | Reduced when concentrating |
|
||||
| Fidgeting frequency | CSI high-freq motion | Increases with frustration |
|
||||
| Micro-saccade proxy | mmWave head micro-movement | Correlated with attention |
|
||||
|
||||
**Application:** Adaptive learning systems that slow down when the student is overloaded. Smart meeting rooms that detect when participants are disengaged.
|
||||
|
||||
### 3.6 Drone/Robot Navigation via RF Sensing
|
||||
|
||||
**CSI mesh as indoor GPS:** A network of CSI nodes creates a spatial RF fingerprint map. A robot or drone with an ESP32 can localize itself by matching its observed CSI to the map.
|
||||
|
||||
**mmWave on the robot:** Obstacle avoidance + human detection (don't collide with people).
|
||||
|
||||
**CSI from the environment:** Tells the robot where people are in adjacent rooms (through walls) so it can plan routes that avoid occupied spaces.
|
||||
|
||||
### 3.7 Building Structural Health Monitoring
|
||||
|
||||
**CSI multipath signature over months/years:**
|
||||
|
||||
- The CSI channel response is a fingerprint of the room's geometry
|
||||
- Subtle shifts in multipath (wall crack propagation, foundation settlement) change the CSI signature
|
||||
- `ruvsense/cross_room.rs` (environment fingerprinting) tracks these long-term drifts
|
||||
- mmWave detects surface vibrations (micro-displacement from traffic, wind, seismic)
|
||||
|
||||
**Application:** Early warning for structural degradation in bridges, tunnels, old buildings.
|
||||
|
||||
### 3.8 Swarm Sensing — Emergent Spatial Awareness
|
||||
|
||||
**50+ nodes across a building:**
|
||||
|
||||
Each node runs local edge intelligence (ADR-039). The `hive-mind` consensus system (ADR-062) aggregates across nodes. Emergent behaviors:
|
||||
|
||||
- **Flow detection:** Track how people move between rooms over time
|
||||
- **Anomaly detection:** "This hallway usually has 5 people/hour but had 0 today"
|
||||
- **Emergency routing:** During fire, track which exits are blocked (no movement) vs available
|
||||
- **Crowd density:** Concert/stadium safety — detect dangerous compression zones through walls
|
||||
|
||||
---
|
||||
|
||||
## Tier 4: Exotic / Sci-Fi Adjacent
|
||||
|
||||
### 4.1 Emotion Contagion Mapping
|
||||
|
||||
If multiple people are in a room and the system can estimate individual HR/HRV (via multi-target mmWave + CSI subcarrier clustering), you can detect:
|
||||
|
||||
- Physiological synchrony (two people's HR converging = rapport/empathy)
|
||||
- Stress propagation (one person's stress → others' HR rises)
|
||||
- "Emotional temperature" of a room
|
||||
|
||||
### 4.2 Dream State Detection and Lucid Dream Induction
|
||||
|
||||
During REM sleep (detected via mmWave HR variability + CSI minimal body movement):
|
||||
|
||||
- Detect REM onset with high confidence
|
||||
- Trigger a subtle environmental cue (gentle light via smart bulb, barely audible tone)
|
||||
- The sleeper incorporates the cue into the dream, recognizing it as a dream trigger
|
||||
- BH1750 confirms room is dark (not a natural awakening)
|
||||
|
||||
Based on published lucid dreaming induction research (e.g., LaBerge's MILD technique with external cues).
|
||||
|
||||
### 4.3 Plant Growth Monitoring
|
||||
|
||||
WiFi signals pass through plant tissue differently based on water content.
|
||||
|
||||
- CSI amplitude through a greenhouse changes as plants absorb/release water
|
||||
- mmWave reflects off leaf surfaces — micro-displacement from growth
|
||||
- Long-term CSI drift correlates with biomass increase
|
||||
|
||||
Academic proof-of-concept: "Sensing Plant Water Content Using WiFi Signals" (2023).
|
||||
|
||||
### 4.4 Pet Behavior Analysis
|
||||
|
||||
- CSI detects pet movement patterns (different phase signature than humans — lower, faster)
|
||||
- mmWave detects breathing rate (pets have higher BR than humans)
|
||||
- System learns pet's daily routine and alerts on deviations (lethargy, pacing, not eating)
|
||||
|
||||
### 4.5 Paranormal Investigation Tool
|
||||
|
||||
(For the entertainment/hobbyist market)
|
||||
|
||||
- CSI detects "unexplained" signal disturbances in empty rooms
|
||||
- mmWave confirms no physical presence
|
||||
- System logs "anomalous RF events" with timestamps
|
||||
- Export as Ghost Hunting report
|
||||
|
||||
**Actual explanation:** Temperature changes, HVAC drafts, and EMI cause CSI fluctuations. But it would sell.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Priority Matrix
|
||||
|
||||
| Application | Sensors Needed | Effort | Value | Priority |
|
||||
|------------|---------------|--------|-------|----------|
|
||||
| Fall detection (zero false positive) | CSI + mmWave | 1 week | Critical (healthcare) | **P0** |
|
||||
| Sleep monitoring | mmWave + BH1750 | 2 weeks | High (wellness) | **P1** |
|
||||
| Occupancy HVAC/lighting | CSI + mmWave | 1 week | High (energy) | **P1** |
|
||||
| Baby breathing monitor | mmWave | 1 week | Critical (safety) | **P1** |
|
||||
| Bathroom safety | CSI + mmWave | 1 week | Critical (elderly) | **P1** |
|
||||
| Gait analysis | CSI + mmWave | 3 weeks | High (clinical) | **P2** |
|
||||
| Gesture control | CSI + mmWave | 4 weeks | Medium (UX) | **P2** |
|
||||
| Multi-room activity | CSI mesh + mmWave | 4 weeks | High (elder care) | **P2** |
|
||||
| Respiratory screening | mmWave longitudinal | 6 weeks | High (health) | **P2** |
|
||||
| Stress/emotion detection | mmWave HRV + CSI | 6 weeks | Medium (wellness) | **P3** |
|
||||
| RF tomography | CSI mesh + mmWave | 8 weeks | Medium (research) | **P3** |
|
||||
| Sign language | CSI + mmWave + ML | 12 weeks | Medium (accessibility) | **P3** |
|
||||
| Cardiac arrhythmia | High-res mmWave | 12 weeks | High (clinical) | **P3** |
|
||||
| Swarm sensing | 50+ nodes | 16 weeks | High (safety) | **P3** |
|
||||
|
||||
## Decision
|
||||
|
||||
Document these possibilities as the product roadmap for the RuView multimodal ambient intelligence platform. Prioritize P0-P1 items (fall detection, sleep, occupancy, baby monitor, bathroom safety) for immediate implementation using the existing hardware (ESP32-S3 + MR60BHA2 + BH1750).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Positions RuView as a platform, not just a WiFi sensing demo
|
||||
- Each application can ship as a WASM edge module (ADR-040), deployable to existing hardware
|
||||
- Healthcare applications have clear regulatory paths (fall detection is FDA Class I exempt)
|
||||
- Most P0-P1 applications require no additional hardware beyond what's already deployed
|
||||
|
||||
### Negative
|
||||
- Clinical applications (arrhythmia, blood pressure) require medical device validation
|
||||
- Privacy concerns scale with capability — need clear data retention policies
|
||||
- Some exotic applications may attract scrutiny (surveillance concerns)
|
||||
|
||||
### Risk Mitigation
|
||||
- All processing happens on-device (edge) — no cloud, no recordings by default
|
||||
- No cameras — signal-based sensing preserves visual privacy
|
||||
- Open source — users can audit exactly what is sensed and transmitted
|
||||
@@ -0,0 +1,234 @@
|
||||
# ADR-065: Hotel Guest Happiness Scoring -- WiFi CSI + Cognitum Seed Bridge
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-03-20
|
||||
**Deciders:** @ruvnet
|
||||
**Related:** ADR-040 (WASM edge modules), ADR-039 (edge intelligence), ADR-042 (CHCI), ADR-064 (multimodal ambient intelligence), ADR-060 (multi-node aggregation)
|
||||
|
||||
## Context
|
||||
|
||||
Hotels lack objective, privacy-preserving methods to measure guest satisfaction in real time. Current approaches (post-stay surveys, NPS scores) are delayed, biased toward extremes, and capture less than 10% of guests. Meanwhile, ambient RF sensing can infer behavioral cues that correlate with comfort and well-being -- without cameras, wearables, or any guest interaction.
|
||||
|
||||
### Hardware
|
||||
|
||||
Two ESP32-S3 variants are deployed:
|
||||
|
||||
| Device | Flash | PSRAM | MAC | Port | Notes |
|
||||
|--------|-------|-------|-----|------|-------|
|
||||
| ESP32-S3 (QFN56 rev 0.2) | 4 MB | 2 MB | 1C:DB:D4:83:D2:40 | COM5 | Budget node, uses `sdkconfig.defaults.4mb` + `partitions_4mb.csv` |
|
||||
| ESP32-S3 | 8 MB | 8 MB | -- | COM7 | Full-featured node, existing deployment |
|
||||
|
||||
Both run the Tier 2 DSP firmware with presence detection, vitals extraction, fall detection, and gait analysis.
|
||||
|
||||
### Cognitum Seed Device
|
||||
|
||||
A Cognitum Seed unit is deployed on the same network segment:
|
||||
|
||||
- **Address:** 169.254.42.1 (link-local)
|
||||
- **Hardware:** Raspberry Pi Zero 2 W
|
||||
- **Firmware:** 0.7.0
|
||||
- **Vector store:** 398 vectors, dim=8
|
||||
- **API endpoints:** 98 (REST, fully documented)
|
||||
- **Sensors:** PIR, reed switch (door), vibration, ADS1115 ADC (4-ch analog), BME280 (temp/humidity/pressure)
|
||||
- **Security:** Ed25519 custody chain with tamper-evident witness log
|
||||
|
||||
The Seed's 8-dimensional vector store and drift detection engine make it a natural aggregation point for behavioral feature vectors extracted from CSI data.
|
||||
|
||||
### Existing WASM Edge Modules
|
||||
|
||||
The following modules already run on-device and produce features relevant to happiness scoring:
|
||||
|
||||
| Module | Event IDs | Outputs |
|
||||
|--------|-----------|---------|
|
||||
| `exo_emotion_detect.rs` | 610-613 | Arousal level, stress index |
|
||||
| `med_gait_analysis.rs` | 130-134 | Cadence, stride length, regularity |
|
||||
| `ret_customer_flow.rs` | 410-413 | Entry/exit count, direction |
|
||||
| `ret_dwell_heatmap.rs` | 420-423 | Dwell time per zone |
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. New WASM Module: `exo_happiness_score.rs`
|
||||
|
||||
Create a new WASM edge module that fuses outputs from existing modules into an 8-dimensional happiness vector, matching the Seed's vector dimensionality (dim=8).
|
||||
|
||||
**Event ID registry (690-694):**
|
||||
|
||||
| Event ID | Name | Description |
|
||||
|----------|------|-------------|
|
||||
| 690 | `HAPPINESS_VECTOR` | Full 8-dim happiness vector emitted per scoring window |
|
||||
| 691 | `HAPPINESS_TREND` | Windowed trend (rising/falling/stable) over last N vectors |
|
||||
| 692 | `HAPPINESS_ALERT` | Score crossed a configured threshold (low satisfaction) |
|
||||
| 693 | `HAPPINESS_GROUP` | Aggregate score for multi-person zone |
|
||||
| 694 | `HAPPINESS_CALIBRATION` | Baseline recalibration event (new guest check-in) |
|
||||
|
||||
### 2. Happiness Vector Schema (8 Dimensions)
|
||||
|
||||
Each dimension is normalized to [0.0, 1.0] where 1.0 = maximal positive signal:
|
||||
|
||||
| Dim | Name | Source | Derivation |
|
||||
|-----|------|--------|------------|
|
||||
| 0 | `gait_speed` | `med_gait_analysis` (130) | Normalized walking velocity. Brisk = positive. |
|
||||
| 1 | `stride_regularity` | `med_gait_analysis` (131) | Low stride-to-stride variance = relaxed gait. |
|
||||
| 2 | `movement_fluidity` | CSI phase jerk (d3/dt3) | Low jerk = smooth, unhurried movement. |
|
||||
| 3 | `breathing_calm` | Vitals BR extraction | BR 12-18 at rest = calm. Deviation penalized. |
|
||||
| 4 | `posture_openness` | CSI subcarrier spread | Wide phase spread across subcarriers = open posture. |
|
||||
| 5 | `dwell_comfort` | `ret_dwell_heatmap` (420) | Moderate dwell in amenity zones = engagement. |
|
||||
| 6 | `direction_entropy` | `ret_customer_flow` (410) | Low entropy = purposeful movement. Wandering penalized. |
|
||||
| 7 | `group_energy` | Multi-target CSI clustering | Synchronized movement of 2+ people = social engagement. |
|
||||
|
||||
The composite scalar happiness score is the weighted L2 norm:
|
||||
|
||||
```
|
||||
score = sum(w[i] * v[i] for i in 0..7) / sum(w[i])
|
||||
```
|
||||
|
||||
Default weights are uniform (all 1.0), configurable via NVS or Seed API.
|
||||
|
||||
### 3. ESP32 to Seed Bridge
|
||||
|
||||
```
|
||||
ESP32-S3 (CSI) Cognitum Seed (169.254.42.1)
|
||||
+------------------+ +----------------------------+
|
||||
| Tier 2 DSP | | |
|
||||
| + WASM modules | UDP 5555 | /api/v1/store/ingest |
|
||||
| exo_happiness |──────────────| (POST, 8-dim vector) |
|
||||
| _score.rs | | |
|
||||
| | | /api/v1/drift/check |
|
||||
| |◄─────────────| (drift alerts via webhook) |
|
||||
| | | |
|
||||
| | | /api/v1/witness/append |
|
||||
| | | (Ed25519 audit trail) |
|
||||
+------------------+ +----------------------------+
|
||||
```
|
||||
|
||||
**Data flow:**
|
||||
|
||||
1. ESP32 runs CSI capture at 20+ Hz and feeds subcarrier data through existing WASM modules.
|
||||
2. `exo_happiness_score.rs` collects outputs from emotion, gait, flow, and dwell modules every scoring window (default: 30 seconds).
|
||||
3. The 8-dim happiness vector is packed as a 32-byte payload (8x float32) and sent via UDP to port 5555 on 169.254.42.1.
|
||||
4. A lightweight bridge task on the Seed receives the UDP packet and POSTs it to `/api/v1/store/ingest` with metadata (room ID, timestamp, MAC).
|
||||
5. The Seed's drift detection engine monitors the happiness vector stream and flags anomalies (sudden drops, sustained low scores).
|
||||
6. Every ingested vector is appended to the Seed's Ed25519 witness chain, providing a tamper-proof audit trail.
|
||||
|
||||
### 4. Seed Drift Detection for Happiness Trends
|
||||
|
||||
The Seed's built-in drift detection compares incoming vectors against a rolling baseline:
|
||||
|
||||
- **Check-in calibration:** When a new guest checks in, event 694 resets the baseline.
|
||||
- **Drift threshold:** Configurable (default: cosine distance > 0.3 from baseline triggers alert).
|
||||
- **Trend window:** Last 20 vectors (~10 minutes at 30s intervals).
|
||||
- **Alert routing:** Seed webhook notifies hotel management system when happiness trend is declining.
|
||||
|
||||
### 5. RuView Live Dashboard Update
|
||||
|
||||
`ruview_live.py` gains a `--seed` flag:
|
||||
|
||||
```bash
|
||||
python ruview_live.py --port COM5 --seed 169.254.42.1 --mode happiness
|
||||
```
|
||||
|
||||
This mode displays:
|
||||
- Real-time 8-dim radar chart of the happiness vector
|
||||
- Scalar happiness score (0-100) with color coding (red/yellow/green)
|
||||
- Trend sparkline over the last hour
|
||||
- Seed witness chain status (last hash, chain length)
|
||||
- Room-level aggregate when multiple ESP32 nodes report
|
||||
|
||||
### 6. Architecture
|
||||
|
||||
```
|
||||
+------------------------------------------+
|
||||
| Hotel Room |
|
||||
| |
|
||||
| [ESP32-S3] [Cognitum Seed] |
|
||||
| COM5 or COM7 169.254.42.1 |
|
||||
| 4MB or 8MB flash Pi Zero 2 W |
|
||||
| | | |
|
||||
| | WiFi CSI | PIR, reed, |
|
||||
| | 20+ Hz | BME280, |
|
||||
| v | vibration |
|
||||
| +-----------+ | |
|
||||
| | Tier 2 DSP| v |
|
||||
| | presence | +-------------+ |
|
||||
| | vitals | | Seed API | |
|
||||
| | gait | | 98 endpoints| |
|
||||
| | fall det | | 398 vectors | |
|
||||
| +-----------+ | dim=8 | |
|
||||
| | +-------------+ |
|
||||
| v ^ |
|
||||
| +-----------+ UDP 5555 | |
|
||||
| | WASM edge |─────────────┘ |
|
||||
| | happiness | |
|
||||
| | score | Drift alerts |
|
||||
| | (690-694) |◄────────────── |
|
||||
| +-----------+ /api/v1/drift/check |
|
||||
| |
|
||||
+------------------------------------------+
|
||||
|
|
||||
| MQTT / HTTP
|
||||
v
|
||||
+------------------+
|
||||
| Hotel Management |
|
||||
| System / RuView |
|
||||
| Live Dashboard |
|
||||
+------------------+
|
||||
```
|
||||
|
||||
### 7. 4MB Flash Support
|
||||
|
||||
The 4MB ESP32-S3 variant (COM5) is officially supported for happiness scoring. The existing `partitions_4mb.csv` and `sdkconfig.defaults.4mb` from ADR-265 provide dual OTA slots (1.856 MB each), sufficient for the full Tier 2 DSP firmware plus `exo_happiness_score.wasm` (estimated < 40 KB).
|
||||
|
||||
Build for 4MB variant:
|
||||
|
||||
```bash
|
||||
cp sdkconfig.defaults.4mb sdkconfig.defaults
|
||||
idf.py build
|
||||
```
|
||||
|
||||
The WASM module loader selects which modules to instantiate based on available heap. On the 4MB/2MB PSRAM variant, happiness scoring runs with a reduced scoring window (60s instead of 30s) to conserve memory.
|
||||
|
||||
### 8. Privacy Considerations
|
||||
|
||||
- **No cameras.** All sensing is RF-based (WiFi subcarrier amplitude/phase).
|
||||
- **No facial recognition.** Happiness is inferred from movement patterns, not expressions.
|
||||
- **No audio capture.** Breathing rate is extracted from chest wall displacement via RF, not microphone.
|
||||
- **No PII stored on device.** Vectors are anonymous; room-to-guest mapping lives only in the hotel PMS.
|
||||
- **Seed witness chain** provides auditable proof of what data was collected and when, satisfying GDPR Article 30 record-keeping requirements.
|
||||
- **Guest opt-out:** A physical switch on the ESP32 node (GPIO connected to a toggle) disables CSI capture entirely. The Seed's reed switch can also serve as a "privacy mode" trigger (door-mounted magnet removed = sensing paused).
|
||||
- **Data retention:** Vectors are retained on the Seed for the duration of the stay plus 24 hours, then purged. The witness chain retains hashes (not vectors) indefinitely for audit.
|
||||
|
||||
### 9. API Integration
|
||||
|
||||
Key Cognitum Seed endpoints used:
|
||||
|
||||
| Endpoint | Method | Purpose |
|
||||
|----------|--------|---------|
|
||||
| `/api/v1/store/ingest` | POST | Ingest 8-dim happiness vector |
|
||||
| `/api/v1/store/query` | POST | Retrieve vectors by room/time range |
|
||||
| `/api/v1/drift/check` | GET | Check if current vector drifts from baseline |
|
||||
| `/api/v1/drift/configure` | PUT | Set drift threshold and window size |
|
||||
| `/api/v1/witness/append` | POST | Append event to Ed25519 custody chain |
|
||||
| `/api/v1/witness/verify` | GET | Verify chain integrity |
|
||||
| `/api/v1/sensors/bme280` | GET | Room temperature/humidity (comfort correlation) |
|
||||
| `/api/v1/sensors/pir` | GET | PIR presence (cross-validate with CSI) |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Provides real-time, objective guest satisfaction measurement without surveys or wearables.
|
||||
- Reuses four existing WASM modules -- the happiness module is a fusion layer, not a rewrite.
|
||||
- The Seed's 8-dim vector store is a natural fit; no schema changes needed.
|
||||
- Ed25519 witness chain satisfies hospitality industry audit requirements and GDPR record-keeping.
|
||||
- Both 4MB and 8MB ESP32-S3 variants are supported, enabling low-cost deployment at scale (~$8 per room for the 4MB node).
|
||||
- Seed's environmental sensors (BME280, PIR) provide complementary context (room temperature, humidity) that can be correlated with happiness scores.
|
||||
- No cloud dependency -- all processing is local (ESP32 edge + Seed link-local network).
|
||||
|
||||
### Negative
|
||||
|
||||
- Happiness inference from movement patterns is a proxy, not a direct measurement. Correlation with actual guest satisfaction must be validated empirically.
|
||||
- The 4MB variant has reduced scoring frequency (60s vs 30s) due to memory constraints.
|
||||
- UDP transport between ESP32 and Seed is unreliable; packets may be lost. Mitigation: sequence numbers and a small retry buffer on the ESP32 side.
|
||||
- Link-local addressing (169.254.x.x) limits the Seed to the same network segment as the ESP32. Multi-room deployments need one Seed per subnet or a routed bridge.
|
||||
- Drift detection thresholds require per-property tuning; a luxury resort has different movement patterns than a budget hotel.
|
||||
- The system cannot distinguish between guests in a multi-occupancy room without additional multi-target CSI clustering, which is experimental (ADR-064, Tier 3).
|
||||
@@ -0,0 +1,278 @@
|
||||
# ADR-066: ESP32 CSI Swarm with Cognitum Seed Coordinator
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-03-20
|
||||
**Deciders:** @ruvnet
|
||||
**Related:** ADR-065 (happiness scoring + Seed bridge), ADR-039 (edge intelligence), ADR-060 (provisioning), ADR-018 (CSI binary protocol), ADR-040 (WASM runtime)
|
||||
|
||||
## Context
|
||||
|
||||
ADR-065 established a single ESP32-S3 node pushing happiness vectors to a Cognitum Seed at `169.254.42.1` (Pi Zero 2 W, firmware 0.7.0). The Seed is now on the same WiFi network (`RedCloverWifi`, `10.1.10.236`) as the ESP32 node (`10.1.10.168`).
|
||||
|
||||
The Seed already exposes REST APIs for:
|
||||
- Peer discovery (`/api/v1/peers`) — 0 peers currently registered
|
||||
- Delta sync (`/api/v1/delta/pull`, `/api/v1/delta/push`) — epoch-based replication
|
||||
- Reflex rules (`/api/v1/sensor/reflex/rules`) — 3 rules (fragility alarm, drift cutoff, HD anomaly indicator)
|
||||
- Actuators (`/api/v1/sensor/actuators`) — relay + PWM outputs
|
||||
- Cognitive engine (`/api/v1/cognitive/tick`) — periodic inference loop
|
||||
- Witness chain (`/api/v1/custody/epoch`) — epoch 316, cryptographically signed
|
||||
- kNN search (`/api/v1/store/search`) — similarity queries across the full vector store
|
||||
|
||||
A hotel deployment requires multiple ESP32 nodes (lobby, hallway, restaurant, rooms) coordinated as a swarm with centralized analytics on the Seed.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a Seed-coordinated ESP32 swarm where each node operates autonomously for CSI sensing and edge processing, while the Seed serves as the swarm coordinator for registration, aggregation, drift detection, cross-zone inference, and actuator control.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
ESP32 Node A ESP32 Node B ESP32 Node C
|
||||
(Lobby) (Hallway) (Restaurant)
|
||||
node_id=1 node_id=2 node_id=3
|
||||
10.1.10.168 10.1.10.xxx 10.1.10.xxx
|
||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||
│ WiFi CSI │ │ WiFi CSI │ │ WiFi CSI │
|
||||
│ Tier 2 DSP │ │ Tier 2 DSP │ │ Tier 2 DSP │
|
||||
│ WASM Tier 3 │ │ WASM Tier 3 │ │ WASM Tier 3 │
|
||||
│ Swarm Bridge │ │ Swarm Bridge │ │ Swarm Bridge │
|
||||
└──────┬───────┘ └──────┬───────┘ └──────┬───────┘
|
||||
│ HTTP POST │ HTTP POST │ HTTP POST
|
||||
│ (happiness vectors, │ │
|
||||
│ heartbeat, events) │ │
|
||||
└──────────┬───────────────┴──────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────┐
|
||||
│ Cognitum Seed │
|
||||
│ (Coordinator) │
|
||||
│ 10.1.10.236 │
|
||||
├───────────────┤
|
||||
│ Vector Store │ ← 8-dim vectors tagged with node_id + zone
|
||||
│ kNN Search │ ← Cross-zone similarity ("which room matches?")
|
||||
│ Drift Detect │ ← Global mood trend across all zones
|
||||
│ Witness Chain │ ← Tamper-proof audit trail per node
|
||||
│ Reflex Rules │ ← Trigger actuators on swarm-wide patterns
|
||||
│ Cognitive Eng │ ← Periodic cross-zone inference
|
||||
│ Peer Registry │ ← Node health, last-seen, capabilities
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
### Swarm Protocol
|
||||
|
||||
#### 1. Node Registration (on boot)
|
||||
|
||||
Each ESP32 registers with the Seed via HTTP POST on startup. The Seed's peer discovery API tracks active nodes.
|
||||
|
||||
```
|
||||
POST /api/v1/store/ingest
|
||||
{
|
||||
"vectors": [{
|
||||
"id": "node-1-reg",
|
||||
"values": [0,0,0,0,0,0,0,0],
|
||||
"metadata": {
|
||||
"type": "registration",
|
||||
"node_id": 1,
|
||||
"zone": "lobby",
|
||||
"mac": "1C:DB:D4:83:D2:40",
|
||||
"ip": "10.1.10.168",
|
||||
"firmware": "0.5.0",
|
||||
"capabilities": ["csi", "tier2", "presence", "vitals", "happiness"],
|
||||
"flash_mb": 4,
|
||||
"psram_mb": 2
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
#### 2. Heartbeat (every 30 seconds)
|
||||
|
||||
```
|
||||
POST /api/v1/store/ingest
|
||||
{
|
||||
"vectors": [{
|
||||
"id": "node-1-hb-{epoch}",
|
||||
"values": [happiness, gait, stride, fluidity, calm, posture, dwell, social],
|
||||
"metadata": {
|
||||
"type": "heartbeat",
|
||||
"node_id": 1,
|
||||
"zone": "lobby",
|
||||
"uptime_s": 3600,
|
||||
"csi_frames": 72000,
|
||||
"free_heap": 317140,
|
||||
"presence_now": true,
|
||||
"persons": 2,
|
||||
"rssi": -60
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
#### 3. Happiness Vector Ingestion (every 5 seconds when presence detected)
|
||||
|
||||
```
|
||||
POST /api/v1/store/ingest
|
||||
{
|
||||
"vectors": [{
|
||||
"id": "node-1-h-{epoch}-{ts}",
|
||||
"values": [0.72, 0.65, 0.80, 0.71, 0.55, 0.60, 0.85, 0.45],
|
||||
"metadata": {
|
||||
"type": "happiness",
|
||||
"node_id": 1,
|
||||
"zone": "lobby",
|
||||
"timestamp_ms": 1742486400000,
|
||||
"persons": 2,
|
||||
"direction": "entering"
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
#### 4. Cross-Zone Queries (Seed-side)
|
||||
|
||||
The Seed can answer questions across the entire swarm:
|
||||
|
||||
```
|
||||
POST /api/v1/store/search
|
||||
{"vector": [0.8, 0.7, 0.9, 0.8, 0.6, 0.7, 0.9, 0.5], "k": 5}
|
||||
|
||||
Response: nearest neighbors across all zones, showing which
|
||||
rooms had the most similar mood to a "happy" reference vector.
|
||||
```
|
||||
|
||||
#### 5. Reflex Rules for Swarm Patterns
|
||||
|
||||
Configure the Seed's reflex engine to act on swarm-wide patterns:
|
||||
|
||||
| Rule | Trigger | Action | Use Case |
|
||||
|------|---------|--------|----------|
|
||||
| `low_happiness_alert` | Mean happiness < 0.3 across 3+ nodes for 5 min | Activate `alarm` relay | Staff alert: guest dissatisfaction |
|
||||
| `crowd_surge` | Presence count > 10 across lobby + hallway | PWM indicator brightness 100% | Lobby congestion warning |
|
||||
| `zone_drift` | Drift score > 0.5 on any node | Log to witness chain | Trend change documentation |
|
||||
| `ghost_anomaly` | Event 650 (anomaly) from any node | Notify + log | Security: unexpected RF disturbance |
|
||||
|
||||
### ESP32 Firmware: Swarm Bridge Module
|
||||
|
||||
New module `swarm_bridge.c` added to the CSI firmware, activated via NVS config:
|
||||
|
||||
```c
|
||||
typedef struct {
|
||||
char seed_url[64]; // e.g. "http://10.1.10.236"
|
||||
char zone_name[16]; // e.g. "lobby"
|
||||
uint16_t heartbeat_sec; // Default: 30
|
||||
uint16_t ingest_sec; // Default: 5
|
||||
uint8_t enabled; // 0 = disabled, 1 = enabled
|
||||
} swarm_config_t;
|
||||
```
|
||||
|
||||
NVS keys (provisioned via `provision.py --seed-url http://10.1.10.236 --zone lobby`):
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `seed_url` | string | (empty) | Seed base URL; empty = swarm disabled |
|
||||
| `zone_name` | string | `"default"` | Zone identifier for this node |
|
||||
| `swarm_hb` | u16 | 30 | Heartbeat interval (seconds) |
|
||||
| `swarm_ingest` | u16 | 5 | Vector ingest interval (seconds) |
|
||||
|
||||
The swarm bridge runs as a FreeRTOS task on Core 0 (separate from DSP on Core 1):
|
||||
|
||||
```
|
||||
swarm_bridge_task (Core 0, priority 3, stack 4096)
|
||||
├── On boot: POST registration to Seed
|
||||
├── Every 30s: POST heartbeat with latest happiness vector
|
||||
├── Every 5s (if presence): POST happiness vector
|
||||
└── On event 650+ (anomaly): POST immediately
|
||||
```
|
||||
|
||||
HTTP client uses `esp_http_client` (already in ESP-IDF, no extra dependencies). JSON is formatted with `snprintf` (no cJSON dependency needed for the small payloads).
|
||||
|
||||
### Node Discovery and Addressing
|
||||
|
||||
Nodes find the Seed via:
|
||||
|
||||
1. **NVS provisioned URL** (primary) — `provision.py --seed-url http://10.1.10.236`
|
||||
2. **mDNS fallback** — Seed advertises `_cognitum._tcp.local`; ESP32 resolves `cognitum.local`
|
||||
3. **Link-local fallback** — `http://169.254.42.1` when connected via USB
|
||||
|
||||
### Vector ID Scheme
|
||||
|
||||
```
|
||||
{node_id}-{type}-{epoch}-{timestamp_ms}
|
||||
```
|
||||
|
||||
Examples:
|
||||
- `1-reg` — Node 1 registration
|
||||
- `1-hb-316` — Node 1 heartbeat at epoch 316
|
||||
- `1-h-316-1742486400000` — Node 1 happiness vector at epoch 316, timestamp T
|
||||
- `2-h-316-1742486401000` — Node 2 happiness vector at same epoch
|
||||
|
||||
### Witness Chain Integration
|
||||
|
||||
Every vector ingested into the Seed increments the epoch and extends the witness chain. The chain provides:
|
||||
|
||||
- **Per-node audit trail** — filter by node_id metadata to get one node's history
|
||||
- **Tamper detection** — Ed25519 signed, hash-chained; break = detectable
|
||||
- **Regulatory compliance** — prove "sensor X reported Y at time Z" for disputes
|
||||
- **Cross-node ordering** — Seed epoch gives total order across all nodes
|
||||
|
||||
### Scaling Considerations
|
||||
|
||||
| Nodes | Vectors/hour | Seed storage/day | kNN latency |
|
||||
|-------|---|---|---|
|
||||
| 1 | 720 | ~1.5 MB | < 1 ms |
|
||||
| 5 | 3,600 | ~7.5 MB | < 2 ms |
|
||||
| 10 | 7,200 | ~15 MB | < 5 ms |
|
||||
| 20 | 14,400 | ~30 MB | < 10 ms |
|
||||
|
||||
The Seed's Pi Zero 2 W has 512 MB RAM and typically an 8-32 GB SD card. At 30 MB/day for 20 nodes, storage lasts 250+ days before compaction is needed. The Seed's optimizer runs automatic compaction in the background.
|
||||
|
||||
### Provisioning for Swarm
|
||||
|
||||
```bash
|
||||
# Node 1: Lobby (COM5, existing)
|
||||
python provision.py --port COM5 \
|
||||
--ssid "RedCloverWifi" --password "redclover2.4" \
|
||||
--node-id 1 --seed-url "http://10.1.10.236" --zone "lobby"
|
||||
|
||||
# Node 2: Hallway (future device)
|
||||
python provision.py --port COM6 \
|
||||
--ssid "RedCloverWifi" --password "redclover2.4" \
|
||||
--node-id 2 --seed-url "http://10.1.10.236" --zone "hallway"
|
||||
|
||||
# Node 3: Restaurant (future device)
|
||||
python provision.py --port COM8 \
|
||||
--ssid "RedCloverWifi" --password "redclover2.4" \
|
||||
--node-id 3 --seed-url "http://10.1.10.236" --zone "restaurant"
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Zero infrastructure** — no cloud, no server, no database. Seed + ESP32s + WiFi router is the entire stack
|
||||
- **Autonomous nodes** — each ESP32 runs full Tier 2 DSP independently; Seed loss degrades gracefully to local-only operation
|
||||
- **Cryptographic audit** — witness chain gives tamper-proof history for every observation across all nodes
|
||||
- **Real-time cross-zone analytics** — Seed kNN search answers "which zones are happy/stressed right now" in < 5 ms
|
||||
- **Physical actuators** — Seed's relay/PWM outputs can trigger real-world actions (lights, alarms, displays) based on swarm-wide patterns
|
||||
- **Horizontal scaling** — add ESP32 nodes by flashing firmware + running provision.py; no Seed reconfiguration needed
|
||||
- **Privacy-preserving** — no cameras, no audio, no PII; only 8-dimensional feature vectors stored
|
||||
|
||||
### Negative
|
||||
|
||||
- **Single point of aggregation** — Seed failure loses cross-zone analytics (nodes continue autonomously)
|
||||
- **WiFi dependency** — nodes must be on the same network as the Seed; no mesh/LoRa fallback yet
|
||||
- **HTTP overhead** — REST/JSON adds ~200 bytes overhead per vector vs raw binary UDP; acceptable at 5-second intervals
|
||||
- **Pi Zero 2 W limits** — 512 MB RAM, single-core ARM; adequate for 20 nodes but not 100+
|
||||
- **No WASM OTA via Seed** — currently WASM modules are uploaded per-node; future work could use Seed as WASM distribution hub
|
||||
|
||||
### Implementation Progress
|
||||
|
||||
**ADR-069** implements the first stage of this swarm vision with live hardware validation (2026-04-02). A single ESP32-S3 node (COM9, firmware v0.5.2) was validated sending CSI-derived feature vectors through a host-side bridge into the Cognitum Seed's RVF store (firmware v0.8.1). The pipeline confirmed: UDP streaming (211 packets/15s), 8-dim feature extraction, batched HTTPS ingest (4 batches of 5 vectors), and witness chain integrity (193 entries, SHA-256 verified). Multi-node deployment (Phase 4 of ADR-069) is the next step toward the full swarm architecture described here.
|
||||
|
||||
### Future Work
|
||||
|
||||
- **Seed-initiated WASM push** — Seed distributes WASM modules to all nodes via their OTA endpoints
|
||||
- **mDNS auto-discovery** — nodes find Seed without provisioned URL
|
||||
- **Mesh fallback** — ESP-NOW peer-to-peer when WiFi is down
|
||||
- **Multi-Seed federation** — multiple Seeds for multi-floor/multi-building deployments
|
||||
- **Seed dashboard** — web UI on the Seed showing live swarm map with per-zone happiness
|
||||
@@ -0,0 +1,151 @@
|
||||
# ADR-067: RuVector v2.0.4 to v2.0.5 Upgrade + New Crate Adoption
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-03-23
|
||||
**Deciders:** @ruvnet
|
||||
**Related:** ADR-016 (RuVector training pipeline integration), ADR-017 (RuVector signal + MAT integration), ADR-029 (RuvSense multistatic sensing)
|
||||
|
||||
## Context
|
||||
|
||||
RuView currently pins all five core RuVector crates at **v2.0.4** (from crates.io) plus a vendored `ruvector-crv` v0.1.1 and optional `ruvector-gnn` v2.0.5. The upstream RuVector workspace has moved to **v2.0.5** with meaningful improvements to the crates we depend on, and has introduced new crates that could benefit RuView's detection pipeline.
|
||||
|
||||
### Current Integration Map
|
||||
|
||||
| RuView Module | RuVector Crate | Current Version | Purpose |
|
||||
|---------------|----------------|-----------------|---------|
|
||||
| `signal/subcarrier.rs` | ruvector-mincut | 2.0.4 | Graph min-cut subcarrier partitioning |
|
||||
| `signal/spectrogram.rs` | ruvector-attn-mincut | 2.0.4 | Attention-gated spectrogram denoising |
|
||||
| `signal/bvp.rs` | ruvector-attention | 2.0.4 | Attention-weighted BVP aggregation |
|
||||
| `signal/fresnel.rs` | ruvector-solver | 2.0.4 | Fresnel geometry estimation |
|
||||
| `mat/triangulation.rs` | ruvector-solver | 2.0.4 | TDoA survivor localization |
|
||||
| `mat/breathing.rs` | ruvector-temporal-tensor | 2.0.4 | Tiered compressed breathing buffer |
|
||||
| `mat/heartbeat.rs` | ruvector-temporal-tensor | 2.0.4 | Tiered compressed heartbeat spectrogram |
|
||||
| `viewpoint/*` (4 files) | ruvector-attention | 2.0.4 | Cross-viewpoint fusion with geometric bias |
|
||||
| `crv/` (optional) | ruvector-crv | 0.1.1 (vendored) | CRV protocol integration |
|
||||
| `crv/` (optional) | ruvector-gnn | 2.0.5 | GNN graph topology |
|
||||
|
||||
### What Changed Upstream (v2.0.4 → v2.0.5 → HEAD)
|
||||
|
||||
**ruvector-mincut:**
|
||||
- Flat capacity matrix + allocation reuse — **10-30% faster** for all min-cut operations
|
||||
- Tier 2-3 Dynamic MinCut (ADR-124): Gomory-Hu tree construction for fast global min-cut, incremental edge insert/delete without full recomputation
|
||||
- Source-anchored canonical min-cut with SHA-256 witness hashing
|
||||
- Fixed: unsafe indexing removed, WASM Node.js panic from `std::time`
|
||||
|
||||
**ruvector-attention / ruvector-attn-mincut:**
|
||||
- Migrated to workspace versioning (no API changes)
|
||||
- Documentation improvements
|
||||
|
||||
**ruvector-temporal-tensor:**
|
||||
- Formatting fixes only (no API changes)
|
||||
|
||||
**ruvector-gnn:**
|
||||
- Panic replaced with `Result` in `MultiHeadAttention` and `RuvectorLayer` constructors (breaking improvement — safer)
|
||||
- Bumped to v2.0.5
|
||||
|
||||
**sona (new — Self-Optimizing Neural Architecture):**
|
||||
- v0.1.6 → v0.1.8: state persistence (`loadState`/`saveState`), trajectory counter fix
|
||||
- Micro-LoRA and Base-LoRA for instant and background learning
|
||||
- EWC++ (Elastic Weight Consolidation) to prevent catastrophic forgetting
|
||||
- ReasoningBank pattern extraction and similarity search
|
||||
- WASM support for edge devices
|
||||
|
||||
**ruvector-coherence (new):**
|
||||
- Spectral coherence scoring for graph index health
|
||||
- Fiedler eigenvalue estimation, effective resistance sampling
|
||||
- HNSW health monitoring with alerts
|
||||
- Batch evaluation of attention mechanism quality
|
||||
|
||||
**ruvector-core (new):**
|
||||
- ONNX embedding support for real semantic embeddings
|
||||
- HNSW index with SIMD-accelerated distance metrics
|
||||
- Quantization (4-32x memory reduction)
|
||||
- Arena allocator for cache-optimized operations
|
||||
|
||||
## Decision
|
||||
|
||||
### Phase 1: Version Bump (Low Risk)
|
||||
|
||||
Bump the 5 core crates from v2.0.4 to v2.0.5 in the workspace `Cargo.toml`:
|
||||
|
||||
```toml
|
||||
ruvector-mincut = "2.0.5" # was 2.0.4 — 10-30% faster, safer
|
||||
ruvector-attn-mincut = "2.0.5" # was 2.0.4 — workspace versioning
|
||||
ruvector-temporal-tensor = "2.0.5" # was 2.0.4 — fmt only
|
||||
ruvector-solver = "2.0.5" # was 2.0.4 — workspace versioning
|
||||
ruvector-attention = "2.0.5" # was 2.0.4 — workspace versioning
|
||||
```
|
||||
|
||||
**Expected impact:** The mincut performance improvement directly benefits `signal/subcarrier.rs` which runs subcarrier graph partitioning every tick. 10-30% faster partitioning reduces per-frame CPU cost.
|
||||
|
||||
### Phase 2: Add ruvector-coherence (Medium Value)
|
||||
|
||||
Add `ruvector-coherence` with `spectral` feature to `wifi-densepose-ruvector`:
|
||||
|
||||
**Use case:** Replace or augment the custom phase coherence logic in `viewpoint/coherence.rs` with spectral graph coherence scoring. The current implementation uses phasor magnitude for phase coherence — spectral Fiedler estimation would provide a more robust measure of multi-node CSI consistency, especially for detecting when a node's signal quality degrades.
|
||||
|
||||
**Integration point:** `viewpoint/coherence.rs` — add `SpectralCoherenceScore` as a secondary coherence metric alongside existing phase phasor coherence. Use spectral gap estimation to detect structural changes in the multi-node CSI graph (e.g., a node dropping out or a new reflector appearing).
|
||||
|
||||
### Phase 3: Add SONA for Adaptive Learning (High Value)
|
||||
|
||||
Replace the logistic regression adaptive classifier in the sensing server with a SONA-backed learning engine:
|
||||
|
||||
**Current state:** The sensing server's adaptive training (`POST /api/v1/adaptive/train`) uses a hand-rolled logistic regression on 15 CSI features. It requires explicit labeled recordings and provides no cross-session persistence.
|
||||
|
||||
**Proposed improvement:** Use `sona::SonaEngine` to:
|
||||
1. **Learn from implicit feedback** — trajectory tracking on person-count decisions (was the count stable? did the user correct it?)
|
||||
2. **Persist across sessions** — `saveState()`/`loadState()` replaces the current `adaptive_model.json`
|
||||
3. **Pattern matching** — `find_patterns()` enables "this CSI signature looks like room X where we learned Y"
|
||||
4. **Prevent forgetting** — EWC++ ensures learning in a new room doesn't overwrite patterns from previous rooms
|
||||
|
||||
**Integration point:** New `adaptive_sona.rs` module in `wifi-densepose-sensing-server`, behind a `sona` feature flag. The existing logistic regression remains the default.
|
||||
|
||||
### Phase 4: Evaluate ruvector-core for CSI Embeddings (Exploratory)
|
||||
|
||||
**Current state:** The person detection pipeline uses hand-crafted features (variance, change_points, motion_band_power, spectral_power) with fixed normalization ranges.
|
||||
|
||||
**Potential:** Use `ruvector-core`'s ONNX embedding support to generate learned CSI embeddings that capture room geometry, person count, and activity patterns in a single vector. This would enable:
|
||||
- Similarity search: "is this CSI frame similar to known 2-person patterns?"
|
||||
- Transfer learning: embeddings learned in one room partially transfer to similar rooms
|
||||
- Quantized storage: 4-32x memory reduction for pattern databases
|
||||
|
||||
**Status:** Exploratory — requires training data collection and embedding model design. Not a near-term target.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- **Phase 1:** Free 10-30% performance gain in subcarrier partitioning. Security fixes (unsafe indexing, WASM panic). Zero API changes required.
|
||||
- **Phase 2:** More robust multi-node coherence detection. Helps with the "flickering persons" issue (#292) by providing a second opinion on signal quality.
|
||||
- **Phase 3:** Fundamentally improves the adaptive learning pipeline. Users no longer need to manually record labeled data — the system learns from ongoing use.
|
||||
- **Phase 4:** Path toward real ML-based detection instead of heuristic thresholds.
|
||||
|
||||
### Negative
|
||||
- **Phase 1:** Minimal risk — semver minor bump, no API breaks.
|
||||
- **Phase 2:** Adds a dependency. Spectral computation has O(n) cost per tick for Fiedler estimation (n = number of subcarriers, typically 56-128). Acceptable.
|
||||
- **Phase 3:** SONA adds ~200KB to the binary. The learning loop needs careful tuning to avoid adapting to noise.
|
||||
- **Phase 4:** Requires significant research and training data. Not guaranteed to outperform tuned heuristics for WiFi CSI.
|
||||
|
||||
### Risks
|
||||
- `ruvector-gnn` v2.0.5 changed constructors from panic to `Result` — any existing `crv` feature users need to handle the `Result`. Our vendored `ruvector-crv` may need updates.
|
||||
- SONA's WASM support is experimental — keep it behind a feature flag until validated.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
| Phase | Scope | Effort | Priority |
|
||||
|-------|-------|--------|----------|
|
||||
| 1 | Bump 5 crates to v2.0.5 | 1 hour | High — free perf + security |
|
||||
| 2 | Add ruvector-coherence | 1 day | Medium — improves multi-node stability |
|
||||
| 3 | SONA adaptive learning | 3 days | Medium — replaces manual training workflow |
|
||||
| 4 | CSI embeddings via ruvector-core | 1-2 weeks | Low — exploratory research |
|
||||
|
||||
## Vendor Submodule
|
||||
|
||||
The `vendor/ruvector` git submodule has been updated from commit `f8f2c60` (v2.0.4 era) to `51a3557` (latest `origin/main`). This provides local reference for the full upstream source when developing Phases 2-4.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream repo: https://github.com/ruvnet/ruvector
|
||||
- ADR-124 (Dynamic MinCut): `vendor/ruvector/docs/adr/ADR-124*.md`
|
||||
- SONA docs: `vendor/ruvector/crates/sona/src/lib.rs`
|
||||
- ruvector-coherence spectral: `vendor/ruvector/crates/ruvector-coherence/src/spectral.rs`
|
||||
- ruvector-core embeddings: `vendor/ruvector/crates/ruvector-core/src/embeddings.rs`
|
||||
@@ -0,0 +1,186 @@
|
||||
# ADR-068: Per-Node State Pipeline for Multi-Node Sensing
|
||||
|
||||
| Field | Value |
|
||||
|------------|-------------------------------------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-03-27 |
|
||||
| Authors | rUv, claude-flow |
|
||||
| Drivers | #249, #237, #276, #282 |
|
||||
| Supersedes | — |
|
||||
|
||||
## Context
|
||||
|
||||
The sensing server (`wifi-densepose-sensing-server`) was originally designed for
|
||||
single-node operation. When multiple ESP32 nodes send CSI frames simultaneously,
|
||||
all data is mixed into a single shared pipeline:
|
||||
|
||||
- **One** `frame_history` VecDeque for all nodes
|
||||
- **One** `smoothed_person_score` / `smoothed_motion` / vital sign buffers
|
||||
- **One** baseline and debounce state
|
||||
|
||||
This means the classification, person count, and vital signs reported to the UI
|
||||
are an uncontrolled aggregate of all nodes' data. The result: the detection
|
||||
window shows identical output regardless of how many nodes are deployed, where
|
||||
people stand, or how many people are in the room (#249 — 24 comments, the most
|
||||
reported issue).
|
||||
|
||||
### Root Cause Verified
|
||||
|
||||
Investigation of `AppStateInner` (main.rs lines 279-367) confirmed:
|
||||
|
||||
| Shared field | Impact |
|
||||
|---------------------------|--------------------------------------------|
|
||||
| `frame_history` | Temporal analysis mixes all nodes' CSI data |
|
||||
| `smoothed_person_score` | Person count aggregates all nodes |
|
||||
| `smoothed_motion` | Motion classification undifferentiated |
|
||||
| `smoothed_hr` / `br` | Vital signs are global, not per-node |
|
||||
| `baseline_motion` | Adaptive baseline learned from mixed data |
|
||||
| `debounce_counter` | All nodes share debounce state |
|
||||
|
||||
## Decision
|
||||
|
||||
Introduce **per-node state tracking** via a `HashMap<u8, NodeState>` in
|
||||
`AppStateInner`. Each ESP32 node (identified by its `node_id` byte) gets an
|
||||
independent sensing pipeline with its own temporal history, smoothing buffers,
|
||||
baseline, and classification state.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
UDP frames │ AppStateInner │
|
||||
───────────► │ │
|
||||
node_id=1 ──► │ node_states: HashMap<u8, NodeState> │
|
||||
node_id=2 ──► │ ├── 1: NodeState { frame_history, │
|
||||
node_id=3 ──► │ │ smoothed_motion, vitals, ... }│
|
||||
│ ├── 2: NodeState { ... } │
|
||||
│ └── 3: NodeState { ... } │
|
||||
│ │
|
||||
│ ┌── Per-Node Pipeline ──┐ │
|
||||
│ │ extract_features() │ │
|
||||
│ │ smooth_and_classify() │ │
|
||||
│ │ smooth_vitals() │ │
|
||||
│ │ score_to_person_count()│ │
|
||||
│ └────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌── Multi-Node Fusion ──┐ │
|
||||
│ │ Aggregate person count │ │
|
||||
│ │ Per-node classification│ │
|
||||
│ │ All-nodes WebSocket msg│ │
|
||||
│ └────────────────────────┘ │
|
||||
│ │
|
||||
│ ──► WebSocket broadcast (sensing_update) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### NodeState Struct
|
||||
|
||||
```rust
|
||||
struct NodeState {
|
||||
frame_history: VecDeque<Vec<f64>>,
|
||||
smoothed_person_score: f64,
|
||||
prev_person_count: usize,
|
||||
smoothed_motion: f64,
|
||||
current_motion_level: String,
|
||||
debounce_counter: u32,
|
||||
debounce_candidate: String,
|
||||
baseline_motion: f64,
|
||||
baseline_frames: u64,
|
||||
smoothed_hr: f64,
|
||||
smoothed_br: f64,
|
||||
smoothed_hr_conf: f64,
|
||||
smoothed_br_conf: f64,
|
||||
hr_buffer: VecDeque<f64>,
|
||||
br_buffer: VecDeque<f64>,
|
||||
rssi_history: VecDeque<f64>,
|
||||
vital_detector: VitalSignDetector,
|
||||
latest_vitals: VitalSigns,
|
||||
last_frame_time: Option<std::time::Instant>,
|
||||
edge_vitals: Option<Esp32VitalsPacket>,
|
||||
}
|
||||
```
|
||||
|
||||
### Multi-Node Aggregation
|
||||
|
||||
- **Person count**: Sum of per-node `prev_person_count` for active nodes
|
||||
(seen within last 10 seconds).
|
||||
- **Classification**: Per-node classification included in `SensingUpdate.nodes`.
|
||||
- **Vital signs**: Per-node vital signs; UI can render per-node or aggregate.
|
||||
- **Signal field**: Generated from the most-recently-updated node's features.
|
||||
- **Stale nodes**: Nodes with no frame for >10 seconds are excluded from
|
||||
aggregation and marked offline (consistent with PR #300).
|
||||
|
||||
### Backward Compatibility
|
||||
|
||||
- The simulated data path (`simulated_data_task`) continues using global state.
|
||||
- Single-node deployments behave identically (HashMap has one entry).
|
||||
- The WebSocket message format (`sensing_update`) remains the same but the
|
||||
`nodes` array now contains all active nodes, and `estimated_persons` reflects
|
||||
the cross-node aggregate.
|
||||
- The edge vitals path (#323 fix) also uses per-node state.
|
||||
|
||||
## Scaling Characteristics
|
||||
|
||||
| Nodes | Per-Node Memory | Total Overhead | Notes |
|
||||
|-------|----------------|----------------|-------|
|
||||
| 1 | ~50 KB | ~50 KB | Identical to current |
|
||||
| 3 | ~50 KB | ~150 KB | Typical home setup |
|
||||
| 10 | ~50 KB | ~500 KB | Small office |
|
||||
| 50 | ~50 KB | ~2.5 MB | Building floor |
|
||||
| 100 | ~50 KB | ~5 MB | Large deployment |
|
||||
| 256 | ~50 KB | ~12.8 MB | Max (u8 node_id) |
|
||||
|
||||
Memory is dominated by `frame_history` (100 frames x ~500 bytes each = ~50 KB
|
||||
per node). This scales linearly and fits comfortably in server memory even at
|
||||
256 nodes.
|
||||
|
||||
## QEMU Validation
|
||||
|
||||
The existing QEMU swarm infrastructure (ADR-062, `scripts/qemu_swarm.py`)
|
||||
supports multi-node simulation with configurable topologies:
|
||||
|
||||
- `star`: Central coordinator + sensor nodes
|
||||
- `mesh`: Fully connected peer network
|
||||
- `line`: Sequential chain
|
||||
- `ring`: Circular topology
|
||||
|
||||
Each QEMU instance runs with a unique `node_id` via NVS provisioning. The
|
||||
swarm health validator (`scripts/swarm_health.py`) checks per-node UART output.
|
||||
|
||||
Validation plan:
|
||||
1. QEMU swarm with 3-5 nodes in mesh topology
|
||||
2. Verify server produces distinct per-node classifications
|
||||
3. Verify aggregate person count reflects multi-node contributions
|
||||
4. Verify stale-node eviction after timeout
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Each node's CSI data is processed independently — no cross-contamination
|
||||
- Person count scales with the number of deployed nodes
|
||||
- Vital signs are per-node, enabling room-level health monitoring
|
||||
- Foundation for spatial localization (per-node positions + triangulation)
|
||||
- Scales to 256 nodes with <13 MB memory overhead
|
||||
|
||||
### Negative
|
||||
- Slightly more memory per node (~50 KB each)
|
||||
- `smooth_and_classify_node` function duplicates some logic from global version
|
||||
- Per-node `VitalSignDetector` instances add CPU cost proportional to node count
|
||||
|
||||
### Risks
|
||||
- Node ID collisions (mitigated by NVS persistence since v0.5.0)
|
||||
- HashMap growth without cleanup (mitigated by stale-node eviction)
|
||||
|
||||
## Related ADRs
|
||||
|
||||
- **ADR-069** (ESP32 CSI → Cognitum Seed RVF Ingest Pipeline) extends this ADR's per-node state architecture with Cognitum Seed integration. Live hardware validation (2026-04-02) confirmed per-node feature vectors flowing through the bridge into the Seed's RVF store with witness chain attestation.
|
||||
|
||||
## References
|
||||
|
||||
- Issue #249: Detection window same regardless (24 comments)
|
||||
- Issue #237: Same display for 0/1/2 people (12 comments)
|
||||
- Issue #276: Only one can be detected (8 comments)
|
||||
- Issue #282: Detection fail (5 comments)
|
||||
- PR #295: Hysteresis smoothing (partial mitigation)
|
||||
- PR #300: ESP32 offline detection after 5s
|
||||
- ADR-062: QEMU Swarm Configurator
|
||||
@@ -0,0 +1,403 @@
|
||||
# ADR-069: ESP32 CSI → Cognitum Seed RVF Ingest Pipeline
|
||||
|
||||
| Field | Value |
|
||||
|------------|----------------------------------------------------------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-04-02 |
|
||||
| Authors | rUv, claude-flow |
|
||||
| Drivers | #348 (multinode mesh accuracy), Research: Arena Physica |
|
||||
| Supersedes | — |
|
||||
| Related | ADR-066 (ESP32 swarm + Seed coordinator), ADR-068 (per-node state), ADR-018 (CSI binary protocol), ADR-039 (edge intelligence), ADR-065 (happiness scoring + Seed bridge) |
|
||||
|
||||
## Context
|
||||
|
||||
The wifi-densepose project has two hardware components that need to work as an integrated sensing pipeline:
|
||||
|
||||
1. **ESP32-S3** (COM9 / 192.168.1.105) — Captures WiFi CSI at 100 Hz, runs dual-core DSP pipeline (phase extraction, subcarrier selection, breathing/heart rate estimation, presence/fall detection), and sends ADR-018 binary frames via UDP.
|
||||
|
||||
2. **Cognitum Seed** (USB / 169.254.42.1 / 192.168.1.109) — A Pi Zero 2 W edge intelligence appliance running firmware v0.8.1. It provides:
|
||||
- **RVF vector store** — Append-only binary format with content-addressed IDs, kNN queries (cosine/L2/dot), and kNN graph with boundary analysis
|
||||
- **Witness chain** — SHA-256 tamper-evident audit trail for every write operation
|
||||
- **Ed25519 custody** — Device-bound keypair for cryptographic attestation
|
||||
- **Sensor pipeline** — 5 sensors (reed switch, PIR, vibration, ADS1115 4-ch ADC, BME280), 13 drift detectors, anti-spoofing
|
||||
- **Cognitive container** — Spectral graph analysis with Stoer-Wagner min-cut fragility scoring
|
||||
- **MCP proxy** — 114 tools via JSON-RPC 2.0 for AI assistant integration
|
||||
- **Thermal governor** — DVFS management with zone-based frequency scaling
|
||||
- **Temporal coherence** — Phase boundary detection across vector store evolution
|
||||
- **Swarm sync** — Epoch-based delta replication between peers
|
||||
- **Reflex rules** — 3 rules (fragility alarm, drift cutoff, HD anomaly indicator)
|
||||
- **98 HTTPS API endpoints** with per-client bearer token authentication
|
||||
|
||||
### Current State
|
||||
|
||||
| Component | Status | Details |
|
||||
|-----------|--------|---------|
|
||||
| ESP32 CSI capture | Working | 100 Hz, ADR-018 binary frames via UDP |
|
||||
| ESP32 edge DSP | Working | 10-stage pipeline on Core 1 (phase, variance, vitals, fall) |
|
||||
| ESP32 → sensing-server | Working | UDP port 5005, binary protocol |
|
||||
| Cognitum Seed | Online | v0.8.1, paired, 19 vectors, epoch 25, WiFi connected |
|
||||
| Seed vector store | Working | 8-dim RVF, kNN queries in 85ms for 20k vectors |
|
||||
| Seed MCP proxy | Working | 114 tools, default-deny policy |
|
||||
| ESP32 → Seed pipeline | **Validated** | Bridge on host laptop, UDP 5006 → HTTPS ingest (see Validation Results) |
|
||||
|
||||
### Gap Analysis (from Arena Physica research)
|
||||
|
||||
Arena Physica's approach (Heaviside-0 forward model, Marconi-0 inverse diffusion) demonstrates that neural surrogates for Maxwell's equations are production-viable. Our research identified that:
|
||||
|
||||
1. **Physics-informed intermediate supervision** — Evaluating pipeline stages independently catches failures that end-to-end metrics miss
|
||||
2. **Vector embeddings for EM fields** — Storing CSI features as vectors enables similarity search for environment fingerprinting and anomaly detection
|
||||
3. **Witness chain for sensing integrity** — Tamper-evident audit trails are critical for healthcare/safety applications (fall detection, vital signs)
|
||||
4. **Edge compute for inference** — Pi Zero 2 W can run ~2.5M parameter models at 10+ Hz with INT8 quantization
|
||||
|
||||
### Problem
|
||||
|
||||
There is no pipeline connecting ESP32 CSI sensing to the Cognitum Seed's vector store. The ESP32 sends raw CSI frames to the Rust sensing-server (typically running on a laptop/desktop), but cannot leverage the Seed's:
|
||||
- Persistent vector storage with kNN search
|
||||
- Cryptographic witness chain for data integrity
|
||||
- Cognitive container for structural analysis
|
||||
- Sensor fusion with environmental sensors (BME280 temperature/humidity, PIR motion)
|
||||
- Swarm sync for multi-Seed deployments
|
||||
|
||||
## Decision
|
||||
|
||||
Build a three-stage pipeline connecting ESP32 CSI capture to Cognitum Seed RVF storage:
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────┐
|
||||
│ ESP32-S3 (COM9) │
|
||||
│ node_id=1 │
|
||||
│ 192.168.1.105 │
|
||||
│ Firmware v0.5.2 │
|
||||
│ ┌──────────────────────┐ │
|
||||
│ │ Core 0: WiFi + CSI │ │
|
||||
│ │ 100 Hz capture │ │
|
||||
│ │ ADR-018 framing │ │
|
||||
│ ├──────────────────────┤ │
|
||||
│ │ Core 1: Edge DSP │ │
|
||||
│ │ Phase extraction │ │
|
||||
│ │ Subcarrier select │ │
|
||||
│ │ Vital signs (HR/BR)│ │
|
||||
│ │ Presence/fall det. │ │
|
||||
│ │ Feature vector │ │◄── 8-dim feature extraction
|
||||
│ └──────────┬───────────┘ │
|
||||
│ │ UDP │
|
||||
└────────────┼─────────────┘
|
||||
│ Port 5005 (raw CSI, magic 0xC5110001)
|
||||
│ + Port 5006 (vitals 0xC5110002 + features 0xC5110003)
|
||||
▼
|
||||
┌────────────────────────────────────────────┐
|
||||
│ Host Laptop (192.168.1.20) │
|
||||
│ Bridge script (Python) │
|
||||
│ ┌────────────────────────────────────────┐ │
|
||||
│ │ Stage 1: CSI Receiver │ │
|
||||
│ │ UDP listener on port 5006 │ │
|
||||
│ │ Parses 0xC5110003 feature packets │ │
|
||||
│ │ (also accepts 0xC5110001/0002) │ │
|
||||
│ │ Batches 10 vectors per ingest │ │
|
||||
│ └──────────┬─────────────────────────────┘ │
|
||||
└────────────┼───────────────────────────────┘
|
||||
│ HTTPS POST (bearer token)
|
||||
▼
|
||||
┌────────────────────────────────────────────┐
|
||||
│ Cognitum Seed (Pi Zero 2 W) │
|
||||
│ 169.254.42.1 / 192.168.1.109 │
|
||||
│ Firmware v0.8.1 │
|
||||
│ ┌────────────────────────────────────────┐ │
|
||||
│ │ Stage 2: RVF Ingest │ │
|
||||
│ │ POST /api/v1/store/ingest │ │
|
||||
│ │ Content-addressed vector ID │ │
|
||||
│ │ Metadata: node_id, timestamp, type │ │
|
||||
│ │ Witness chain entry per batch │ │
|
||||
│ ├────────────────────────────────────────┤ │
|
||||
│ │ Stage 3: Cognitive Analysis │ │
|
||||
│ │ kNN graph rebuild (every 10s) │ │
|
||||
│ │ Boundary analysis (fragility) │ │
|
||||
│ │ Temporal coherence (phase detect) │ │
|
||||
│ │ Reflex rules (alarm triggers) │ │
|
||||
│ ├────────────────────────────────────────┤ │
|
||||
│ │ Existing Sensors │ │
|
||||
│ │ BME280 → temp/humidity/pressure │ │
|
||||
│ │ PIR → motion ground truth │ │
|
||||
│ │ Reed switch → door/window state │ │
|
||||
│ │ ADS1115 → analog inputs │ │
|
||||
│ └────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ Outputs: │
|
||||
│ • /api/v1/store/query — kNN search │
|
||||
│ • /api/v1/boundary — fragility score │
|
||||
│ • /api/v1/coherence/profile — phases │
|
||||
│ • /api/v1/cognitive/snapshot — graph │
|
||||
│ • /api/v1/custody/attestation — signed │
|
||||
│ • MCP proxy — 114 tools for AI agents │
|
||||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Stage 1: ESP32 Feature Vector Extraction
|
||||
|
||||
The ESP32 edge processing pipeline (Core 1) already computes all signals needed. We add a compact 8-dimensional feature vector extracted from the existing DSP outputs:
|
||||
|
||||
| Dimension | Feature | Source | Range |
|
||||
|-----------|---------|--------|-------|
|
||||
| 0 | Presence score | `s_presence_score / 10.0` (clamped) | 0.0–1.0 |
|
||||
| 1 | Motion energy | `s_motion_energy / 10.0` (clamped) | 0.0–1.0 |
|
||||
| 2 | Breathing rate | `s_breathing_bpm / 30.0` (clamped) | 0.0–1.0 |
|
||||
| 3 | Heart rate | `s_heartrate_bpm / 120.0` (clamped) | 0.0–1.0 |
|
||||
| 4 | Phase variance (mean) | Top-K subcarrier Welford variance mean | 0.0–1.0 |
|
||||
| 5 | Person count | `n_active_persons / 4.0` (clamped) | 0.0–1.0 |
|
||||
| 6 | Fall detected | Binary: 1.0 if `s_fall_detected`, else 0.0 | 0.0 or 1.0 |
|
||||
| 7 | RSSI (normalized) | `(s_latest_rssi + 100) / 100` (clamped) | 0.0–1.0 |
|
||||
|
||||
This maps directly to the Seed's store dimension of 8, enabling kNN queries like "find the 10 most similar sensing states to the current one."
|
||||
|
||||
**Packet format** (magic `0xC5110003`, defined as `edge_feature_pkt_t` in `edge_processing.h`):
|
||||
|
||||
```c
|
||||
typedef struct __attribute__((packed)) {
|
||||
uint32_t magic; // EDGE_FEATURE_MAGIC = 0xC5110003
|
||||
uint8_t node_id; // ESP32 node identifier
|
||||
uint8_t reserved; // alignment padding
|
||||
uint16_t seq; // sequence number
|
||||
int64_t timestamp_us; // microseconds since boot
|
||||
float features[8]; // 8-dim normalized feature vector (32 bytes)
|
||||
} edge_feature_pkt_t; // Total: 48 bytes (static_assert enforced)
|
||||
```
|
||||
|
||||
**Transmission rate:** 1 Hz (one feature vector per second, aggregated from 100 Hz CSI). This keeps UDP bandwidth under 50 bytes/s per node and avoids overwhelming the Seed's vector store.
|
||||
|
||||
### Stage 2: Seed-Side RVF Ingest
|
||||
|
||||
A lightweight Rust service on the Seed (or a Python bridge script) listens for feature packets on UDP port 5006 and ingests them via the Seed's REST API:
|
||||
|
||||
```bash
|
||||
# Ingest a feature vector with metadata
|
||||
curl -sk -X POST https://169.254.42.1:8443/api/v1/store/ingest \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"vectors": [[0, [0.85, 0.3, 0.52, 0.65, 0.4, 0.78, 0.1, -0.45]]],
|
||||
"metadata": {
|
||||
"node_id": 1,
|
||||
"type": "csi_feature",
|
||||
"timestamp": 1775166970
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Batching:** Accumulate 10 vectors (10 seconds) per ingest call to reduce HTTP overhead (`--batch-size 10` default in `seed_csi_bridge.py`; also supports time-based flushing via `--flush-interval`). At 1 vector/second per node, a 4-node mesh generates 14,400 vectors/hour (345,600/day). Daily compaction is required to stay within the Seed's 100K vector working set (see Storage Budget).
|
||||
|
||||
**Witness chain:** Each ingest automatically appends a witness entry, providing a tamper-evident record of all sensing data. The epoch increments monotonically, and the SHA-256 chain can be verified at any time via `POST /api/v1/witness/verify`.
|
||||
|
||||
### Stage 3: Cognitive Analysis & Sensor Fusion
|
||||
|
||||
Once CSI feature vectors are in the RVF store, the Seed's existing subsystems activate:
|
||||
|
||||
1. **kNN Graph** — Rebuilt every 10 seconds. Similar sensing states cluster together. Anomalous states (intruder, fall, unusual breathing) appear as outliers.
|
||||
|
||||
2. **Boundary Analysis** — Stoer-Wagner min-cut computes a fragility score (0.0–1.0). High fragility indicates the vector space is splitting — a regime change in the environment (door opened, person entered/left, HVAC state change).
|
||||
|
||||
3. **Temporal Coherence** — Phase boundary detection across the vector store timeline identifies when the environment transitions between states (occupied → empty, day → night, normal → abnormal).
|
||||
|
||||
4. **Reflex Rules** — Three pre-configured rules fire automatically:
|
||||
- `fragility_alarm` (threshold 0.3) → relay actuator for presence alert
|
||||
- `drift_cutoff` (threshold 1.0) → cutoff when sensor drift detected
|
||||
- `hd_anomaly_indicator` (threshold 200) → PWM brightness for anomaly severity
|
||||
|
||||
5. **Sensor Fusion** — The Seed's BME280 (temperature/humidity/pressure) and PIR sensor provide environmental ground truth that correlates with CSI features:
|
||||
- PIR motion validates CSI presence detection
|
||||
- Temperature changes correlate with occupancy
|
||||
- Humidity changes correlate with breathing detection fidelity
|
||||
|
||||
6. **MCP Integration** — AI assistants can query the full pipeline via the 114-tool MCP proxy:
|
||||
```json
|
||||
{"method": "tools/call", "params": {"name": "seed.memory.query", "arguments": {"vector": [0.8, 0.5, 0.4, 0.6, 0.3, 0.7, 0.1, -0.3], "k": 5}}}
|
||||
```
|
||||
|
||||
### ESP32 Provisioning
|
||||
|
||||
The ESP32's existing NVS provisioning system supports configuring the Seed as the target:
|
||||
|
||||
```bash
|
||||
python firmware/esp32-csi-node/provision.py \
|
||||
--port COM9 \
|
||||
--target-ip 192.168.1.20 \
|
||||
--target-port 5006 \
|
||||
--node-id 1
|
||||
```
|
||||
|
||||
Note: `--target-ip` is the host laptop (192.168.1.20), not the Seed IP, because the bridge runs on the host and forwards to the Seed via HTTPS (see Known Issue 4).
|
||||
|
||||
No firmware recompilation needed — the `stream_sender` module reads target IP/port from NVS at boot.
|
||||
|
||||
### Data Flow Rates
|
||||
|
||||
| Path | Rate | Size | Bandwidth |
|
||||
|------|------|------|-----------|
|
||||
| CSI capture → ring buffer | 100 Hz | ~400 B | 40 KB/s (internal) |
|
||||
| Edge DSP → sensing-server | 100 Hz | ~200 B | 20 KB/s (existing) |
|
||||
| Edge DSP → Seed features | 1 Hz | 48 B | 48 B/s (new) |
|
||||
| Seed ingest (batched) | 0.1 Hz | ~500 B | 50 B/s (HTTP) |
|
||||
| Seed kNN graph rebuild | 0.1 Hz | internal | — |
|
||||
| Seed witness chain | per batch | 32 B hash | — |
|
||||
|
||||
### Storage Budget
|
||||
|
||||
| Timeframe | Vectors/node | 4 nodes | RVF size | RAM |
|
||||
|-----------|-------------|---------|----------|-----|
|
||||
| 1 hour | 3,600 | 14,400 | ~580 KB | ~6 MB |
|
||||
| 24 hours | 86,400 | 345,600 | ~14 MB | ~140 MB |
|
||||
| 7 days | 604,800 | 2,419,200 | ~97 MB | exceeds |
|
||||
|
||||
**Compaction policy:** Run `POST /api/v1/store/compact` daily at 03:00, retaining only the last 24 hours of vectors. Archive older vectors to USB drive via `POST /api/v1/store/export` before compaction.
|
||||
|
||||
**Dimension reduction:** For deployments exceeding 100K vectors, reduce feature extraction rate to 0.1 Hz (one vector per 10 seconds) or increase compaction frequency.
|
||||
|
||||
## Validation Results
|
||||
|
||||
**Live hardware test performed 2026-04-02.**
|
||||
|
||||
### Hardware Under Test
|
||||
|
||||
| Component | Port | IP | Firmware | WiFi | RSSI |
|
||||
|-----------|------|----|----------|------|------|
|
||||
| ESP32-S3 (8MB) | COM9 | 192.168.1.105 | v0.5.2 | ruv.net (ch 5) | -34 dBm |
|
||||
| Cognitum Seed | USB | 169.254.42.1 / 192.168.1.109 | v0.8.1 | ruv.net | — |
|
||||
| Host laptop | — | 192.168.1.20 | — | ruv.net | — |
|
||||
|
||||
Seed device_id: `ecaf97dd-fc90-4b0e-b0e7-e9f896b9fbb6`. Pairing token issued to `wifi-densepose-claude`.
|
||||
|
||||
### Pipeline Validated
|
||||
|
||||
1. **UDP streaming** -- 211 packets captured in 15 seconds:
|
||||
- 196 raw CSI frames (magic `0xC5110001`)
|
||||
- 15 vitals frames (magic `0xC5110002`)
|
||||
|
||||
2. **Bridge pipeline** -- 20 vitals packets (`0xC5110002`) parsed, converted to 8-dim feature vectors via the bridge's `parse_vitals_packet()` fallback path, ingested in 4 batches of 5 vectors each (`--batch-size 5`). The native `0xC5110003` feature packet path is implemented in firmware but was not exercised in this validation run (firmware was v0.5.2; the `send_feature_vector()` addition requires a reflash).
|
||||
|
||||
3. **RVF ingest** -- All 20 vectors accepted by Seed. Epochs advanced 88 to 91. Witness chain verified valid (193 entries, SHA-256 chain intact).
|
||||
|
||||
4. **Seed sensors** -- BME280, PIR, reed switch, ADS1115, vibration sensor all present and healthy.
|
||||
|
||||
### Live Vital Signs Captured
|
||||
|
||||
| Metric | Observed Range | Expected | Notes |
|
||||
|--------|---------------|----------|-------|
|
||||
| Presence score | 1.41 -- 14.92 | 0.0 -- 1.0 | **Needs normalization** (see Known Issues) |
|
||||
| Motion energy | 1.41 -- 14.92 | 0.0 -- 1.0 | Same raw value as presence score |
|
||||
| Breathing rate | 19.8 -- 33.5 BPM | 12 -- 25 BPM | Plausible but slightly high |
|
||||
| Heart rate | 75.3 -- 99.1 BPM | 60 -- 100 BPM | Plausible range |
|
||||
| RSSI | -43 to -72 dBm | -30 to -80 dBm | Normal |
|
||||
| Fall detected | No | — | Correct (no falls occurred) |
|
||||
| n_persons | 4 | 1 | **Miscalibrated** (see Known Issues) |
|
||||
|
||||
### Known Issues Found
|
||||
|
||||
1. **`presence_score` exceeds 1.0 in vitals packets** -- Raw values range 1.41 to 14.92 in the vitals packet (`0xC5110002`). The bridge's vitals-to-feature conversion clamps to 1.0 for dim 0 and divides by 10.0 for dim 1 (`motion_energy / 10.0`), but dim 0 clamps without scaling. **Note:** The firmware's native feature vector (`0xC5110003`) already normalizes correctly by dividing `s_presence_score` by 10.0 (see `edge_processing.c` line 657). This issue only affects the vitals-packet fallback path in the bridge.
|
||||
|
||||
2. **`n_persons = 4` with 1 person present** -- The multi-person counting algorithm is miscalibrated for single-occupancy scenarios. The per-node state pipeline (ADR-068) may mitigate this when the baseline is properly trained, but the raw edge count is unreliable.
|
||||
|
||||
3. **Content-addressed vector IDs cause deduplication** -- Similar feature vectors hash to the same ID, causing the Seed to silently drop duplicates. **Fixed in bridge:** `seed_csi_bridge.py` now uses `_make_vector_id()` which generates a SHA-256 hash of `node_id:timestamp_us:seq_counter`, producing unique 32-bit IDs. This was observed during validation and fixed before the final test run.
|
||||
|
||||
4. **Bridge runs on host, not Seed** -- The ESP32 target IP must be the host laptop (192.168.1.20), not the Seed IP. The bridge script on the host forwards to the Seed via HTTPS. This adds a hop but avoids running a UDP listener on the Pi Zero 2 W.
|
||||
|
||||
5. **PIR GPIO read returned 404** -- `GET /api/v1/sensor/gpio/read?pin=6` returned 404. The PIR endpoint may require a different pin number or endpoint format. Ground-truth validation against PIR is deferred to Phase 3.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: ESP32 Feature Extraction (firmware change) -- DONE
|
||||
|
||||
Implemented as `send_feature_vector()` in `edge_processing.c` (lines 644-699) and `edge_feature_pkt_t` in `edge_processing.h` (lines 112-124). The function reads from static globals (`s_presence_score`, `s_motion_energy`, `s_breathing_bpm`, `s_heartrate_bpm`, subcarrier Welford variance, person tracker, fall flag, RSSI) and normalizes each dimension to 0.0-1.0 with clamping.
|
||||
|
||||
Called at the same 1 Hz cadence as `send_vitals_packet()` in Step 13 of the edge processing pipeline (line 855). The compressed frame magic was reassigned from `0xC5110003` to `0xC5110005` to free up `0xC5110003` for feature vectors (`EDGE_COMPRESSED_MAGIC` in `edge_processing.h` line 29).
|
||||
|
||||
### Phase 2: Seed Ingest Bridge (Python script on host) -- DONE
|
||||
|
||||
Implemented as `scripts/seed_csi_bridge.py`. The bridge:
|
||||
1. Listens on UDP port 5006 (configurable via `--udp-port`)
|
||||
2. Accepts all three packet formats: `0xC5110003` (ADR-069 features), `0xC5110002` (vitals, converted to 8-dim), and `0xC5110001` (raw CSI, minimal features)
|
||||
3. Generates unique vector IDs via SHA-256 hash of `node_id:timestamp:seq` (avoids content-addressed deduplication -- see Known Issue 3)
|
||||
4. Batches vectors (default 10, configurable via `--batch-size`) with time-based flush fallback (`--flush-interval`)
|
||||
5. POSTs to Seed's `/api/v1/store/ingest` with bearer token
|
||||
6. Supports `--validate` mode (kNN query + PIR comparison after each batch)
|
||||
7. Supports `--stats` mode (print Seed status, boundary, coherence, graph)
|
||||
8. Supports `--compact` mode (trigger store compaction)
|
||||
|
||||
### Phase 3: Validation & Ground Truth -- BLOCKED
|
||||
|
||||
Use the Seed's PIR sensor as ground truth for presence detection:
|
||||
1. Query PIR state: `GET /api/v1/sensor/gpio/read?pin=6`
|
||||
2. Compare with CSI presence score (feature dim 0)
|
||||
3. Log agreement/disagreement rate
|
||||
4. Use kNN to find historical vectors matching current PIR state → validate CSI accuracy
|
||||
|
||||
**Status:** The bridge implements `--validate` mode with PIR comparison (see `_run_validation()` in `seed_csi_bridge.py`). However, the PIR endpoint returned 404 during validation (Known Issue 5). This phase is blocked until the correct PIR API endpoint is identified.
|
||||
|
||||
### Phase 4: Multi-Node Mesh (addresses #348)
|
||||
|
||||
Deploy 3 ESP32 nodes, each sending feature vectors to the bridge host (which forwards to the Seed):
|
||||
- Node 1 (lobby): `--node-id 1 --target-ip 192.168.1.20 --target-port 5006`
|
||||
- Node 2 (hallway): `--node-id 2 --target-ip 192.168.1.20 --target-port 5006`
|
||||
- Node 3 (room): `--node-id 3 --target-ip 192.168.1.20 --target-port 5006`
|
||||
|
||||
All nodes target the host laptop (192.168.1.20) where the bridge script runs. The bridge batches and forwards all nodes' vectors to the Seed via HTTPS. The Seed's kNN graph naturally clusters vectors by node and by sensing state. Cross-node analysis via boundary fragility detects when a person moves between zones.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Bearer token** — All write operations require the pairing token. Token stored as SHA-256 hash on device.
|
||||
2. **TLS** — All API calls over HTTPS (port 8443) with device-provisioned CA certificate.
|
||||
3. **Witness chain** — Every ingest is cryptographically chained. Tampering detection via `POST /api/v1/witness/verify`.
|
||||
4. **Ed25519 attestation** — Device identity bound to hardware keypair. Attestation includes epoch, vector count, and witness head.
|
||||
5. **Anti-spoofing** — Sensor pipeline has entropy-based spoofing detection (min 0.5 bits entropy, streak threshold 3).
|
||||
6. **USB-only pairing** — Pairing window can only be opened from USB interface (169.254.42.1), not from WiFi.
|
||||
|
||||
## Hardware Bill of Materials
|
||||
|
||||
| Component | Port | IP | Cost |
|
||||
|-----------|------|----|------|
|
||||
| ESP32-S3 (8MB) | COM9 | 192.168.1.105 (DHCP) | ~$9 |
|
||||
| Cognitum Seed (Pi Zero 2W) | USB | 169.254.42.1 / 192.168.1.109 | ~$15 |
|
||||
| USB-C cable (data) | — | — | ~$3 |
|
||||
| **Total** | | | **~$27** |
|
||||
|
||||
### Seed Sensors (included)
|
||||
|
||||
| Sensor | Interface | Channels | Purpose |
|
||||
|--------|-----------|----------|---------|
|
||||
| Reed switch | GPIO 5 | 1 | Door/window state |
|
||||
| PIR motion | GPIO 6 | 1 | Motion ground truth |
|
||||
| Vibration | GPIO 13 | 1 | Structural vibration |
|
||||
| ADS1115 | I2C 0x48 | 4 | Analog inputs (extensible) |
|
||||
| BME280 | I2C 0x76 | 3 | Temperature, humidity, pressure |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| Pi Zero thermal throttling at sustained ingest | Medium | Performance degrades | Thermal governor already manages DVFS; 1 Hz ingest is minimal load |
|
||||
| WiFi congestion with ESP32 CSI + UDP | Low | Lost packets | Feature vectors are 48 bytes at 1 Hz; negligible vs CSI traffic |
|
||||
| RVF store exceeds RAM at high vector count | Medium | OOM | Compaction policy + dimension reduction + daily export |
|
||||
| Bearer token exposure | Low | Unauthorized writes | TLS encryption + USB-only pairing + token hashing |
|
||||
| ESP32 NVS corruption | Low | Config lost | NVS is wear-leveled flash with CRC; re-provision via USB |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- ESP32 CSI features become persistent, searchable, and cryptographically attested
|
||||
- kNN similarity search enables environment fingerprinting and anomaly detection
|
||||
- PIR + BME280 provide ground truth for CSI validation
|
||||
- MCP proxy enables AI assistants to query sensing state directly
|
||||
- Witness chain provides audit trail for healthcare/safety applications
|
||||
- Architecture aligns with Arena Physica's insight: store embeddings, not raw signals
|
||||
|
||||
### Negative
|
||||
- Additional firmware packet type (48 bytes, trivial)
|
||||
- Bridge script needed on Seed or host machine
|
||||
- Daily compaction required for long-running deployments
|
||||
- Bearer token must be managed (stored securely, rotated if compromised)
|
||||
|
||||
### Neutral
|
||||
- Existing sensing-server pipeline unchanged (ESP32 still sends to port 5005)
|
||||
- Seed's existing sensors continue operating independently
|
||||
- Target IP/port configurable via NVS provisioning (no recompilation for deployment changes)
|
||||
- Firmware recompilation needed once to add `send_feature_vector()` (Phase 1), but subsequent node deployments only need provisioning
|
||||
@@ -0,0 +1,203 @@
|
||||
# ADR-070: Self-Supervised Pretraining from Live ESP32 CSI + Cognitum Seed
|
||||
|
||||
| Field | Value |
|
||||
|------------|----------------------------------------------------------|
|
||||
| Status | Accepted |
|
||||
| Date | 2026-04-02 |
|
||||
| Authors | rUv, claude-flow |
|
||||
| Drivers | README limitation "No pre-trained model weights provided"|
|
||||
| Related | ADR-069 (Cognitum Seed pipeline), ADR-027 (MERIDIAN), ADR-024 (AETHER contrastive), ADR-015 (MM-Fi dataset) |
|
||||
|
||||
## Context
|
||||
|
||||
The README lists "No pre-trained model weights are provided; training from scratch is required" as a known limitation. Users must collect their own CSI dataset and train from scratch, which is a significant barrier to adoption.
|
||||
|
||||
We now have the infrastructure to generate pre-trained weights directly from live hardware:
|
||||
|
||||
- **2 ESP32-S3 nodes** (COM8 node_id=2 at 192.168.1.104, COM9 node_id=1 at 192.168.1.105) streaming CSI + vitals + 8-dim feature vectors at 1 Hz each
|
||||
- **Cognitum Seed** (Pi Zero 2 W) with RVF vector store, kNN search, witness chain, and environmental sensors (BME280, PIR, vibration)
|
||||
- **Recording API** in sensing-server (`POST /api/v1/recording/start`) that saves CSI frames to `.csi.jsonl`
|
||||
- **Self-supervised training** via `rapid_adapt.rs` (contrastive TTT + entropy minimization)
|
||||
- **AETHER contrastive embeddings** (ADR-024) for environment-independent representations
|
||||
|
||||
### Why Self-Supervised?
|
||||
|
||||
No cameras or labels are needed. The system learns from:
|
||||
|
||||
1. **Temporal coherence** — Frames close in time should have similar embeddings (positive pairs), frames far apart should differ (negative pairs)
|
||||
2. **Multi-node consistency** — The same person seen from 2 nodes should produce correlated features, different people should produce decorrelated features
|
||||
3. **Cognitum Seed ground truth** — PIR sensor, BME280 environment changes, and kNN cluster transitions provide weak supervision without human labeling
|
||||
4. **Physical constraints** — Breathing 6-30 BPM, heart rate 40-150 BPM, person count 0-4, RSSI physics
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a 4-phase pretraining pipeline that collects CSI from 2 ESP32 nodes, stores feature vectors in the Cognitum Seed, and produces distributable pre-trained weights.
|
||||
|
||||
### Phase 1: Data Collection (30 min)
|
||||
|
||||
Capture labeled scenarios using the sensing-server recording API and Cognitum Seed:
|
||||
|
||||
| Scenario | Duration | Label | Activity |
|
||||
|----------|----------|-------|----------|
|
||||
| Empty room | 5 min | `empty` | No one present, establish baseline |
|
||||
| 1 person stationary | 5 min | `1p-still` | Sit at desk, normal breathing |
|
||||
| 1 person walking | 5 min | `1p-walk` | Walk around room, varied paths |
|
||||
| 1 person varied | 5 min | `1p-varied` | Stand, sit, wave arms, turn |
|
||||
| 2 people | 5 min | `2p` | Both moving in room |
|
||||
| Transitions | 5 min | `transitions` | Enter/exit room, appear/disappear |
|
||||
|
||||
**Data rate per scenario:**
|
||||
- 2 nodes × 100 Hz CSI = 200 frames/sec = 60,000 frames per 5 min
|
||||
- 2 nodes × 1 Hz features = 2 vectors/sec = 600 vectors per 5 min
|
||||
- Total: 360,000 CSI frames + 3,600 feature vectors per collection run
|
||||
|
||||
**Cognitum Seed role:**
|
||||
- Stores all feature vectors with witness chain attestation
|
||||
- PIR sensor provides binary presence ground truth
|
||||
- BME280 tracks environmental conditions during collection
|
||||
- kNN graph clusters naturally emerge from the vector distribution
|
||||
|
||||
### Phase 2: Contrastive Pretraining
|
||||
|
||||
Train a contrastive encoder on the collected CSI data:
|
||||
|
||||
```
|
||||
Input: Raw CSI frame (128 subcarriers × 2 I/Q = 256 features)
|
||||
↓
|
||||
TCN temporal encoder (3 layers, kernel=7)
|
||||
↓
|
||||
Projection head → 128-dim embedding
|
||||
↓
|
||||
Contrastive loss (InfoNCE):
|
||||
positive: frames within 0.5s window from same node
|
||||
negative: frames >5s apart or from different scenario
|
||||
cross-node positive: same timestamp, different node
|
||||
```
|
||||
|
||||
**Self-supervised signals:**
|
||||
- Temporal adjacency (frames within 500ms = positive pair)
|
||||
- Cross-node agreement (same person seen from 2 viewpoints)
|
||||
- PIR consistency (embedding should cluster by PIR state)
|
||||
- Scenario boundary (embeddings should shift at label transitions)
|
||||
|
||||
### Phase 3: Downstream Head Training
|
||||
|
||||
Attach lightweight heads for each task:
|
||||
|
||||
| Head | Architecture | Output | Supervision |
|
||||
|------|-------------|--------|-------------|
|
||||
| Presence | Linear(128→1) + sigmoid | 0.0-1.0 | PIR sensor (free) |
|
||||
| Person count | Linear(128→4) + softmax | 0-3 people | Scenario labels |
|
||||
| Activity | Linear(128→4) + softmax | still/walk/varied/empty | Scenario labels |
|
||||
| Vital signs | Linear(128→2) | BR, HR (BPM) | ESP32 edge vitals |
|
||||
|
||||
### Phase 4: Package & Distribute
|
||||
|
||||
Produce distributable artifacts:
|
||||
|
||||
| Artifact | Format | Size | Description |
|
||||
|----------|--------|------|-------------|
|
||||
| `pretrained-encoder.onnx` | ONNX | ~2 MB | Contrastive encoder (TCN backbone) |
|
||||
| `pretrained-heads.onnx` | ONNX | ~100 KB | Task-specific heads |
|
||||
| `pretrained.rvf` | RVF | ~500 KB | RuVector format with metadata |
|
||||
| `room-profiles.json` | JSON | ~10 KB | Environment calibration profiles |
|
||||
| `collection-witness.json` | JSON | ~5 KB | Seed witness chain attestation proving data provenance |
|
||||
|
||||
Include in GitHub release alongside firmware binaries. Users download and run:
|
||||
|
||||
```bash
|
||||
# Use pre-trained model (no training needed)
|
||||
cargo run -p wifi-densepose-sensing-server -- --model pretrained.rvf --http-port 3000
|
||||
```
|
||||
|
||||
## Hardware Setup
|
||||
|
||||
```
|
||||
192.168.1.20 (Host laptop)
|
||||
┌──────────────────────────┐
|
||||
│ sensing-server │
|
||||
│ Recording API │
|
||||
│ Training pipeline │
|
||||
│ │
|
||||
│ seed_csi_bridge.py │
|
||||
│ Feature → Seed ingest │
|
||||
└────┬──────────┬───────────┘
|
||||
│ │
|
||||
UDP:5006 │ │ HTTPS:8443
|
||||
┌───────────────────┤ ├───────────────┐
|
||||
│ │ │ │
|
||||
▼ ▼ ▼ │
|
||||
┌──────────┐ ┌──────────┐ ┌──────────────┐ │
|
||||
│ ESP32 #1 │ │ ESP32 #2 │ │Cognitum Seed │◄───┘
|
||||
│ COM9 │ │ COM8 │ │ Pi Zero 2W │
|
||||
│ node=1 │ │ node=2 │ │ USB │
|
||||
│ .1.105 │ │ .1.104 │ │ .42.1/8443 │
|
||||
│ v0.5.4 │ │ v0.5.4 │ │ v0.8.1 │
|
||||
└──────────┘ └──────────┘ │ PIR, BME280 │
|
||||
│ RVF store │
|
||||
│ Witness chain│
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
## Data Collection Protocol
|
||||
|
||||
### Step 1: Start Seed ingest (background)
|
||||
|
||||
```bash
|
||||
export SEED_TOKEN="your-token"
|
||||
python scripts/seed_csi_bridge.py \
|
||||
--seed-url https://169.254.42.1:8443 --token "$SEED_TOKEN" \
|
||||
--udp-port 5006 --batch-size 10 --validate &
|
||||
```
|
||||
|
||||
### Step 2: Start sensing-server with recording
|
||||
|
||||
```bash
|
||||
cargo run -p wifi-densepose-sensing-server -- \
|
||||
--source esp32 --udp-port 5006 --http-port 3000
|
||||
```
|
||||
|
||||
### Step 3: Record each scenario
|
||||
|
||||
```bash
|
||||
# Empty room (leave room for 5 min)
|
||||
curl -X POST http://localhost:3000/api/v1/recording/start \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"session_name":"pretrain-empty","label":"empty","duration_secs":300}'
|
||||
|
||||
# 1 person stationary (sit at desk for 5 min)
|
||||
curl -X POST http://localhost:3000/api/v1/recording/start \
|
||||
-d '{"session_name":"pretrain-1p-still","label":"1p-still","duration_secs":300}'
|
||||
|
||||
# ... repeat for each scenario
|
||||
```
|
||||
|
||||
### Step 4: Verify with Seed
|
||||
|
||||
```bash
|
||||
python scripts/seed_csi_bridge.py --token "$SEED_TOKEN" --stats
|
||||
# Should show 3,600+ vectors from the collection run
|
||||
```
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| 2 nodes insufficient for spatial diversity | Medium | Lower pretraining quality | Place nodes 3-5m apart at different heights |
|
||||
| PIR sensor has limited range | Low | Weak presence labels | BME280 temp changes + kNN clusters as backup |
|
||||
| Contrastive pretraining collapses | Low | Useless embeddings | Temperature scheduling, hard negative mining |
|
||||
| Model too large for ESP32 inference | N/A | N/A | Inference on host/Seed, not on ESP32 |
|
||||
| Room-specific overfitting | Medium | Poor generalization | MERIDIAN domain randomization (ADR-027), LoRA adaptation |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Users get working model out of the box — no training needed
|
||||
- Witness chain proves data provenance (when/where/which hardware)
|
||||
- Pre-trained encoder transfers to new environments via LoRA fine-tuning
|
||||
- Removes the #1 adoption barrier from the README
|
||||
|
||||
### Negative
|
||||
- 30 min of manual data collection per pretraining run
|
||||
- Pre-trained weights are room-specific without adaptation
|
||||
- ONNX runtime dependency for inference
|
||||
@@ -0,0 +1,408 @@
|
||||
# ADR-071: ruvllm Training Pipeline for CSI Sensing Models
|
||||
|
||||
- **Status**: Proposed
|
||||
- **Date**: 2026-04-02
|
||||
- **Deciders**: ruv
|
||||
- **Relates to**: ADR-069 (Cognitum Seed CSI Pipeline), ADR-070 (Self-Supervised Pretraining), ADR-024 (Contrastive CSI Embedding / AETHER), ADR-016 (RuVector Training Pipeline)
|
||||
|
||||
## Context
|
||||
|
||||
The WiFi-DensePose project needs a training pipeline to convert collected CSI data
|
||||
(`.csi.jsonl` frames from ESP32 nodes) into deployable models for presence detection,
|
||||
activity classification, and vital sign estimation.
|
||||
|
||||
Previous ADRs established the data collection protocol (ADR-070) and Cognitum Seed
|
||||
inference target (ADR-069). What was missing was the actual training, refinement,
|
||||
quantization, and export pipeline connecting raw CSI recordings to deployable models.
|
||||
|
||||
### Why ruvllm instead of PyTorch
|
||||
|
||||
| Criterion | ruvllm | PyTorch | ONNX Runtime |
|
||||
|-----------|--------|---------|--------------|
|
||||
| Runtime dependency | Node.js only | Python + CUDA + pip | C++ runtime |
|
||||
| Install size | ~5 MB (npm) | ~2 GB (torch+cuda) | ~50 MB |
|
||||
| SONA adaptation | <1ms native | N/A | N/A |
|
||||
| Quantization | 2/4/8-bit TurboQuant | INT8/FP16 (separate tool) | INT8 only |
|
||||
| LoRA fine-tuning | Built-in LoraAdapter | Requires PEFT library | N/A |
|
||||
| EWC protection | Built-in EwcManager | Manual implementation | N/A |
|
||||
| SafeTensors export | Native SafeTensorsWriter | Via safetensors library | N/A |
|
||||
| Contrastive training | Built-in ContrastiveTrainer | Manual triplet loss | N/A |
|
||||
| Edge deployment | ESP32, Pi Zero, browser | GPU servers only | ARM (limited) |
|
||||
| M4 Pro performance | 88-135 tok/s native | ~30 tok/s (MPS) | ~50 tok/s |
|
||||
| Ecosystem integration | RuVector, Cognitum Seed | Standalone | Standalone |
|
||||
|
||||
The ruvllm package (`@ruvector/ruvllm` v2.5.4) provides the complete training
|
||||
lifecycle in a single dependency: contrastive pretraining, task head training,
|
||||
LoRA refinement, EWC consolidation, quantization, and SafeTensors/RVF export.
|
||||
No Python dependency means the entire pipeline runs on the same Node.js runtime
|
||||
as the Cognitum Seed inference engine.
|
||||
|
||||
## Decision
|
||||
|
||||
Use ruvllm's `ContrastiveTrainer`, `TrainingPipeline`, `LoraAdapter`, `EwcManager`,
|
||||
`SafeTensorsWriter`, and `ModelExporter` for the complete CSI model training lifecycle.
|
||||
|
||||
### Training Phases
|
||||
|
||||
The pipeline executes five sequential phases:
|
||||
|
||||
#### Phase 1: Contrastive Pretraining
|
||||
|
||||
Learns an embedding space where temporally and spatially similar CSI states are close
|
||||
and dissimilar states are far apart.
|
||||
|
||||
- **Encoder architecture**: 8-dim CSI feature vector -> 64-dim hidden (ReLU) -> 128-dim embedding (L2-normalized)
|
||||
- **Loss functions**: Triplet loss (margin=0.3) + InfoNCE (temperature=0.07)
|
||||
- **Triplet strategies**:
|
||||
- Temporal positive: frames within 1 second (same environment state)
|
||||
- Temporal negative: frames >30 seconds apart (different state)
|
||||
- Cross-node positive: same timestamp from different ESP32 nodes (same person, different viewpoint)
|
||||
- Cross-node negative: different timestamp + different node
|
||||
- Hard negatives: frames near motion energy transition boundaries
|
||||
- **Hyperparameters**: 20 epochs, batch size 32, hard negative ratio 0.7
|
||||
- **Implementation**: `ContrastiveTrainer.addTriplet()` + `.train()`
|
||||
|
||||
#### Phase 2: Task Head Training
|
||||
|
||||
Trains supervised heads on top of the frozen embedding for specific sensing tasks.
|
||||
|
||||
- **Presence head**: 128 -> 1 (sigmoid), threshold at presence_score > 0.3
|
||||
- **Activity head**: 128 -> 3 (softmax: still/moving/empty), derived from motion_energy thresholds
|
||||
- **Vitals head**: 128 -> 2 (linear: breathing BPM, heart rate BPM), normalized targets
|
||||
- **Implementation**: `TrainingPipeline.addData()` + `.train()` with cosine LR scheduler,
|
||||
early stopping (patience=5), and quality-weighted MSE loss
|
||||
|
||||
#### Phase 3: LoRA Refinement
|
||||
|
||||
Per-node LoRA adapters for room-specific adaptation without forgetting the base model.
|
||||
|
||||
- **Configuration**: rank=4, alpha=8, dropout=0.1
|
||||
- **Per-node training**: Each ESP32 node gets its own LoRA adapter trained on
|
||||
node-specific data with reduced learning rate (0.5x base)
|
||||
- **Implementation**: `LoraManager.create()` for each node, `TrainingPipeline` with
|
||||
`LoraAdapter` passed to constructor
|
||||
|
||||
#### Phase 4: Quantization (TurboQuant)
|
||||
|
||||
Reduces model size for edge deployment with minimal quality loss.
|
||||
|
||||
| Bit Width | Compression | Typical RMSE | Target Device |
|
||||
|-----------|-------------|-------------|---------------|
|
||||
| 8-bit | 4x | <0.001 | Cognitum Seed (Pi Zero) |
|
||||
| 4-bit | 8x | <0.01 | Standard edge inference |
|
||||
| 2-bit | 16x | <0.05 | ESP32-S3 feature extraction |
|
||||
|
||||
- **Method**: Uniform affine quantization with scale/zero-point per tensor
|
||||
- **Quality validation**: RMSE between original fp32 and dequantized weights
|
||||
|
||||
#### Phase 5: EWC Consolidation
|
||||
|
||||
Elastic Weight Consolidation prevents catastrophic forgetting when the model
|
||||
is later fine-tuned on new room data or updated CSI conditions.
|
||||
|
||||
- **Fisher information**: Computed from training data gradients
|
||||
- **Lambda**: 2000 (base), 3000 (per-node)
|
||||
- **Tasks registered**: Base pretraining + one per ESP32 node
|
||||
- **Implementation**: `EwcManager.registerTask()` for each training phase
|
||||
|
||||
### Data Pipeline
|
||||
|
||||
```
|
||||
.csi.jsonl files
|
||||
|
|
||||
v
|
||||
Parse frames: feature (8-dim), vitals, raw CSI
|
||||
|
|
||||
v
|
||||
Generate contrastive triplets (temporal, cross-node, hard negatives)
|
||||
|
|
||||
v
|
||||
Encode through CsiEncoder (8 -> 64 -> 128)
|
||||
|
|
||||
v
|
||||
Phase 1: ContrastiveTrainer (triplet + InfoNCE loss)
|
||||
|
|
||||
v
|
||||
Phase 2: TrainingPipeline (presence + activity + vitals heads)
|
||||
|
|
||||
v
|
||||
Phase 3: LoRA per-node refinement
|
||||
|
|
||||
v
|
||||
Phase 4: TurboQuant (2/4/8-bit quantization)
|
||||
|
|
||||
v
|
||||
Phase 5: EWC consolidation
|
||||
|
|
||||
v
|
||||
Export: SafeTensors, JSON config, RVF manifest, per-node LoRA adapters
|
||||
```
|
||||
|
||||
### Export Formats
|
||||
|
||||
| Format | File | Consumer |
|
||||
|--------|------|----------|
|
||||
| SafeTensors | `model.safetensors` | HuggingFace ecosystem, general inference |
|
||||
| JSON config | `config.json` | Model loading metadata |
|
||||
| JSON model | `model.json` | Full model state for Node.js loading |
|
||||
| Quantized binaries | `quantized/model-q{2,4,8}.bin` | Edge deployment |
|
||||
| Per-node LoRA | `lora/node-{id}.json` | Room-specific adaptation |
|
||||
| RVF manifest | `model.rvf.jsonl` | Cognitum Seed ingest (ADR-069) |
|
||||
| Training metrics | `training-metrics.json` | Dashboards, CI validation |
|
||||
|
||||
### Hardware Targets
|
||||
|
||||
| Device | Role | Quantization | Expected Latency |
|
||||
|--------|------|-------------|-----------------|
|
||||
| Mac Mini M4 Pro | Training (primary) | fp32 | <5 min total |
|
||||
| Cognitum Seed Pi Zero | Inference | 4-bit / 8-bit | <10 ms per frame |
|
||||
| ESP32-S3 | Feature extraction only | 2-bit (encoder weights) | <5 ms per frame |
|
||||
| Browser (WASM) | Visualization | 4-bit | <20 ms per frame |
|
||||
|
||||
### Performance Targets
|
||||
|
||||
| Metric | Target | Measured |
|
||||
|--------|--------|----------|
|
||||
| Training time (5,783 frames, M4 Pro) | <5 min | TBD |
|
||||
| Inference latency (M4 Pro) | <1 ms | TBD |
|
||||
| Inference latency (Pi Zero) | <10 ms | TBD |
|
||||
| SONA adaptation | <1 ms | <0.05 ms (ruvllm spec) |
|
||||
| Presence detection accuracy | >85% | TBD |
|
||||
| 4-bit quality loss (RMSE) | <0.01 | TBD |
|
||||
| 2-bit quality loss (RMSE) | <0.05 | TBD |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **Zero Python dependency**: The entire training and inference pipeline runs on
|
||||
Node.js, eliminating Python/CUDA/pip dependency management on training and
|
||||
deployment targets.
|
||||
- **Integrated lifecycle**: Contrastive pretraining, task heads, LoRA refinement,
|
||||
EWC consolidation, and quantization in a single script using one library.
|
||||
- **Edge-first**: 2-bit quantization enables running the encoder on ESP32-S3.
|
||||
4-bit quantization fits comfortably on Cognitum Seed Pi Zero.
|
||||
- **Continual learning**: EWC protection means the model can be updated with new
|
||||
room data without losing previously learned patterns.
|
||||
- **Per-node adaptation**: LoRA adapters allow room-specific fine-tuning with
|
||||
minimal storage overhead (rank-4 adapter ~2KB per node).
|
||||
- **HuggingFace compatibility**: SafeTensors export enables sharing models on the
|
||||
HuggingFace Hub and loading in other frameworks.
|
||||
- **Reproducibility**: Seeded encoder initialization and deterministic data pipeline
|
||||
ensure reproducible training runs.
|
||||
|
||||
### Negative
|
||||
|
||||
- **No GPU acceleration**: ruvllm's JS training loop does not use GPU compute.
|
||||
For the small model sizes in CSI sensing (8->64->128), this is acceptable
|
||||
(~seconds on M4 Pro), but would not scale to large vision models.
|
||||
- **Simplified backpropagation**: The LoRA backward pass and contrastive training
|
||||
use approximate gradient updates rather than full automatic differentiation.
|
||||
Sufficient for the target model sizes but not equivalent to PyTorch autograd.
|
||||
- **Quantization is post-training only**: No quantization-aware training (QAT).
|
||||
For 4-bit and 8-bit this produces acceptable quality loss; 2-bit may need
|
||||
QAT in future if quality degrades.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Quality ceiling**: The simplified training may produce lower accuracy than a
|
||||
PyTorch-trained equivalent. Mitigated by: (a) the model is small enough that
|
||||
the training loop converges quickly, (b) SONA adaptation can compensate at
|
||||
inference time, (c) we can switch to PyTorch for training only if needed
|
||||
while keeping ruvllm for inference.
|
||||
- **ruvllm API stability**: The library is at v2.5.4 with active development.
|
||||
Mitigated by vendoring the package in `vendor/ruvector/npm/packages/ruvllm/`.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Scripts
|
||||
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/train-ruvllm.js` | Full 5-phase training pipeline |
|
||||
| `scripts/benchmark-ruvllm.js` | Model benchmarking (latency, quality, accuracy) |
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Train on collected CSI data
|
||||
node scripts/train-ruvllm.js \
|
||||
--data data/recordings/pretrain-1775182186.csi.jsonl \
|
||||
--output models/csi-v1 \
|
||||
--epochs 20
|
||||
|
||||
# Train with benchmark
|
||||
node scripts/train-ruvllm.js \
|
||||
--data data/recordings/pretrain-*.csi.jsonl \
|
||||
--output models/csi-v1 \
|
||||
--benchmark
|
||||
|
||||
# Standalone benchmark
|
||||
node scripts/benchmark-ruvllm.js \
|
||||
--model models/csi-v1 \
|
||||
--data data/recordings/pretrain-*.csi.jsonl \
|
||||
--samples 5000 \
|
||||
--json
|
||||
```
|
||||
|
||||
### Output Structure
|
||||
|
||||
```
|
||||
models/csi-v1/
|
||||
model.safetensors # SafeTensors (HuggingFace compatible)
|
||||
config.json # Model configuration
|
||||
model.json # Full JSON model state
|
||||
model.rvf.jsonl # RVF manifest for Cognitum Seed
|
||||
training-metrics.json # Training loss curves, timing, config
|
||||
contrastive/
|
||||
triplets.jsonl # Contrastive training pairs
|
||||
triplets.csv # CSV format for analysis
|
||||
embeddings.json # Embedding matrices
|
||||
quantized/
|
||||
model-q2.bin # 2-bit quantized (ESP32 edge)
|
||||
model-q4.bin # 4-bit quantized (Pi Zero default)
|
||||
model-q8.bin # 8-bit quantized (high quality)
|
||||
lora/
|
||||
node-1.json # LoRA adapter for ESP32 node 1
|
||||
node-2.json # LoRA adapter for ESP32 node 2
|
||||
```
|
||||
|
||||
## Camera-Free Supervision
|
||||
|
||||
### Motivation
|
||||
|
||||
Traditional WiFi-based pose estimation (WiFlow, Person-in-WiFi) requires camera-supervised
|
||||
training: a camera captures ground-truth poses during CSI collection, and the model learns
|
||||
to map CSI to those poses. This creates a deployment paradox — the camera is needed for
|
||||
training but the whole point of WiFi sensing is to avoid cameras.
|
||||
|
||||
The camera-free pipeline (`scripts/train-camera-free.js`) replaces camera supervision with
|
||||
10 sensor signals from the Cognitum Seed and 2 ESP32 nodes, generating weak labels through
|
||||
sensor fusion.
|
||||
|
||||
### 10 Supervision Signals (No Camera)
|
||||
|
||||
| # | Signal | Source | Provides |
|
||||
|---|--------|--------|----------|
|
||||
| 1 | PIR sensor | Seed GPIO 6 | Binary presence ground truth |
|
||||
| 2 | BME280 temperature | Seed I2C 0x76 | Occupancy proxy (temp rises with people) |
|
||||
| 3 | BME280 humidity | Seed I2C 0x76 | Breathing confirmation / zone |
|
||||
| 4 | Cross-node RSSI | 2 ESP32 nodes | Rough XY position (differential triangulation) |
|
||||
| 5 | Vitals stability | ESP32 CSI | HR/BR variance indicates activity level |
|
||||
| 6 | Temporal CSI patterns | ESP32 CSI | Periodic=walking, stable=sitting, flat=empty |
|
||||
| 7 | kNN cluster labels | Seed vector store | Natural groupings in embedding space |
|
||||
| 8 | Boundary fragility | Seed Stoer-Wagner | Regime change detection (entry/exit/activity) |
|
||||
| 9 | Reed switch | Seed GPIO 5 | Door open/close events |
|
||||
| 10 | Vibration sensor | Seed GPIO 13 | Footstep detection |
|
||||
|
||||
### Camera-Free Training Phases
|
||||
|
||||
The pipeline extends the base 5 phases with camera-free-specific phases:
|
||||
|
||||
```
|
||||
Phase 0: Multi-Modal Data Collection
|
||||
├── UDP port 5006 → ESP32 CSI features + vitals
|
||||
├── HTTPS → Seed sensor embeddings (45-dim, every 100ms)
|
||||
├── HTTPS → Seed boundary/coherence (every 10s)
|
||||
└── Build synchronized MultiModalFrame timeline
|
||||
|
||||
Phase 1: Weak Label Generation
|
||||
├── Presence: PIR || CSI_presence > 0.3 || temp_rising > 0.1°C/min
|
||||
├── Position: RSSI differential → 5×5 grid (25 zones)
|
||||
├── Activity: CSI variance + FFT periodicity → stationary/walking/gesture/empty
|
||||
├── Occupancy: max(node1_persons, node2_persons) validated by temp
|
||||
├── Body region: upper/lower subcarrier groups → which body part moves
|
||||
├── Entry/exit: reed_switch + PIR transition + boundary fragility spike
|
||||
├── Breathing zone: humidity change rate → person location
|
||||
└── Pose proxy: 5-keypoint coarse pose from RSSI + subcarrier asymmetry + vibration
|
||||
|
||||
Phase 2: Enhanced Contrastive Pretraining
|
||||
├── Base triplets (temporal, cross-node, transition, scenario boundary)
|
||||
├── Sensor-verified negatives: PIR=0 vs PIR=1 must differ
|
||||
├── Activity boundary: before/after fragility spike must differ
|
||||
└── Cross-modal: CSI embedding ≈ Seed embedding for same state
|
||||
|
||||
Phase 3: Pose Proxy Training (5-keypoint)
|
||||
├── Head: RSSI centroid between 2 nodes
|
||||
├── Hands: per-subcarrier variance asymmetry (left/right from 2 nodes)
|
||||
├── Feet: vibration sensor + RSSI ground reflection
|
||||
└── Skeleton physics constraints (anthropometric bone length limits)
|
||||
|
||||
Phase 4: 17-Keypoint Interpolation
|
||||
├── Shoulders = 0.3 × head + 0.7 × hands
|
||||
├── Elbows = midpoint(shoulder, hand)
|
||||
├── Hips = midpoint(head, feet)
|
||||
├── Knees = midpoint(hip, foot)
|
||||
├── Face = derived from head position
|
||||
└── Iterative bone length constraint projection (3 iterations)
|
||||
|
||||
Phase 5: Self-Refinement Loop (3 rounds)
|
||||
├── Run inference on all collected data
|
||||
├── Keep predictions where temporal consistency confidence > 0.8
|
||||
├── Use as pseudo-labels for next training round
|
||||
└── Decaying learning rate per round (diminishing returns)
|
||||
```
|
||||
|
||||
### Seed API Endpoints Used
|
||||
|
||||
| Endpoint | Data | Collection Rate |
|
||||
|----------|------|----------------|
|
||||
| `GET /api/v1/sensor/stream` | SSE sensor readings | Continuous (100ms) |
|
||||
| `GET /api/v1/sensor/embedding/latest` | 45-dim sensor embedding | Per-frame |
|
||||
| `GET /api/v1/boundary` | Fragility score | Every 10s |
|
||||
| `GET /api/v1/coherence/profile` | Temporal phase boundaries | Every 10s |
|
||||
| `GET /api/v1/store/query` | kNN similarity search | On demand |
|
||||
| `POST /api/v1/boundary/recompute` | Trigger analysis | On regime change |
|
||||
|
||||
### Graceful Degradation
|
||||
|
||||
The pipeline works with or without the Cognitum Seed:
|
||||
|
||||
| Mode | Signals | Pose Quality |
|
||||
|------|---------|-------------|
|
||||
| Full (Seed + 2 ESP32) | 10 signals | 5-keypoint trained, 17-keypoint interpolated |
|
||||
| CSI-only (2 ESP32) | 3 signals (RSSI, vitals, temporal) | Coarser position/activity only |
|
||||
| Single node | 2 signals (vitals, temporal) | Presence + activity only |
|
||||
|
||||
When the Seed API is unreachable, the pipeline automatically falls back to
|
||||
CSI-only training, producing the same output format (SafeTensors, HuggingFace,
|
||||
quantized) with reduced label quality.
|
||||
|
||||
### Output Format
|
||||
|
||||
Same as the base pipeline (SafeTensors + HuggingFace compatible), plus:
|
||||
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `pose-decoder.json` | 5-keypoint pose decoder weights |
|
||||
| `model.rvf.jsonl` | Extended with `camera_free_supervision` record |
|
||||
| `training-metrics.json` | Includes weak label stats and multi-modal triplet counts |
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Full pipeline with Seed
|
||||
node scripts/train-camera-free.js \
|
||||
--data data/recordings/pretrain-*.csi.jsonl \
|
||||
--seed-url https://169.254.42.1:8443 \
|
||||
--output models/csi-camerafree-v1
|
||||
|
||||
# CSI-only (no Seed)
|
||||
node scripts/train-camera-free.js \
|
||||
--data data/recordings/pretrain-*.csi.jsonl \
|
||||
--no-seed \
|
||||
--output models/csi-camerafree-v1
|
||||
|
||||
# With benchmark
|
||||
node scripts/train-camera-free.js \
|
||||
--data data/recordings/*.csi.jsonl \
|
||||
--benchmark
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- [ruvllm source](vendor/ruvector/npm/packages/ruvllm/) — v2.5.4
|
||||
- [ADR-069](ADR-069-cognitum-seed-csi-pipeline.md) — Cognitum Seed CSI Pipeline
|
||||
- [ADR-070](ADR-070-self-supervised-pretraining.md) — Self-Supervised Pretraining Protocol
|
||||
- [ADR-024](ADR-024-contrastive-csi-embedding.md) — Contrastive CSI Embedding / AETHER
|
||||
- [ADR-016](ADR-016-ruvector-training-pipeline.md) — RuVector Training Pipeline Integration
|
||||
@@ -0,0 +1,238 @@
|
||||
# ADR-072: WiFlow Pose Estimation Architecture
|
||||
|
||||
- **Status**: Proposed
|
||||
- **Date**: 2026-04-02
|
||||
- **Deciders**: ruv
|
||||
- **Relates to**: ADR-071 (ruvllm Training Pipeline), ADR-070 (Self-Supervised Pretraining), ADR-024 (Contrastive CSI Embedding / AETHER), ADR-069 (Cognitum Seed CSI Pipeline)
|
||||
|
||||
## Context
|
||||
|
||||
The WiFi-DensePose project needs a neural architecture that can convert raw CSI amplitude
|
||||
data into 17-keypoint COCO pose estimates. The existing `train-ruvllm.js` pipeline uses a
|
||||
simple 2-layer FC encoder (8 -> 64 -> 128) that produces contrastive embeddings for
|
||||
presence detection but cannot output spatial keypoint coordinates.
|
||||
|
||||
We evaluated published WiFi-based pose estimation architectures:
|
||||
|
||||
| Architecture | Params | Input | Key Innovation | Publication |
|
||||
|-------------|--------|-------|---------------|-------------|
|
||||
| **WiFlow** | 4.82M | 540x20 | TCN + AsymConv + Axial Attention | arXiv:2602.08661 |
|
||||
| WiPose | 11.2M | 3x3x30x20 | 3D CNN + heatmap regression | CVPR 2021 |
|
||||
| MetaFi++ | 8.6M | 114x30x20 | Transformer + meta-learning | NeurIPS 2023 |
|
||||
| Person-in-WiFi 3D | 15.3M | Multi-antenna | Deformable attention + 3D | CVPR 2024 |
|
||||
|
||||
WiFlow is the lightest published SOTA architecture, designed specifically for commercial
|
||||
WiFi hardware. Its key advantage is operating on CSI amplitude only (no phase), which
|
||||
is critical for ESP32-S3 where phase calibration is unreliable.
|
||||
|
||||
### Why WiFlow
|
||||
|
||||
1. **Lightest SOTA**: 4.82M parameters at original scale; our adaptation targets ~2.5M
|
||||
2. **Amplitude-only**: Discards phase, which is noisy on consumer hardware
|
||||
3. **Published architecture**: Fully specified in arXiv:2602.08661, reproducible
|
||||
4. **Temporal modeling**: TCN with dilated causal convolutions captures motion dynamics
|
||||
5. **Efficient attention**: Axial attention reduces O(H^2W^2) to O(H^2W + HW^2)
|
||||
6. **Proven on commercial WiFi**: Validated on commodity Intel 5300 and Atheros hardware
|
||||
|
||||
## Decision
|
||||
|
||||
Implement the WiFlow architecture in pure JavaScript (ruvllm native) with the following
|
||||
adaptations for our ESP32 single TX/RX deployment.
|
||||
|
||||
### Architecture Overview
|
||||
|
||||
```
|
||||
CSI Amplitude [128, 20]
|
||||
|
|
||||
Stage 1: TCN (Dilated Causal Conv)
|
||||
dilation = (1, 2, 4, 8), kernel = 7
|
||||
128 -> 256 -> 192 -> 128 channels
|
||||
|
|
||||
Stage 2: Asymmetric Conv Encoder
|
||||
1xk conv (k=3), stride (1,2)
|
||||
[1, 128, 20] -> [256, 8, 20]
|
||||
|
|
||||
Stage 3: Axial Self-Attention
|
||||
Width (temporal): 8 heads
|
||||
Height (feature): 8 heads
|
||||
|
|
||||
Decoder: Adaptive Avg Pool + Linear
|
||||
[256, 8, 20] -> pool -> [2048] -> [17, 2]
|
||||
|
|
||||
17 COCO Keypoints [x, y] in [0, 1]
|
||||
```
|
||||
|
||||
### Our Adaptation vs Original WiFlow
|
||||
|
||||
| Aspect | WiFlow Original | Our Adaptation | Reason |
|
||||
|--------|----------------|----------------|--------|
|
||||
| Input channels | 540 (18 links x 30 SC) | 128 (1 TX x 1 RX x 128 SC) | Single ESP32 link |
|
||||
| Time steps | 20 | 20 | Same |
|
||||
| TCN channels | 540 -> 256 -> 128 -> 64 | 128 -> 256 -> 192 -> 128 | Proportional reduction |
|
||||
| Spatial blocks | 4 (stride 2) | 4 (stride 2) | Same |
|
||||
| Attention heads | 8 | 8 | Same |
|
||||
| Parameters | 4.82M | ~1.8M | Fewer input channels |
|
||||
| Input type | Amplitude only | Amplitude only | Same |
|
||||
| Output | 17 x 2 | 17 x 2 | Same |
|
||||
|
||||
### Parameter Budget Breakdown
|
||||
|
||||
| Stage | Parameters | % of Total |
|
||||
|-------|-----------|------------|
|
||||
| TCN (4 blocks, k=7, d=1,2,4,8) | ~969K | 54% |
|
||||
| Asymmetric Conv (4 blocks, 1x3, stride 2) | ~174K | 10% |
|
||||
| Axial Attention (width + height, 8 heads) | ~592K | 33% |
|
||||
| Pose Decoder (pool + linear -> 17x2) | ~70K | 4% |
|
||||
| **Total** | **~1.8M** | **100%** |
|
||||
|
||||
### Loss Function
|
||||
|
||||
```
|
||||
L = L_H + 0.2 * L_B
|
||||
|
||||
L_H = SmoothL1(predicted, target, beta=0.1)
|
||||
L_B = (1/14) * sum_b (bone_length_b - prior_b)^2
|
||||
```
|
||||
|
||||
14 bone connections enforce anatomical constraints:
|
||||
- Nose-eye (x2): 0.06
|
||||
- Eye-ear (x2): 0.06
|
||||
- Shoulder-elbow (x2): 0.15
|
||||
- Elbow-wrist (x2): 0.13
|
||||
- Shoulder-hip (x2): 0.26
|
||||
- Hip-knee (x2): 0.25
|
||||
- Knee-ankle (x2): 0.25
|
||||
- Shoulder width: 0.20
|
||||
|
||||
All lengths normalized to person height.
|
||||
|
||||
### Training Strategy (Camera-Free Pipeline)
|
||||
|
||||
Since we have no ground-truth pose labels from cameras, training proceeds in three phases:
|
||||
|
||||
#### Phase 1: Contrastive Pretraining
|
||||
- Temporal triplets: adjacent windows are positive pairs, distant windows are negative
|
||||
- Cross-node triplets: same-time windows from different ESP32 nodes are positive
|
||||
- Uses ruvllm `ContrastiveTrainer` with triplet + InfoNCE loss
|
||||
- Learns a representation where similar CSI states cluster together
|
||||
|
||||
#### Phase 2: Pose Proxy Training
|
||||
- Generate coarse pose proxies from vitals data:
|
||||
- Person detected (presence > 0.3): place standing skeleton at center
|
||||
- High motion: perturb limb positions proportional to motion energy
|
||||
- Breathing: add micro-oscillation to torso keypoints
|
||||
- Train with SmoothL1 + bone constraint loss
|
||||
- Confidence-weighted updates (higher presence = stronger gradient)
|
||||
|
||||
#### Phase 3: Self-Refinement (Future)
|
||||
- Multi-node consistency: same person seen from different nodes should produce
|
||||
consistent pose after geometric transform
|
||||
- Temporal smoothness: adjacent frames should produce similar poses
|
||||
- Bone constraint tightening: gradually reduce tolerance
|
||||
|
||||
### Integration with Existing Pipeline
|
||||
|
||||
```
|
||||
train-ruvllm.js (ADR-071) train-wiflow.js (ADR-072)
|
||||
| |
|
||||
| 8-dim features | 128-dim raw CSI amplitude
|
||||
| -> 128-dim embedding | -> 17x2 keypoint coordinates
|
||||
| -> presence/activity/vitals | -> bone-constrained pose
|
||||
| |
|
||||
+-- ContrastiveTrainer -----+------+
|
||||
+-- TrainingPipeline -------+------+
|
||||
+-- LoRA per-node ----------+------+
|
||||
+-- TurboQuant quantize ----+------+
|
||||
+-- SafeTensors export -----+------+
|
||||
```
|
||||
|
||||
Both pipelines share the ruvllm infrastructure; WiFlow adds the deeper architecture
|
||||
for direct pose regression while the simple encoder handles embedding tasks.
|
||||
|
||||
### Performance Targets
|
||||
|
||||
| Metric | Target | Notes |
|
||||
|--------|--------|-------|
|
||||
| PCK@20 | > 80% | On lab data with 2+ nodes |
|
||||
| Forward latency | < 50ms | Pi Zero 2W at INT8 |
|
||||
| Model size (INT8) | < 2 MB | TurboQuant |
|
||||
| Bone violation rate | < 10% | 50% tolerance |
|
||||
| Temporal jitter | < 3cm | Exponential smoothing |
|
||||
|
||||
### Risk Assessment
|
||||
|
||||
| Risk | Severity | Mitigation |
|
||||
|------|----------|------------|
|
||||
| Single TX/RX has less spatial info than 18 links | High | 2-node multi-static compensates; cross-node fusion from ADR-029 |
|
||||
| Camera-free labels are coarse | Medium | Bone constraints enforce anatomy; contrastive pretrain provides structure |
|
||||
| Pure JS too slow for real-time | Medium | INT8 quantization; axial attention is O(H^2W+HW^2) not O(H^2W^2) |
|
||||
| Overfitting with ~5K frames | Medium | Temporal augmentation + noise + cross-node interpolation |
|
||||
| Phase not available (amplitude-only) | Low | WiFlow was designed amplitude-only; not a limitation |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Proven SOTA architecture adapted to our hardware constraints
|
||||
- Pure JavaScript implementation runs everywhere ruvllm runs (Node.js, browser WASM)
|
||||
- Bone constraints enforce physically plausible outputs even with noisy inputs
|
||||
- Shares training infrastructure with existing ruvllm pipeline
|
||||
- Modular: each stage (TCN, AsymConv, Axial, Decoder) is independently testable
|
||||
|
||||
### Negative
|
||||
- ~1.8M parameters is 193x larger than simple CsiEncoder (9,344 params)
|
||||
- Forward pass is slower (~50ms vs <1ms for simple encoder)
|
||||
- Camera-free training will produce lower accuracy than supervised WiFlow
|
||||
- No ground-truth PCK evaluation possible without camera labels
|
||||
- Axial attention is O(N^2) within each axis, limiting scalability
|
||||
|
||||
### Neutral
|
||||
- FLOPs dominated by TCN (~48%) due to dilated convolutions
|
||||
- INT8 quantization brings model to ~1.7MB, viable for edge deployment
|
||||
- Architecture is fixed (no NAS); future work could explore lighter variants
|
||||
|
||||
## Implementation
|
||||
|
||||
### Files Created
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `scripts/wiflow-model.js` | WiFlow architecture (all stages, loss, metrics) |
|
||||
| `scripts/train-wiflow.js` | Training pipeline (contrastive + pose proxy + LoRA + quant) |
|
||||
| `scripts/benchmark-wiflow.js` | Benchmarking (latency, params, FLOPs, memory, quality) |
|
||||
| `docs/adr/ADR-072-wiflow-architecture.md` | This document |
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Train on collected data
|
||||
node scripts/train-wiflow.js --data data/recordings/pretrain-*.csi.jsonl
|
||||
|
||||
# Train with more epochs and custom output
|
||||
node scripts/train-wiflow.js --data data/recordings/*.csi.jsonl --epochs 50 --output models/wiflow-v2
|
||||
|
||||
# Contrastive pretraining only (no labels needed)
|
||||
node scripts/train-wiflow.js --data data/recordings/*.csi.jsonl --contrastive-only
|
||||
|
||||
# Benchmark
|
||||
node scripts/benchmark-wiflow.js
|
||||
|
||||
# Benchmark with trained model
|
||||
node scripts/benchmark-wiflow.js --model models/wiflow-v1
|
||||
```
|
||||
|
||||
### Dependencies
|
||||
|
||||
- ruvllm (vendored at `vendor/ruvector/npm/packages/ruvllm/src/`)
|
||||
- `ContrastiveTrainer`, `tripletLoss`, `infoNCELoss`, `computeGradient`
|
||||
- `TrainingPipeline`
|
||||
- `LoraAdapter`, `LoraManager`
|
||||
- `EwcManager`
|
||||
- `ModelExporter`, `SafeTensorsWriter`
|
||||
- No external ML frameworks (no PyTorch, no TensorFlow, no ONNX Runtime)
|
||||
|
||||
## References
|
||||
|
||||
- WiFlow: arXiv:2602.08661
|
||||
- COCO Keypoints: https://cocodataset.org/#keypoints-2020
|
||||
- Axial Attention: Wang et al., "Axial-DeepLab", ECCV 2020
|
||||
- TCN: Bai et al., "An Empirical Evaluation of Generic Convolutional and Recurrent Networks for Sequence Modeling", 2018
|
||||
@@ -0,0 +1,202 @@
|
||||
# ADR-073: Multi-Frequency Mesh Scanning
|
||||
|
||||
| Field | Value |
|
||||
|-------------|--------------------------------------------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-04-02 |
|
||||
| **Authors** | ruv |
|
||||
| **Depends** | ADR-018 (binary frame), ADR-029 (channel hopping), ADR-039 (edge processing), ADR-060 (channel override) |
|
||||
|
||||
## Context
|
||||
|
||||
The current WiFi-DensePose deployment uses 2 ESP32-S3 nodes operating on a single WiFi channel (channel 5, 2432 MHz). A scan of the office environment reveals 9 WiFi networks across 6 distinct channels (1, 3, 5, 6, 9, 11), each broadcasting continuously. These neighbor networks are free RF illuminators whose signals pass through the room and interact with objects, people, and walls.
|
||||
|
||||
**Current single-channel limitations:**
|
||||
|
||||
1. **19% null subcarriers** — metal objects (desk, monitor frame, filing cabinet) create frequency-selective fading that blocks specific subcarriers on channel 5. These nulls are permanent blind spots in the RF map.
|
||||
|
||||
2. **No frequency diversity** — objects that are transparent at 2432 MHz may be opaque at 2412 MHz or 2462 MHz, and vice versa. A metal mesh that blocks one wavelength (122.5 mm at 2432 MHz) may pass another (124.0 mm at 2412 MHz) due to the mesh aperture-to-wavelength ratio.
|
||||
|
||||
3. **Single-perspective CSI** — both nodes see the same 52-64 subcarriers on the same channel. The subcarrier indices map to the same frequency bins, providing no spectral diversity.
|
||||
|
||||
4. **Neighbor illuminator waste** — 6 other APs broadcast continuously in the room. Their signals pass through walls, furniture, and people, creating CSI-measurable reflections that we currently ignore because we only listen on channel 5.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement interleaved multi-frequency channel hopping across the 2 ESP32-S3 nodes, scanning 6 WiFi channels to build a wideband RF map of the room.
|
||||
|
||||
### Channel Allocation Strategy
|
||||
|
||||
The 2.4 GHz ISM band has 3 non-overlapping 20 MHz channels (1, 6, 11) and several partially-overlapping channels between them. We allocate channels to maximize both spectral coverage and illuminator exploitation:
|
||||
|
||||
```
|
||||
Node 1: ch 1, 6, 11 (non-overlapping, full band coverage)
|
||||
Node 2: ch 3, 5, 9 (interleaved, near neighbor APs)
|
||||
```
|
||||
|
||||
**Rationale for this split:**
|
||||
|
||||
| Channel | Freq (MHz) | Node | Neighbor Illuminators | Purpose |
|
||||
|---------|------------|------|----------------------------------------------|-----------------------------------|
|
||||
| 1 | 2412 | 1 | (none visible, but lower freq = better penetration) | Low-frequency penetration |
|
||||
| 3 | 2422 | 2 | conclusion mesh (signal 44) | Exploit neighbor AP as illuminator |
|
||||
| 5 | 2432 | 2 | ruv.net (100), Cohen-Guest (100), HP LaserJet (94) | Primary channel, strongest illuminators |
|
||||
| 6 | 2437 | 1 | Innanen (signal 19) | Center band, non-overlapping |
|
||||
| 9 | 2452 | 2 | NETGEAR72 (42), NETGEAR72-Guest (42) | Exploit dual NETGEAR illuminators |
|
||||
| 11 | 2462 | 1 | COGECO-21B20 (100), COGECO-4321 (30) | High-frequency, strong illuminators |
|
||||
|
||||
Each node dwells on a channel for 250 ms (configurable), collects 3-4 CSI frames, then hops to the next. The 3-channel rotation completes in 750 ms, giving ~1.3 full rotations per second.
|
||||
|
||||
### Physics Basis
|
||||
|
||||
At 2.4 GHz, WiFi wavelength ranges from 122.0 mm (ch 14, 2484 MHz) to 124.0 mm (ch 1, 2412 MHz). While this is a narrow range (~2%), the effect on multipath is significant:
|
||||
|
||||
1. **Frequency-selective fading**: multipath reflections create constructive/destructive interference patterns that vary with frequency. A 2 cm path length difference produces a null at 2432 MHz but constructive interference at 2412 MHz.
|
||||
|
||||
2. **Diffraction around objects**: Huygens-Fresnel diffraction depends on wavelength. Objects smaller than ~lambda/2 (61 mm) scatter differently across the band. Common office objects (monitor bezels, chair legs, cable bundles) are in this range.
|
||||
|
||||
3. **Material transparency**: some materials (wire mesh, perforated metal, PCB ground planes) have frequency-dependent transmission. A monitor's EMI shielding mesh with 5 mm apertures blocks 2.4 GHz signals but the exact attenuation varies with frequency due to slot antenna effects.
|
||||
|
||||
4. **Subcarrier orthogonality**: OFDM subcarriers on different channels are in different frequency bins. A null on subcarrier 15 of channel 5 does not imply a null on subcarrier 15 of channel 1, because they map to different absolute frequencies.
|
||||
|
||||
### Null Diversity Mechanism
|
||||
|
||||
```
|
||||
Channel 5 subcarriers: ▅▆█▇▅▃▁_▁▃▅▆█▇▅▃▁_▁▃▅▆█▇▅▃
|
||||
^ null (metal desk)
|
||||
Channel 1 subcarriers: ▃▅▆█▇▅▃▅▆█▇▅▃▅▆█▇▅▃▅▆█▇▅▃▅▃
|
||||
^ resolved! Different freq = different null pattern
|
||||
|
||||
Channel 11 subcarriers: ▅▃▁_▁▃▅▆█▇▅▃▅▆▅▃▁_▁▃▅▆█▇▅▃▅
|
||||
^ null here instead (shifted by frequency offset)
|
||||
```
|
||||
|
||||
By fusing subcarrier data across channels, nulls that exist on one channel are filled by non-null data from other channels. The remaining nulls (present on ALL channels) represent truly opaque objects — large metal surfaces that block all 2.4 GHz frequencies.
|
||||
|
||||
### Wideband View
|
||||
|
||||
Single channel: ~52-64 subcarriers (20 MHz bandwidth)
|
||||
Multi-channel (6 channels): ~312-384 effective subcarrier observations (120 MHz coverage)
|
||||
|
||||
This is not simply 6x the resolution (the subcarrier spacing within each channel is the same), but it provides:
|
||||
- 6x the spectral diversity for null mitigation
|
||||
- 6x the illuminator variety (different APs = different signal paths)
|
||||
- Frequency-dependent scattering signatures for material classification
|
||||
|
||||
## Integration
|
||||
|
||||
### Firmware (already supported)
|
||||
|
||||
The channel hopping infrastructure is already implemented in the ESP32 firmware (ADR-029):
|
||||
|
||||
```c
|
||||
// csi_collector.h — already exists
|
||||
void csi_collector_set_hop_table(const uint8_t *channels, uint8_t hop_count, uint32_t dwell_ms);
|
||||
void csi_collector_start_hop_timer(void);
|
||||
```
|
||||
|
||||
The ADR-018 binary frame header already includes the channel/frequency field at bytes [8..11], so the server-side parser can distinguish frames from different channels without any firmware changes.
|
||||
|
||||
### Provisioning Commands
|
||||
|
||||
```bash
|
||||
# Node 1 (COM7): non-overlapping channels 1, 6, 11
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "ruv.net" --password "..." --target-ip 192.168.1.20 \
|
||||
--hop-channels 1,6,11 --hop-dwell-ms 250
|
||||
|
||||
# Node 2 (COM_): interleaved channels 3, 5, 9
|
||||
python firmware/esp32-csi-node/provision.py --port COM_ \
|
||||
--ssid "ruv.net" --password "..." --target-ip 192.168.1.20 \
|
||||
--hop-channels 3,5,9 --hop-dwell-ms 250
|
||||
```
|
||||
|
||||
Note: `--hop-channels` and `--hop-dwell-ms` require provision.py support for writing these values to NVS. If not yet implemented, the firmware's `csi_collector_set_hop_table()` can be called directly from the main init code with compile-time constants.
|
||||
|
||||
### Server-Side Processing
|
||||
|
||||
Three new Node.js scripts consume the multi-channel CSI data:
|
||||
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/rf-scan.js` | Single-channel live RF room scanner with ASCII spectrum |
|
||||
| `scripts/rf-scan-multifreq.js` | Multi-channel scanner with null diversity analysis |
|
||||
| `scripts/benchmark-rf-scan.js` | Quantitative benchmark of multi-channel performance |
|
||||
|
||||
All scripts parse the ADR-018 binary UDP format and use the frequency field to separate frames by channel.
|
||||
|
||||
### Cognitum Seed Integration
|
||||
|
||||
The Cognitum Seed vector store (ADR-069) currently stores 1,605 vectors from single-channel CSI. With multi-frequency scanning:
|
||||
|
||||
1. **Per-channel feature vectors**: store separate 8-dim feature vectors for each channel, tagged with channel number. This increases the vector count to ~9,630 (6 channels x 1,605).
|
||||
|
||||
2. **Wideband feature vector**: concatenate or average per-channel features into a 48-dim wideband vector for richer kNN search. Objects that are ambiguous on one channel may be clearly distinguishable in the wideband representation.
|
||||
|
||||
3. **Null-aware embeddings**: encode null subcarrier patterns as part of the feature vector. The null pattern itself is informative — a consistent null at subcarrier 15 across all channels indicates a large metal object, while a null only on channel 5 indicates a frequency-dependent scatterer.
|
||||
|
||||
## Performance Targets
|
||||
|
||||
| Metric | Single-Channel Baseline | Multi-Channel Target | Method |
|
||||
|--------|------------------------|---------------------|--------|
|
||||
| Subcarrier count | ~52-64 | ~312-384 (6x) | 6 channels x 52-64 subcarriers |
|
||||
| Null gap | 19% | <5% | Null diversity across channels |
|
||||
| Position resolution | ~30 cm | ~15 cm | sqrt(6) improvement from independent observations |
|
||||
| Per-channel FPS | 12 fps | ~4 fps | 250 ms dwell x 3 channels = 750 ms rotation |
|
||||
| Total FPS (all channels) | 12 fps | ~12 fps per node (4 fps x 3 channels) |
|
||||
| Wideband rotation | N/A | ~1.3 Hz | Full 3-channel rotation in 750 ms |
|
||||
|
||||
## Risks
|
||||
|
||||
### Per-Channel Sample Rate Reduction
|
||||
|
||||
Channel hopping reduces the per-channel sample rate from 12 fps (single channel) to approximately 4 fps per channel (250 ms dwell, 3 channels). This affects:
|
||||
|
||||
- **Vitals extraction**: breathing rate (0.1-0.5 Hz) requires at least 2 fps (Nyquist). At 4 fps per channel, this is met. Heart rate (0.8-2.0 Hz) requires at least 4 fps, which is marginal. Mitigation: keep one channel as "primary" with longer dwell for vitals, or fuse phase data across channels.
|
||||
|
||||
- **Motion tracking**: 4 fps is sufficient for walking speed (<2 m/s) but insufficient for fast gestures. If gesture recognition is needed, reduce to 2-channel hopping or increase dwell rate.
|
||||
|
||||
### Channel Hopping Latency
|
||||
|
||||
`esp_wifi_set_channel()` takes ~1-5 ms on ESP32-S3. During the transition, no CSI frames are captured. At 250 ms dwell, this is <2% overhead.
|
||||
|
||||
### AP Disconnection
|
||||
|
||||
Channel hopping may cause the ESP32 to lose connection to the home AP (ruv.net on channel 5) when dwelling on other channels. The STA reconnects automatically, but there may be brief UDP packet loss. Mitigation: the firmware already handles this gracefully — CSI collection works in promiscuous mode regardless of STA connection state.
|
||||
|
||||
### Increased Server Load
|
||||
|
||||
2 nodes x 3 channels x 4 fps = 24 frames/second total UDP traffic. Each frame is ~150-200 bytes (20-byte header + 64 subcarriers x 2 bytes I/Q). Total: ~4.8 KB/s — negligible.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **5 GHz channels**: ESP32-S3 supports 5 GHz CSI, and the shorter wavelength (60 mm) provides better spatial resolution. Rejected because: (a) no 5 GHz APs visible in the current environment, so no free illuminators; (b) 5 GHz has worse wall penetration, reducing the effective sensing volume.
|
||||
|
||||
2. **More nodes**: adding a 3rd or 4th ESP32 node would increase spatial diversity without channel hopping. Rejected for now due to cost, but this is complementary — more nodes + channel hopping would give both spatial and spectral diversity.
|
||||
|
||||
3. **Wider bandwidth (HT40)**: using 40 MHz channels doubles subcarrier count per channel. Rejected because: (a) HT40 requires a secondary channel, reducing available channels for hopping; (b) many neighbor APs use HT20, so their illumination only covers 20 MHz.
|
||||
|
||||
## SNN Integration (ADR-074)
|
||||
|
||||
Multi-frequency scanning produces subcarrier data across 6 channels, creating temporal patterns that are well-suited for spiking neural network processing. ADR-074 introduces an SNN with STDP learning that consumes the multi-channel CSI stream.
|
||||
|
||||
**Key interactions with multi-frequency data:**
|
||||
|
||||
1. **Null diversity as SNN input**: subcarriers that are null on one channel but active on another produce a distinctive spike pattern (spikes only during certain channel dwells). STDP learns to associate these cross-channel patterns with specific objects or zones — something a single-channel SNN cannot do.
|
||||
|
||||
2. **Channel-interleaved temporal coding**: because each node dwells on 3 channels in a 750ms rotation, the SNN receives subcarrier data in a repeating temporal pattern (ch1 → ch2 → ch3 → ch1 ...). The SNN's LIF membrane dynamics integrate spikes across the rotation, naturally performing cross-channel fusion through temporal summation. A hidden neuron that receives spikes from subcarrier 15 on channel 1 AND subcarrier 15 on channel 6 will fire more strongly than one receiving either alone.
|
||||
|
||||
3. **Expanded input mode**: on the server (not constrained by ESP32 memory), the SNN can use 384 input neurons (6 channels x 64 subcarriers) instead of 128. This provides maximum spectral diversity per frame but requires ~150 KB of weight storage. The `snn-csi-processor.js` script supports this via the `--hidden` flag to scale the network.
|
||||
|
||||
4. **Illuminator fingerprinting**: different neighbor APs have different beamforming patterns and power levels. The SNN learns which subcarrier patterns belong to which illuminator, enabling it to distinguish AP-specific signatures from human-caused perturbations. This is especially useful for the NETGEAR dual-AP setup on channel 9, where two illuminators from different positions create stereo-like RF coverage.
|
||||
|
||||
## References
|
||||
|
||||
- ADR-018: CSI binary frame format
|
||||
- ADR-029: Channel hopping infrastructure
|
||||
- ADR-039: Edge processing pipeline
|
||||
- ADR-060: Channel override provisioning
|
||||
- ADR-069: Cognitum Seed CSI pipeline
|
||||
- ADR-074: Spiking neural network for CSI sensing
|
||||
- IEEE 802.11-2020, Section 21 (OFDM PHY)
|
||||
- ESP-IDF CSI Guide: https://docs.espressif.com/projects/esp-idf/en/v5.4/esp32s3/api-guides/wifi.html#wi-fi-channel-state-information
|
||||
@@ -0,0 +1,208 @@
|
||||
# ADR-074: Spiking Neural Network for CSI Sensing
|
||||
|
||||
| Field | Value |
|
||||
|-------------|--------------------------------------------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-04-02 |
|
||||
| **Authors** | ruv |
|
||||
| **Depends** | ADR-018 (binary frame), ADR-029 (channel hopping), ADR-069 (Cognitum Seed), ADR-073 (multi-frequency mesh) |
|
||||
|
||||
## Context
|
||||
|
||||
The current WiFi-DensePose CSI sensing pipeline uses two approaches for interpreting subcarrier data:
|
||||
|
||||
1. **Static thresholds** — presence detection fires when subcarrier variance exceeds a fixed value. This works in calibrated environments but fails when the RF landscape changes (furniture moved, new objects, temperature drift). Recalibration requires manual intervention or batch retraining.
|
||||
|
||||
2. **Batch-trained FC encoder** — the neural network in `wifi-densepose-nn` maps CSI frames to 8-dimensional feature vectors. It requires labeled training data, offline training epochs, and model deployment. The encoder cannot adapt to a new environment without collecting new data and retraining.
|
||||
|
||||
Neither approach handles online adaptation. When an ESP32 node is deployed in a new room, the first hours produce noisy, unreliable output until the thresholds are tuned or a model is trained. In disaster scenarios (ADR MAT), there is no time for calibration.
|
||||
|
||||
**Spiking Neural Networks (SNNs)** offer an alternative. Unlike traditional ANNs that process continuous values in batch mode, SNNs communicate through discrete spike events and learn online via Spike-Timing-Dependent Plasticity (STDP). This is a natural fit for CSI data:
|
||||
|
||||
- CSI subcarrier amplitudes are temporal signals sampled at 12-22 fps
|
||||
- Amplitude changes (not absolute values) carry the information about motion, breathing, and presence
|
||||
- STDP learns temporal correlations between subcarriers without labels
|
||||
- Event-driven processing means idle rooms (no motion) consume near-zero compute
|
||||
|
||||
The `@ruvector/spiking-neural` package (vendored at `vendor/ruvector/npm/packages/spiking-neural/`) provides production-ready LIF neurons, STDP learning, lateral inhibition, and SIMD-optimized vector math in pure JavaScript with zero dependencies.
|
||||
|
||||
## Decision
|
||||
|
||||
Integrate `@ruvector/spiking-neural` into the CSI sensing pipeline as an online unsupervised pattern learner that runs alongside the existing FC encoder. The SNN provides real-time adaptation while the FC encoder provides stable baseline predictions.
|
||||
|
||||
### Network Architecture
|
||||
|
||||
```
|
||||
CSI Frame (128 subcarriers)
|
||||
|
|
||||
v
|
||||
[ Rate Encoding ] -----> 128 input neurons (one per subcarrier)
|
||||
| amplitude delta -> spike rate
|
||||
v
|
||||
[ LIF Hidden Layer ] ---> 64 hidden neurons (tau=20ms)
|
||||
| STDP learns subcarrier correlations
|
||||
| lateral inhibition -> sparse codes
|
||||
v
|
||||
[ LIF Output Layer ] ---> 8 output neurons
|
||||
|
|
||||
v
|
||||
presence | motion | breathing | heart_rate | phase_var | persons | fall | rssi
|
||||
```
|
||||
|
||||
**Layer parameters:**
|
||||
|
||||
| Layer | Neurons | tau (ms) | v_thresh (mV) | Function |
|
||||
|-------|---------|----------|---------------|----------|
|
||||
| Input | 128 | N/A | N/A | Rate-coded spike generation from subcarrier deltas |
|
||||
| Hidden | 64 | 20.0 | -50.0 | STDP learns correlated subcarrier groups |
|
||||
| Output | 8 | 25.0 | -50.0 | Each neuron specializes in one sensing modality |
|
||||
|
||||
**Synapse parameters:**
|
||||
|
||||
| Connection | Count | a_plus | a_minus | w_init | Lateral Inhibition |
|
||||
|------------|-------|--------|---------|--------|-------------------|
|
||||
| Input -> Hidden | 8,192 | 0.005 | 0.005 | 0.3 | No |
|
||||
| Hidden -> Output | 512 | 0.003 | 0.003 | 0.2 | Yes (strength=15.0) |
|
||||
|
||||
Total synapses: 8,704. At 4 bytes per weight, this is 34 KB — fits in ESP32 SRAM.
|
||||
|
||||
### Input Encoding
|
||||
|
||||
CSI amplitudes are converted to spike rates using rate coding:
|
||||
|
||||
1. Compute per-subcarrier amplitude: `amp[i] = sqrt(I[i]^2 + Q[i]^2)` from the ADR-018 binary frame
|
||||
2. Compute amplitude delta from previous frame: `delta[i] = |amp[i] - prev_amp[i]|`
|
||||
3. Normalize deltas to [0, 1] range: `norm[i] = min(delta[i] / max_delta, 1.0)`
|
||||
4. Feed `norm` to `rateEncoding(norm, dt, max_rate)` which produces Poisson spikes
|
||||
|
||||
Higher amplitude changes produce more spikes. Static subcarriers (no motion) produce few or no spikes. This is the key energy advantage: an empty room generates almost no spikes, so the SNN does almost no work.
|
||||
|
||||
### STDP Learning Rule
|
||||
|
||||
STDP strengthens connections between neurons that fire together (within a time window) and weakens connections between neurons that fire out of sync:
|
||||
|
||||
- **LTP (Long-Term Potentiation)**: if a presynaptic neuron fires before a postsynaptic neuron within 20ms, the weight increases by `a_plus * exp(-dt/tau_stdp)`
|
||||
- **LTD (Long-Term Depression)**: if a postsynaptic neuron fires before a presynaptic neuron, the weight decreases by `a_minus * exp(-dt/tau_stdp)`
|
||||
|
||||
Over time, this causes the hidden layer neurons to specialize. Subcarriers that consistently change together (e.g., subcarriers 10-20 affected by a person walking through zone A) become strongly connected to the same hidden neuron. Different motion patterns activate different hidden neuron clusters.
|
||||
|
||||
### Lateral Inhibition (Winner-Take-All)
|
||||
|
||||
The output layer uses lateral inhibition with strength 15.0. When one output neuron fires, it suppresses all others. This forces each output neuron to specialize in a distinct pattern:
|
||||
|
||||
- Output 0: presence (any subcarrier activity above baseline)
|
||||
- Output 1: motion (widespread subcarrier changes, high spike rate)
|
||||
- Output 2: breathing (periodic 0.1-0.5 Hz modulation on chest-area subcarriers)
|
||||
- Output 3: heart rate (periodic 0.8-2.0 Hz modulation, lower amplitude than breathing)
|
||||
- Output 4: phase variance (phase instability across subcarriers)
|
||||
- Output 5: person count (number of distinct active subcarrier clusters)
|
||||
- Output 6: fall (sudden high-amplitude burst followed by silence)
|
||||
- Output 7: RSSI trend (overall signal strength change)
|
||||
|
||||
The neuron-to-label mapping is not fixed by training. Instead, the mapping is discovered by observing which output neuron fires most for each known condition during an optional calibration phase. If no calibration is available, the output is reported as raw spike counts per output neuron, and downstream consumers (Cognitum Seed, SONA) interpret the patterns.
|
||||
|
||||
### Integration with Existing Pipeline
|
||||
|
||||
The SNN does not replace the FC encoder. It runs in parallel:
|
||||
|
||||
```
|
||||
CSI Frame ----+----> FC Encoder --------> 8-dim feature vector (stable, trained)
|
||||
|
|
||||
+----> SNN (STDP) --------> 8-dim spike rate vector (adaptive, online)
|
||||
|
|
||||
+----> SONA Adapter -------> Weighted fusion of both signals
|
||||
```
|
||||
|
||||
SONA (Self-Optimizing Neural Architecture) receives both signals and learns which source is more reliable for each output dimension. In a new environment where the FC encoder has not been retrained, SONA automatically weights the SNN output higher because it adapts faster. As the FC encoder is retrained on local data, SONA shifts weight back toward it.
|
||||
|
||||
### Energy and Compute Budget
|
||||
|
||||
| Metric | FC Encoder | SNN (STDP) | Ratio |
|
||||
|--------|-----------|------------|-------|
|
||||
| Compute per frame (idle room) | 8,192 MACs | ~50 spike events | ~160x less |
|
||||
| Compute per frame (active room) | 8,192 MACs | ~500 spike events | ~16x less |
|
||||
| Memory | 34 KB weights | 34 KB weights | Equal |
|
||||
| Adaptation | Offline retraining | Online, continuous | SNN wins |
|
||||
| Stability | High (frozen weights) | Lower (weights drift) | FC wins |
|
||||
| Latency to first useful output | Hours (needs training data) | ~30 seconds | SNN wins |
|
||||
|
||||
The SNN's event-driven nature means it processes only spikes, not every subcarrier on every frame. In an idle room with no motion, subcarrier deltas are near zero, spike rates drop to near zero, and the SNN consumes negligible compute. This is ideal for battery-powered or thermally constrained deployments (ESP32, Cognitum Seed Pi Zero).
|
||||
|
||||
### Deployment Targets
|
||||
|
||||
| Platform | Runtime | Notes |
|
||||
|----------|---------|-------|
|
||||
| Node.js server | `require('@ruvector/spiking-neural')` | Primary. Receives UDP frames, runs SNN. |
|
||||
| Cognitum Seed (Pi Zero) | Node.js ARM | 34 KB model fits. ~0.06ms per step at 100 neurons. |
|
||||
| ESP32-S3 (WASM) | wasm3 interpreter | Optional. SNN weights exported as flat Float32Array. |
|
||||
| Browser | WebAssembly or JS | Via `wifi-densepose-wasm` crate's JS bindings. |
|
||||
|
||||
### Multi-Channel SNN (ADR-073 Integration)
|
||||
|
||||
With multi-frequency mesh scanning (ADR-073), the SNN input expands:
|
||||
|
||||
- **Single-channel mode**: 128 input neurons (64 subcarriers x 2 for I/Q or amplitude/phase)
|
||||
- **Multi-channel mode**: 128 input neurons, but the subcarrier index rotates across channels. Each channel's subcarriers map to the same neuron indices, but at different time slots. The SNN's temporal dynamics naturally integrate cross-channel information because STDP operates across time.
|
||||
|
||||
Alternatively, for maximum spectral diversity, a wider SNN (384 input neurons for 6 channels x 64 subcarriers) can be used on the server where memory is not constrained.
|
||||
|
||||
## Performance Targets
|
||||
|
||||
| Metric | Target | Method |
|
||||
|--------|--------|--------|
|
||||
| SNN step latency | <0.1ms | 128-64-8 network, ~8,700 synapses |
|
||||
| STDP convergence | <30 seconds | ~360 frames at 12 fps, patterns stabilize |
|
||||
| Output accuracy (after adaptation) | >80% | Compared to manually labeled ground truth |
|
||||
| Memory footprint | <50 KB | Weights + neuron state |
|
||||
| Idle room spike rate | <10 spikes/frame | Event-driven: near-zero compute when nothing moves |
|
||||
| Adaptation to new environment | <2 minutes | STDP relearns subcarrier correlations |
|
||||
|
||||
## Risks
|
||||
|
||||
### Weight Drift
|
||||
|
||||
STDP learning never stops. In a stable environment, weights can slowly drift as the network over-fits to the current RF landscape. Mitigation: implement weight decay (multiply all weights by 0.999 per second) and clamp weights to [w_min, w_max].
|
||||
|
||||
### Output Neuron Reassignment
|
||||
|
||||
If the RF environment changes significantly (new furniture, different room), output neurons may reassign their specialization. The mapping from output neuron index to label (presence, motion, etc.) may change. Mitigation: periodically log the output neuron activity and detect reassignment events. Downstream consumers should use the spike pattern, not the neuron index, for classification.
|
||||
|
||||
### Interference with FC Encoder
|
||||
|
||||
If SONA naively averages the SNN and FC encoder outputs, a poorly adapted SNN could degrade overall accuracy. Mitigation: SONA uses confidence-weighted fusion. The SNN output includes a confidence signal (total spike count / expected spike count). Low confidence = low weight.
|
||||
|
||||
### STDP Learning Rate Sensitivity
|
||||
|
||||
If `a_plus` and `a_minus` are too high, the SNN oscillates and never converges. If too low, adaptation takes too long. The default values (0.005 and 0.003) are conservative. The script includes a `--learning-rate` flag for tuning.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **Online gradient descent on FC encoder** — backprop through the FC network with each new frame. Rejected because: (a) requires a loss function, which requires labels; (b) continuous gradient updates on a small model lead to catastrophic forgetting of the pretrained representations.
|
||||
|
||||
2. **Adaptive thresholds only** — replace fixed thresholds with exponentially-weighted moving averages. Rejected because: (a) single-variable thresholds cannot capture multi-subcarrier correlations; (b) no representation learning — each subcarrier is still processed independently.
|
||||
|
||||
3. **Reservoir computing (Echo State Network)** — use a fixed random recurrent network as a temporal feature extractor. Partially viable, but: (a) requires a linear readout layer trained with labels; (b) the random reservoir does not adapt to the specific RF environment.
|
||||
|
||||
4. **Train SNN with supervision** — use surrogate gradient methods to train the SNN on labeled data. Rejected because: (a) defeats the purpose of online unsupervised learning; (b) the `@ruvector/spiking-neural` package does not implement surrogate gradients.
|
||||
|
||||
## Implementation
|
||||
|
||||
The integration is implemented in `scripts/snn-csi-processor.js`, a standalone Node.js script that:
|
||||
|
||||
1. Receives live CSI frames via UDP (port 5006, ADR-018 binary format)
|
||||
2. Decodes subcarrier I/Q data and computes amplitude deltas
|
||||
3. Feeds deltas through rate encoding into the SNN
|
||||
4. Applies STDP learning on every frame (online, unsupervised)
|
||||
5. Maps output neuron spike counts to sensing labels
|
||||
6. Prints real-time ASCII visualization of SNN activity
|
||||
7. Optionally forwards learned patterns to Cognitum Seed
|
||||
|
||||
## References
|
||||
|
||||
- ADR-018: CSI binary frame format
|
||||
- ADR-029: Channel hopping infrastructure
|
||||
- ADR-069: Cognitum Seed CSI pipeline
|
||||
- ADR-073: Multi-frequency mesh scanning
|
||||
- Maass, W. (1997). "Networks of spiking neurons: The third generation of neural network models." Neural Networks, 10(9), 1659-1671.
|
||||
- Bi, G. & Poo, M. (1998). "Synaptic modifications in cultured hippocampal neurons: Dependence on spike timing." Journal of Neuroscience, 18(24), 10464-10472.
|
||||
- `@ruvector/spiking-neural` v1.0.1 — LIF, STDP, lateral inhibition, SIMD
|
||||
@@ -0,0 +1,195 @@
|
||||
# ADR-075: Min-Cut Based Person Separation from Subcarrier Correlation
|
||||
|
||||
- **Status:** Proposed
|
||||
- **Date:** 2026-04-02
|
||||
- **Issue:** #348 — `n_persons` always reports 4 regardless of actual occupancy
|
||||
- **Depends on:** ADR-016 (RuVector integration), ADR-041 (person tracking), ADR-073 (multifrequency mesh scan)
|
||||
|
||||
## Context
|
||||
|
||||
### The Bug
|
||||
|
||||
Issue #348 reports that the ESP32 firmware's multi-person counting always reports
|
||||
`n_persons = 4`. The root cause is in the WASM edge module
|
||||
`sig_mincut_person_match.rs`, which uses a fixed `MAX_PERSONS = 4` constant and a
|
||||
threshold-based variance classifier to populate person slots. The classifier bins
|
||||
subcarriers into "dynamic" vs "static" using a single fixed variance threshold
|
||||
(`DYNAMIC_VAR_THRESH = 0.15`). In practice:
|
||||
|
||||
1. The threshold is miscalibrated for real-world CSI data — almost any room with
|
||||
multipath reflections pushes a majority of subcarriers above 0.15 variance.
|
||||
2. The subcarrier-to-person assignment uses a greedy Hungarian-lite matcher that
|
||||
fills all 4 slots once there are >= 4 dynamic subcarriers (which is nearly
|
||||
always the case).
|
||||
3. There is no mechanism to determine how many independent movers exist — the
|
||||
algorithm assumes all 4 slots should be filled.
|
||||
|
||||
### Prior Art
|
||||
|
||||
The Rust crate `ruvector-mincut` (vendored at `vendor/ruvector/crates/ruvector-mincut/`)
|
||||
implements a full dynamic min-cut algorithm with O(n^{o(1)}) amortized update time,
|
||||
Stoer-Wagner exact min-cut, and online edge insert/delete. It is already integrated
|
||||
in the training pipeline (`wifi-densepose-train/src/metrics.rs`) via
|
||||
`DynamicPersonMatcher`.
|
||||
|
||||
### WiFi Sensing Insight
|
||||
|
||||
When a person moves through a room, they perturb the Fresnel zones of specific
|
||||
subcarrier frequencies. Subcarriers whose Fresnel zones overlap the person's body
|
||||
change **together** — their amplitudes are temporally correlated. When two people
|
||||
move independently, they create two **separate** groups of correlated subcarriers.
|
||||
This correlation structure forms a natural graph partitioning problem.
|
||||
|
||||
## Decision
|
||||
|
||||
Replace the fixed-threshold person counter with a spectral min-cut algorithm
|
||||
operating on the subcarrier temporal correlation graph. This runs in the bridge
|
||||
script (`scripts/mincut-person-counter.js`) or on Cognitum Seed, and feeds the
|
||||
corrected person count back to the feature vector before ingest.
|
||||
|
||||
### Algorithm
|
||||
|
||||
1. **Sliding window accumulation**: Maintain the last 2 seconds of subcarrier
|
||||
amplitude data (~40 frames at 20 fps). Each frame provides a 64-element
|
||||
amplitude vector (one per subcarrier).
|
||||
|
||||
2. **Pairwise Pearson correlation**: For all subcarrier pairs (i, j), compute
|
||||
the Pearson correlation coefficient over the sliding window:
|
||||
|
||||
```
|
||||
r(i,j) = cov(amp_i, amp_j) / (std(amp_i) * std(amp_j))
|
||||
```
|
||||
|
||||
This produces a 64x64 correlation matrix.
|
||||
|
||||
3. **Graph construction**: Build a weighted undirected graph:
|
||||
- **Nodes** = subcarriers (64 for single-antenna ESP32-S3, up to 128 for dual)
|
||||
- **Edges** = pairs with |r(i,j)| > 0.3 (correlation threshold)
|
||||
- **Weight** = |r(i,j)| (correlation strength)
|
||||
- Discard null subcarriers (amplitude consistently near zero)
|
||||
- Expected: ~1500-2500 edges for 64 active subcarriers
|
||||
|
||||
4. **Iterative Stoer-Wagner min-cut**: Apply the Stoer-Wagner algorithm to find
|
||||
the global minimum cut. If the min-cut weight is below a separation threshold
|
||||
(empirically 2.0), the cut represents a real boundary between independent
|
||||
movers. Split the graph at the cut and recurse on each partition.
|
||||
|
||||
5. **Person count**: The number of partitions after all valid cuts = number of
|
||||
independent movers = person count. A single connected component with high
|
||||
internal correlation and no low-weight cut = 1 person (or 0 if variance is
|
||||
also low).
|
||||
|
||||
6. **Empty room detection**: If the total variance across all subcarriers is
|
||||
below a noise floor threshold, report 0 persons regardless of graph structure.
|
||||
|
||||
### Stoer-Wagner Algorithm
|
||||
|
||||
Stoer-Wagner finds the exact global minimum cut of an undirected weighted graph
|
||||
in O(V * E) time using a sequence of "minimum cut phases":
|
||||
|
||||
```
|
||||
function stoerWagner(G):
|
||||
best_cut = infinity
|
||||
while |V(G)| > 1:
|
||||
(s, t, cut_of_phase) = minimumCutPhase(G)
|
||||
if cut_of_phase < best_cut:
|
||||
best_cut = cut_of_phase
|
||||
best_partition = partition induced by t
|
||||
merge(s, t) // contract vertices s and t
|
||||
return best_cut, best_partition
|
||||
|
||||
function minimumCutPhase(G):
|
||||
A = {arbitrary start vertex}
|
||||
while A != V(G):
|
||||
z = vertex most tightly connected to A
|
||||
// "most tightly connected" = max sum of edge weights to A
|
||||
add z to A
|
||||
s = second-to-last vertex added
|
||||
t = last vertex added (most tightly connected)
|
||||
cut_of_phase = sum of weights of edges incident to t
|
||||
return (s, t, cut_of_phase)
|
||||
```
|
||||
|
||||
For V=64 subcarriers and E~2000 edges, this runs in ~8 million operations,
|
||||
well under 1ms on modern hardware and under 10ms even on ESP32-S3.
|
||||
|
||||
### Integration Points
|
||||
|
||||
```
|
||||
ESP32 Node 1 ──UDP 5006──┐
|
||||
├──> mincut-person-counter.js ──> corrected n_persons
|
||||
ESP32 Node 2 ──UDP 5006──┘ │
|
||||
├──> seed_csi_bridge.py (feature dim 5 override)
|
||||
└──> csi-graph-visualizer.js (debug view)
|
||||
```
|
||||
|
||||
The person counter runs as a standalone Node.js process alongside the existing
|
||||
`rf-scan.js` and `seed_csi_bridge.py` bridge scripts. It can also replay
|
||||
recorded `.csi.jsonl` files for offline analysis.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### 1. Threshold-based peak counting (current, broken)
|
||||
|
||||
Count subcarriers with variance above a threshold, then cluster by proximity.
|
||||
**Problem:** threshold is environment-dependent, miscalibrates easily, and
|
||||
cannot distinguish correlated from independent motion.
|
||||
|
||||
### 2. PCA / spectral clustering on correlation matrix
|
||||
|
||||
Compute eigenvectors of the correlation matrix; the number of large eigenvalues
|
||||
indicates the number of independent sources. **Problem:** requires choosing an
|
||||
eigenvalue gap threshold, which is as fragile as the current variance threshold.
|
||||
Also does not give per-person subcarrier assignments.
|
||||
|
||||
### 3. Min-cut on correlation graph (this ADR)
|
||||
|
||||
**Advantages:**
|
||||
- Directly models the physical structure (Fresnel zone groupings)
|
||||
- Threshold-free person counting (cut weight is a natural separation metric)
|
||||
- Produces per-person subcarrier groups as a side effect
|
||||
- Stoer-Wagner is simple to implement (~100 lines) and runs in polynomial time
|
||||
- Already validated in Rust via `ruvector-mincut` integration
|
||||
|
||||
## Performance
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Graph size | V=64, E~2000 |
|
||||
| Stoer-Wagner complexity | O(V * E) = O(128,000) per cut |
|
||||
| Iterative cuts (max 4) | O(512,000) total |
|
||||
| Wall time (Node.js) | < 5 ms per 2-second window |
|
||||
| Wall time (Rust/WASM) | < 0.5 ms |
|
||||
| Memory | ~32 KB for correlation matrix + graph |
|
||||
| Sliding window | 2 seconds = ~40 frames * 64 subcarriers * 8 bytes = 20 KB |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Fixes #348: person count now reflects actual independent movers
|
||||
- Robust across environments (no per-room threshold calibration)
|
||||
- Per-person subcarrier groups enable per-person feature extraction
|
||||
- Graph visualization aids debugging and room mapping
|
||||
- Algorithm is well-understood (Stoer-Wagner, 1997)
|
||||
|
||||
### Negative
|
||||
|
||||
- Adds a new process to the sensing pipeline
|
||||
- 2-second latency for person count changes (sliding window)
|
||||
- Correlation-based: cannot detect stationary persons (no motion = no signal)
|
||||
- Assumes independent motion — two people walking in sync may be counted as one
|
||||
|
||||
### Migration
|
||||
|
||||
1. Deploy `scripts/mincut-person-counter.js` alongside existing bridge
|
||||
2. Override feature vector dimension 5 (`n_persons`) with corrected count
|
||||
3. Once validated, port Stoer-Wagner to C for direct ESP32-S3 firmware integration
|
||||
4. Deprecate the fixed-threshold `PersonMatcher` in `sig_mincut_person_match.rs`
|
||||
|
||||
## References
|
||||
|
||||
- Stoer, M. & Wagner, F. (1997). "A Simple Min-Cut Algorithm." JACM 44(4).
|
||||
- `vendor/ruvector/crates/ruvector-mincut/src/algorithm/mod.rs` — DynamicMinCut API
|
||||
- `rust-port/.../sig_mincut_person_match.rs` — current (broken) WASM edge matcher
|
||||
- `scripts/rf-scan.js` — CSI packet parsing and subcarrier classification
|
||||
@@ -0,0 +1,259 @@
|
||||
# ADR-076: CSI Spectrogram Embeddings via CNN + Graph Transformer
|
||||
|
||||
| Field | Value |
|
||||
|-------------|--------------------------------------------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-04-02 |
|
||||
| **Authors** | ruv |
|
||||
| **Depends** | ADR-018 (binary frame), ADR-024 (AETHER contrastive embeddings), ADR-029 (RuvSense), ADR-069 (Cognitum Seed bridge), ADR-073 (multi-frequency mesh scan) |
|
||||
|
||||
## Context
|
||||
|
||||
The current CSI processing pipeline extracts an 8-dimensional hand-crafted feature vector per frame: mean amplitude, amplitude variance, max amplitude, mean phase, phase variance, bandwidth, spectral centroid, and RSSI. These features are effective for basic presence detection and room fingerprinting but discard the rich spatial-frequency structure present in the raw subcarrier data.
|
||||
|
||||
A single CSI frame from an ESP32-S3 contains 64 subcarriers (or 128 in HT40 mode), each with I/Q components. When stacked over time, 20 consecutive frames form a **64x20 subcarrier-by-time matrix** — effectively a grayscale spectrogram image. This matrix encodes:
|
||||
|
||||
1. **Frequency-selective fading** — metal objects create persistent null zones at specific subcarrier indices (visible as dark vertical stripes)
|
||||
2. **Doppler signatures** — human motion produces time-varying amplitude patterns across subcarriers (visible as horizontal wave patterns)
|
||||
3. **Multipath structure** — room geometry creates characteristic interference patterns unique to each environment
|
||||
4. **Activity fingerprints** — walking, sitting, breathing, and falling produce distinct 2D texture patterns in the subcarrier-time matrix
|
||||
|
||||
These 2D structural patterns are invisible to the 8-dim feature vector, which collapses all subcarrier information into scalar statistics. A CNN embedding can preserve this spatial structure.
|
||||
|
||||
### Existing Vendor Libraries
|
||||
|
||||
**@ruvector/cnn** (v0.1.0) provides:
|
||||
- WASM-based CNN feature extraction (~5ms per 224x224 image, ~900KB model)
|
||||
- Configurable embedding dimension (default 512, we use 128 for compact storage)
|
||||
- L2-normalized embeddings with cosine similarity search
|
||||
- Contrastive training via InfoNCE and triplet loss
|
||||
- SIMD-optimized layer operations (batch norm, global average pooling, ReLU)
|
||||
- Works in both Node.js and browser environments
|
||||
|
||||
**ruvector-graph-transformer** provides:
|
||||
- Sublinear O(n log n) graph attention via LSH bucketing and PPR sampling
|
||||
- Proof-gated mutation substrate for verified computations
|
||||
- Temporal causal attention with Granger causality (relevant for CSI time series)
|
||||
- Manifold attention on product spaces S^n x H^m x R^k
|
||||
|
||||
**@ruvector/graph-wasm** (v2.0.2) provides:
|
||||
- Neo4j-compatible property graph database in WASM
|
||||
- Node/edge creation with arbitrary properties and embeddings
|
||||
- Hyperedge support for multi-node relationships
|
||||
- Cypher query language
|
||||
|
||||
### Current Limitations of 8-dim Features
|
||||
|
||||
| Limitation | Impact |
|
||||
|------------|--------|
|
||||
| No subcarrier-level information | Cannot distinguish frequency-selective vs broadband fading |
|
||||
| No temporal pattern encoding | Walking gait (periodic) looks identical to random motion (aperiodic) |
|
||||
| No 2D structure | Room fingerprint reduced to 8 numbers; two rooms with similar statistics are indistinguishable |
|
||||
| No cross-subcarrier correlation | Cannot detect standing waves, node patterns, or multipath clusters |
|
||||
| Poor kNN discrimination | 8 dimensions provides limited hypersphere surface area for separating environments |
|
||||
|
||||
## Decision
|
||||
|
||||
Treat the CSI subcarrier-by-time matrix as a grayscale spectrogram image and apply CNN embedding to produce a 128-dimensional representation that preserves 2D spatial-frequency structure. Use a graph transformer to fuse embeddings across multiple ESP32 nodes.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
ESP32 Node 1 ESP32 Node 2
|
||||
| |
|
||||
v v
|
||||
UDP 5006 UDP 5006
|
||||
| |
|
||||
v v
|
||||
[64 subcarriers] [64 subcarriers]
|
||||
[20-frame window] [20-frame window]
|
||||
| |
|
||||
v v
|
||||
64x20 amplitude 64x20 amplitude
|
||||
matrix (grayscale) matrix (grayscale)
|
||||
| |
|
||||
v v
|
||||
@ruvector/cnn @ruvector/cnn
|
||||
CnnEmbedder CnnEmbedder
|
||||
| |
|
||||
v v
|
||||
128-dim vector 128-dim vector
|
||||
| |
|
||||
+-------+ +----------+
|
||||
| |
|
||||
v v
|
||||
Graph Transformer (2-node graph)
|
||||
Edge weight = cross-node correlation
|
||||
|
|
||||
v
|
||||
Fused 128-dim vector
|
||||
|
|
||||
+-------+-------+
|
||||
| |
|
||||
v v
|
||||
Cognitum Seed kNN Search
|
||||
(128-dim store) (similar rooms)
|
||||
```
|
||||
|
||||
### Step 1: CSI-to-Spectrogram Conversion
|
||||
|
||||
Each ESP32 transmits CSI frames via UDP in ADR-018 binary format. The `iq_hex` field contains I/Q pairs for each subcarrier (2 bytes per subcarrier: I + Q as unsigned 8-bit values).
|
||||
|
||||
```
|
||||
Amplitude[sc] = sqrt(I[sc]^2 + Q[sc]^2)
|
||||
```
|
||||
|
||||
A sliding window of 20 frames produces a 64x20 matrix. Normalization to 0-255 grayscale:
|
||||
|
||||
```
|
||||
pixel[sc][t] = clamp(255 * (amplitude[sc][t] - min) / (max - min), 0, 255)
|
||||
```
|
||||
|
||||
Where `min` and `max` are computed over the entire 64x20 window for per-window contrast normalization. This ensures the CNN sees the relative structure regardless of absolute signal strength (which varies with distance, TX power, and environmental absorption).
|
||||
|
||||
### Step 2: CNN Embedding
|
||||
|
||||
The 64x20 grayscale matrix is resized to the CNN's expected input size (224x224 via nearest-neighbor upsampling, since we want to preserve the discrete subcarrier structure rather than blur it with bilinear interpolation). The input is replicated across 3 channels (RGB) since @ruvector/cnn expects RGB input.
|
||||
|
||||
Configuration:
|
||||
- **Input**: 224x224x3 (upsampled from 64x20, grayscale replicated to RGB)
|
||||
- **Embedding dimension**: 128 (reduced from default 512 for compact storage and faster kNN)
|
||||
- **Normalization**: L2-enabled (cosine similarity = dot product on unit sphere)
|
||||
- **Latency**: ~5ms per window on modern hardware
|
||||
|
||||
The 128-dim embedding encodes the 2D structure of the spectrogram: null zones, Doppler patterns, multipath signatures, and activity textures.
|
||||
|
||||
### Step 3: Graph Transformer for Multi-Node Fusion
|
||||
|
||||
With 2 ESP32 nodes (generalizable to N), we construct a graph:
|
||||
|
||||
```
|
||||
Nodes: {Node_1, Node_2}
|
||||
Edges: {(Node_1, Node_2, weight=cross_correlation)}
|
||||
Node features: 128-dim CNN embedding per node
|
||||
```
|
||||
|
||||
The graph attention mechanism learns which node is more informative for each prediction:
|
||||
|
||||
1. **Query/Key/Value** from each node's 128-dim embedding
|
||||
2. **Edge weight** = Pearson cross-correlation between the two nodes' raw amplitude vectors (captures how much their CSI observations agree)
|
||||
3. **Attention score** = softmax(Q_i * K_j / sqrt(d) + edge_weight_bias)
|
||||
4. **Output** = weighted sum of value vectors
|
||||
|
||||
This produces a fused 128-dim vector that combines both nodes' perspectives, automatically weighting the node with cleaner signal (higher SNR, less fading) more heavily.
|
||||
|
||||
**Generalization to 3+ nodes**: Adding a third ESP32 adds one node and 2 edges to the graph. The attention mechanism handles variable-size graphs without architecture changes.
|
||||
|
||||
### Step 4: Storage and Search
|
||||
|
||||
The fused 128-dim embedding is stored in Cognitum Seed (ADR-069) alongside the existing 8-dim features:
|
||||
|
||||
| Store | Dimension | Content | Use Case |
|
||||
|-------|-----------|---------|----------|
|
||||
| `csi-features` | 8-dim | Hand-crafted statistics | Fast presence detection |
|
||||
| `csi-spectrograms` | 128-dim | CNN spectrogram embedding | Environment fingerprinting, anomaly detection |
|
||||
| `csi-spectrograms-fused` | 128-dim | Graph-fused multi-node embedding | Cross-viewpoint room signature |
|
||||
|
||||
kNN search on the 128-dim store finds past spectrograms that "look like" the current one:
|
||||
- **Environment fingerprinting**: "What room does this RF pattern match?"
|
||||
- **Cross-room transfer**: "Which training room is most similar to this deployment room?"
|
||||
- **Anomaly detection**: Low similarity to all known patterns = unknown environment or novel activity
|
||||
- **Temporal segmentation**: Similarity drops = activity transition boundaries
|
||||
|
||||
### Comparison: 8-dim vs 128-dim vs Combined
|
||||
|
||||
| Property | 8-dim hand-crafted | 128-dim CNN | Combined |
|
||||
|----------|-------------------|-------------|----------|
|
||||
| Subcarrier structure | Lost | Preserved | Both available |
|
||||
| Temporal patterns | Lost | Preserved (20-frame window) | Both |
|
||||
| Computation | ~0.1ms | ~5ms | ~5ms |
|
||||
| Storage per vector | 32 bytes | 512 bytes | 544 bytes |
|
||||
| kNN discrimination | Low (8-dim curse) | High (128-dim surface) | Highest |
|
||||
| Interpretability | High (named features) | Low (learned) | Mixed |
|
||||
| Training required | No | Optional (pre-trained works) | Optional |
|
||||
| Multi-node fusion | Average/max | Graph attention | Graph attention |
|
||||
|
||||
### Contrastive Training (Optional Enhancement)
|
||||
|
||||
The CNN embedding works out-of-the-box with the pre-trained weights. For domain-specific improvements, contrastive training with CSI data:
|
||||
|
||||
1. **Positive pairs**: Same room, different time windows (should embed similarly)
|
||||
2. **Negative pairs**: Different rooms or different activities (should embed differently)
|
||||
3. **Loss**: InfoNCE with temperature 0.07 (standard SimCLR)
|
||||
4. **Augmentation**: Time-shift (slide window by 1-5 frames), subcarrier dropout (zero 10% of rows), amplitude jitter (multiply by uniform [0.8, 1.2])
|
||||
|
||||
This teaches the CNN that "same room at different times" should produce similar embeddings, while "different rooms" should produce different embeddings.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
1. **Richer representation**: 128 dimensions capture 2D structure that 8 dimensions cannot
|
||||
2. **Environment fingerprinting**: kNN on spectrograms can distinguish rooms that look identical in 8-dim feature space
|
||||
3. **Activity detection**: Temporal patterns (gait periodicity, breathing frequency) are encoded in the spectrogram texture
|
||||
4. **Multi-node fusion**: Graph attention automatically weights the most informative node, improving robustness to single-node occlusion or interference
|
||||
5. **Incremental adoption**: 128-dim store operates alongside 8-dim store; no migration needed
|
||||
6. **Browser-compatible**: WASM-based CNN runs in the sensing-server UI for live visualization
|
||||
|
||||
### Negative
|
||||
|
||||
1. **5ms latency per window**: Acceptable for 1.3 Hz update rate (750ms rotation from ADR-073), but constrains real-time applications
|
||||
2. **900KB model download**: One-time cost, cached after first load
|
||||
3. **128-dim storage**: 16x more bytes per vector than 8-dim; mitigated by the fact that we store one embedding per 20-frame window (not per frame)
|
||||
4. **Opaque embeddings**: Unlike named 8-dim features, CNN embeddings are not human-interpretable
|
||||
5. **Input size mismatch**: 64x20 matrix must be upsampled to 224x224; nearest-neighbor preserves structure but wastes computation on padded regions
|
||||
|
||||
### Risks and Mitigations
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| CNN embeddings not discriminative enough for CSI | Contrastive fine-tuning on CSI spectrograms; fall back to 8-dim if 128-dim kNN recall is worse |
|
||||
| Graph transformer overhead for 2-node graph | Lightweight attention (single head, no MLP); O(1) for 2 nodes |
|
||||
| Upsampling artifacts from 64x20 to 224x224 | Nearest-neighbor preserves discrete structure; consider training a smaller CNN on native 64x20 input |
|
||||
| WASM initialization delay | Call `init()` at server startup, not per-request |
|
||||
|
||||
## Implementation
|
||||
|
||||
### Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `scripts/csi-spectrogram.js` | CSI-to-spectrogram pipeline with CNN embedding, ASCII visualization, Cognitum Seed ingest |
|
||||
| `scripts/mesh-graph-transformer.js` | Multi-node graph attention fusion using @ruvector/graph-wasm |
|
||||
| `docs/adr/ADR-076-csi-spectrogram-embeddings.md` | This ADR |
|
||||
|
||||
### Dependencies
|
||||
|
||||
| Package | Version | Source |
|
||||
|---------|---------|--------|
|
||||
| `@ruvector/cnn` | 0.1.0 | `vendor/ruvector/npm/packages/ruvector-cnn/` |
|
||||
| `@ruvector/graph-wasm` | 2.0.2 | `vendor/ruvector/npm/packages/graph-wasm/` |
|
||||
|
||||
### Data Format
|
||||
|
||||
CSI JSONL frames from `data/recordings/pretrain-1775182186.csi.jsonl`:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": 1775182186.123,
|
||||
"node_id": 1,
|
||||
"magic": 3289481217,
|
||||
"size": 148,
|
||||
"rssi": -45,
|
||||
"type": "CSI",
|
||||
"iq_hex": "00000f030d030e040d030d030d030c020d020d01...",
|
||||
"subcarriers": 64
|
||||
}
|
||||
```
|
||||
|
||||
`iq_hex` encoding: 2 hex characters per byte, 4 hex characters per subcarrier (I byte + Q byte). Total length = `subcarriers * 4` hex characters.
|
||||
|
||||
## References
|
||||
|
||||
- ADR-018: Binary CSI frame format
|
||||
- ADR-024: AETHER contrastive CSI embeddings (Rust-side)
|
||||
- ADR-029: RuvSense multistatic sensing mode
|
||||
- ADR-069: Cognitum Seed RVF ingest bridge
|
||||
- ADR-073: Multi-frequency mesh scanning
|
||||
- SimCLR: Chen et al., "A Simple Framework for Contrastive Learning of Visual Representations" (2020)
|
||||
- GATv2: Brody et al., "How Attentive are Graph Attention Networks?" (2021)
|
||||
@@ -0,0 +1,284 @@
|
||||
# ADR-077: Novel RF Sensing Applications
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-04-02
|
||||
**Authors:** ruv
|
||||
**Depends on:** ADR-018 (CSI binary protocol), ADR-073 (multifrequency mesh scan), ADR-075 (MinCut person separation), ADR-076 (CSI spectrogram embeddings)
|
||||
|
||||
## Context
|
||||
|
||||
The existing ESP32 CSI + Cognitum Seed infrastructure collects rich multi-modal data:
|
||||
- 2 ESP32-S3 nodes streaming CSI at ~22 fps each (64-128 subcarriers, channel hopping ch 1/3/5/6/9/11)
|
||||
- Vitals extraction: breathing rate, heart rate, motion energy, presence score (1 Hz per node)
|
||||
- 8-dimensional feature vectors per frame
|
||||
- Cognitum Seed with BME280 (temp/humidity/pressure), PIR, reed switch, vibration sensor
|
||||
|
||||
No new hardware is required. All 6 applications below derive novel insights from data already being collected via the ADR-018 binary protocol over UDP port 5006.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement 6 novel RF sensing applications as standalone Node.js scripts that process live UDP or replayed `.csi.jsonl` recordings.
|
||||
|
||||
---
|
||||
|
||||
## Application 1: Sleep Quality Monitoring
|
||||
|
||||
### Input
|
||||
Breathing rate (BR) and heart rate (HR) time series from vitals packets (0xC5110002), sampled at ~1 Hz per node over 6-8 hours.
|
||||
|
||||
### Algorithm
|
||||
Sliding window analysis (5-minute windows, 1-minute stride) classifying sleep stages:
|
||||
|
||||
| Stage | BR (BPM) | BR Variance | HR Pattern | Motion |
|
||||
|-------|----------|-------------|------------|--------|
|
||||
| **Deep (N3)** | 6-12 | Very low (<2.0) | Slow, regular | None |
|
||||
| **Light (N1/N2)** | 12-18 | Moderate (2.0-8.0) | Normal | Minimal |
|
||||
| **REM** | 15-25 | High (>8.0), irregular | Elevated | Eyes only (low CSI motion) |
|
||||
| **Awake** | >18 or <6 | Any | Variable | Moderate-high |
|
||||
|
||||
Each 5-minute window is scored by:
|
||||
1. Compute BR mean and variance within the window
|
||||
2. Compute HR mean and coefficient of variation (CV)
|
||||
3. Compute motion energy mean (from vitals `motion_energy` field)
|
||||
4. Classify stage using threshold hierarchy: Awake > REM > Light > Deep
|
||||
|
||||
### Output
|
||||
- Real-time sleep stage classification
|
||||
- ASCII hypnogram (time vs. stage)
|
||||
- Summary: total sleep time, sleep efficiency (TST / time in bed), time per stage
|
||||
- Optional JSON for health app integration
|
||||
|
||||
### Validation
|
||||
Overnight recording (`overnight-1775217646.csi.jsonl`, 113k frames, ~40 min) should show:
|
||||
- Transition from active (awake) to resting states
|
||||
- Decreased motion energy over time
|
||||
- BR stabilization in sleeping segments
|
||||
|
||||
### Clinical Relevance
|
||||
Consumer-grade sleep tracking without wearables. RF-based sensing avoids compliance issues (forgotten wristbands, dead batteries). Not diagnostic; informational only.
|
||||
|
||||
---
|
||||
|
||||
## Application 2: Breathing Disorder Screening (Apnea Detection)
|
||||
|
||||
### Input
|
||||
Breathing rate time series from vitals packets at ~1 Hz.
|
||||
|
||||
### Algorithm
|
||||
Detect respiratory events in the BR time series:
|
||||
|
||||
| Event | Definition | Duration |
|
||||
|-------|-----------|----------|
|
||||
| **Apnea** | BR drops below 3 BPM (effective cessation) | >= 10 seconds |
|
||||
| **Hypopnea** | BR drops > 50% from 5-min rolling baseline | >= 10 seconds |
|
||||
|
||||
Scoring:
|
||||
1. Maintain 5-minute rolling baseline BR (exponential moving average)
|
||||
2. Flag apnea when BR < 3 BPM for >= 10 consecutive seconds
|
||||
3. Flag hypopnea when BR < 50% of baseline for >= 10 consecutive seconds
|
||||
4. Compute AHI (Apnea-Hypopnea Index) = total events / hours monitored
|
||||
|
||||
| AHI | Severity |
|
||||
|-----|----------|
|
||||
| < 5 | Normal |
|
||||
| 5-15 | Mild |
|
||||
| 15-30 | Moderate |
|
||||
| > 30 | Severe |
|
||||
|
||||
### Output
|
||||
- Per-event log: type (apnea/hypopnea), start time, duration, BR during event
|
||||
- Hourly AHI and overall AHI
|
||||
- Severity classification
|
||||
- Alert on severe events (consecutive apneas > 30s)
|
||||
|
||||
### Clinical Relevance
|
||||
Pre-screening tool for obstructive sleep apnea (OSA). Provides motivation for clinical polysomnography referral. Not a diagnostic device; informational pre-screen only.
|
||||
|
||||
---
|
||||
|
||||
## Application 3: Emotional State / Stress Detection
|
||||
|
||||
### Input
|
||||
Heart rate time series from vitals packets at ~1 Hz.
|
||||
|
||||
### Algorithm
|
||||
Heart Rate Variability (HRV) analysis:
|
||||
|
||||
1. **RMSSD** (Root Mean Square of Successive Differences):
|
||||
- Compute successive HR differences within 5-minute windows
|
||||
- RMSSD = sqrt(mean(diff^2))
|
||||
- High RMSSD = high vagal tone = relaxed
|
||||
- Low RMSSD = sympathetic dominance = stressed
|
||||
|
||||
2. **LF/HF Ratio** (via FFT on 5-minute HR windows):
|
||||
- LF band: 0.04-0.15 Hz (sympathetic + parasympathetic)
|
||||
- HF band: 0.15-0.40 Hz (parasympathetic)
|
||||
- High LF/HF (> 2.0) = stressed
|
||||
- Low LF/HF (< 1.0) = relaxed
|
||||
|
||||
3. **Stress Score** (0-100):
|
||||
- `score = 50 * (1 - RMSSD_norm) + 50 * LF_HF_norm`
|
||||
- Where `RMSSD_norm` = RMSSD / max_expected_RMSSD (capped at 1.0)
|
||||
- And `LF_HF_norm` = min(LF_HF / 4.0, 1.0)
|
||||
|
||||
### Output
|
||||
- Real-time stress score (0-100)
|
||||
- RMSSD and LF/HF ratio per window
|
||||
- ASCII trend chart over hours
|
||||
- Activity context correlation (motion level vs. stress)
|
||||
|
||||
### Validation
|
||||
- Periods of activity (walking, working) should correlate with higher stress scores
|
||||
- Quiet rest should show lower scores
|
||||
- Sleeping should show lowest scores (high HRV, low LF/HF)
|
||||
|
||||
---
|
||||
|
||||
## Application 4: Gait Analysis / Movement Disorder Detection
|
||||
|
||||
### Input
|
||||
- Motion energy time series from vitals packets
|
||||
- CSI phase variance from raw CSI frames (0xC5110001)
|
||||
- Cross-node RSSI from vitals packets
|
||||
|
||||
### Algorithm
|
||||
|
||||
1. **Cadence Extraction**: FFT on motion_energy within 5-second sliding windows
|
||||
- Walking cadence: dominant frequency 0.8-2.0 Hz (normal: ~1.0 Hz = 120 steps/min)
|
||||
- Running: > 2.0 Hz
|
||||
- Stationary: no dominant peak
|
||||
|
||||
2. **Stride Regularity**: Autocorrelation of motion_energy
|
||||
- Regular walking: strong autocorrelation peak at step period
|
||||
- Irregularity score = 1 - (peak_height / baseline)
|
||||
|
||||
3. **Asymmetry Detection**: Compare motion energy oscillation between two ESP32 nodes
|
||||
- Symmetric gait: both nodes see similar oscillation period and amplitude
|
||||
- Asymmetry index = |period_node1 - period_node2| / mean_period
|
||||
|
||||
4. **Tremor Detection**: High-frequency phase variance analysis
|
||||
- Compute phase variance per subcarrier in 2-second windows
|
||||
- Tremor band: 3-8 Hz component in phase variance time series
|
||||
- Parkinsonian tremor: 4-6 Hz, resting
|
||||
- Essential tremor: 5-8 Hz, action
|
||||
|
||||
### Output
|
||||
- Cadence (steps/min)
|
||||
- Stride regularity score (0-1)
|
||||
- Asymmetry index (0 = symmetric, 1 = highly asymmetric)
|
||||
- Tremor score and dominant frequency
|
||||
- Walking vs. stationary classification
|
||||
|
||||
### Validation
|
||||
Overnight data should show clear stationary periods with no cadence detected. Any walking segments should show cadence in the 0.8-2.0 Hz range.
|
||||
|
||||
---
|
||||
|
||||
## Application 5: Material/Object Change Detection
|
||||
|
||||
### Input
|
||||
Per-subcarrier amplitude from raw CSI frames (0xC5110001).
|
||||
|
||||
### Algorithm
|
||||
|
||||
1. **Baseline Establishment** (first 10 minutes or configurable):
|
||||
- Record mean amplitude per subcarrier (Welford online mean)
|
||||
- Record null pattern: which subcarriers are below null threshold (amplitude < 2.0)
|
||||
|
||||
2. **Change Detection** (sliding 30-second windows):
|
||||
- Compare current null pattern to baseline
|
||||
- New nulls appearing = new metal object blocking RF path
|
||||
- Existing nulls disappearing = metal object removed
|
||||
- Null position shifted = object moved
|
||||
- Amplitude change without null change = non-metal material (wood, water, glass)
|
||||
|
||||
3. **Material Classification** heuristic:
|
||||
- Metal: sharp null (amplitude drops to near 0 on specific subcarriers)
|
||||
- Water/human: broad amplitude reduction across many subcarriers
|
||||
- Wood/plastic: minimal amplitude change, mostly phase shift
|
||||
- Glass: frequency-selective (affects higher subcarriers more)
|
||||
|
||||
### Output
|
||||
- Change events with timestamp, type (add/remove/move), affected subcarrier range
|
||||
- Estimated material category
|
||||
- Null pattern delta visualization (ASCII)
|
||||
- Event timeline for monitoring
|
||||
|
||||
### Validation
|
||||
Overnight data has 19% null baseline. Changes in null pattern over the recording period indicate environment changes (doors opening/closing, person entering/leaving).
|
||||
|
||||
---
|
||||
|
||||
## Application 6: Room Environment Fingerprinting
|
||||
|
||||
### Input
|
||||
- 8-dimensional feature vectors from feature packets (0xC5110003)
|
||||
- Motion energy and presence score from vitals packets
|
||||
|
||||
### Algorithm
|
||||
|
||||
1. **Online Clustering** using running k-means (k=5, updateable centroids):
|
||||
- Each incoming 8-dim feature vector is assigned to nearest centroid
|
||||
- Centroid updated via exponential moving average (alpha=0.01)
|
||||
- New cluster created if distance to all centroids exceeds threshold
|
||||
|
||||
2. **State Labeling** (heuristic from vitals correlation):
|
||||
- Cluster with lowest motion_energy = "empty/sleeping"
|
||||
- Cluster with highest motion_energy = "active/walking"
|
||||
- Intermediate clusters = "resting", "working", "transitional"
|
||||
|
||||
3. **Transition Tracking**:
|
||||
- Build state transition matrix (from_state -> to_state counts)
|
||||
- Detect anomalous transitions (rare in historical data)
|
||||
|
||||
4. **Daily Profile**:
|
||||
- Aggregate state durations per hour
|
||||
- Compare across days for routine detection
|
||||
|
||||
### Output
|
||||
- Current room state and confidence
|
||||
- State timeline (ASCII)
|
||||
- Transition matrix
|
||||
- Daily pattern profile
|
||||
- Anomaly score (deviation from established daily pattern)
|
||||
|
||||
### Validation
|
||||
Overnight recording should show 2-3 stable clusters corresponding to activity periods at different times. Transitions should be infrequent and correspond to real behavioral changes.
|
||||
|
||||
---
|
||||
|
||||
## Implementation
|
||||
|
||||
All scripts share common infrastructure:
|
||||
- ADR-018 binary packet parsing (same as rf-scan.js, mincut-person-counter.js)
|
||||
- JSONL replay via readline interface
|
||||
- Live UDP via dgram
|
||||
- Pure Node.js, no external dependencies
|
||||
- CLI: `--replay <file>` for offline, `--port <N>` for live, `--json` for programmatic output
|
||||
|
||||
| Script | Primary Packets | Key Algorithm |
|
||||
|--------|----------------|---------------|
|
||||
| `sleep-monitor.js` | vitals (0xC5110002) | BR/HR window classification |
|
||||
| `apnea-detector.js` | vitals (0xC5110002) | BR pause detection, AHI scoring |
|
||||
| `stress-monitor.js` | vitals (0xC5110002) | HRV RMSSD + FFT LF/HF |
|
||||
| `gait-analyzer.js` | vitals + raw CSI | FFT cadence + phase tremor |
|
||||
| `material-detector.js` | raw CSI (0xC5110001) | Null pattern baseline + delta |
|
||||
| `room-fingerprint.js` | feature (0xC5110003) + vitals | Online k-means clustering |
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- 6 new sensing applications from existing hardware (zero additional cost)
|
||||
- All offline-capable via JSONL replay (no live hardware needed for development)
|
||||
- Pure JS, no native dependencies, runs on any platform with Node.js
|
||||
- Each script is standalone and composable
|
||||
|
||||
### Negative
|
||||
- Vitals accuracy depends on ESP32 CSI quality (RSSI, multipath)
|
||||
- HRV analysis at 1 Hz HR sampling is coarse compared to ECG
|
||||
- Material classification is heuristic, not definitive
|
||||
- Sleep staging without EEG is approximate (consumer-grade accuracy)
|
||||
|
||||
### Risks
|
||||
- Users may misinterpret health-related outputs as clinical diagnoses
|
||||
- Mitigation: all scripts include disclaimers in output headers
|
||||
@@ -0,0 +1,354 @@
|
||||
# ADR-078: Multi-Frequency Mesh Sensing Applications
|
||||
|
||||
| Field | Value |
|
||||
|-------------|--------------------------------------------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-04-02 |
|
||||
| **Authors** | ruv |
|
||||
| **Depends** | ADR-018 (binary frame), ADR-029 (channel hopping), ADR-073 (multi-frequency mesh scan) |
|
||||
|
||||
## Context
|
||||
|
||||
ADR-073 established multi-frequency mesh scanning: 2 ESP32-S3 nodes hopping across 6 WiFi channels (1, 3, 5, 6, 9, 11) with 9 neighbor WiFi networks as passive illuminators. This ADR defines 5 sensing applications that are **unique to multi-frequency mesh scanning** and impossible with single-channel WiFi sensing.
|
||||
|
||||
### Why Multi-Frequency is Required
|
||||
|
||||
Single-channel WiFi sensing captures CSI on one frequency (e.g., channel 5 at 2432 MHz). This provides amplitude and phase across ~52-64 OFDM subcarriers within a 20 MHz bandwidth. Multi-frequency mesh scanning extends this to 6 channels spanning 2412-2462 MHz (50 MHz total), with each channel providing independent multipath observations. The applications below exploit the frequency dimension that single-channel sensing cannot access.
|
||||
|
||||
### Available Infrastructure
|
||||
|
||||
| Resource | Detail |
|
||||
|----------|--------|
|
||||
| Node 1 (COM7) | ESP32-S3, channels 1, 6, 11 (non-overlapping), 200ms dwell |
|
||||
| Node 2 | ESP32-S3, channels 3, 5, 9 (interleaved, near neighbor APs), 200ms dwell |
|
||||
| Neighbor APs | 9 networks across channels 3, 5, 6, 9, 11 |
|
||||
| Data transport | UDP port 5006, ADR-018 binary format |
|
||||
| Recorded data | `data/recordings/overnight-*.csi.jsonl` |
|
||||
|
||||
### Neighbor AP Illuminator Table
|
||||
|
||||
| SSID | Channel | Freq (MHz) | Signal (%) | Role |
|
||||
|------|---------|------------|------------|------|
|
||||
| ruv.net | 5 | 2432 | 100 | Primary illuminator |
|
||||
| Cohen-Guest | 5 | 2432 | 100 | Co-channel illuminator |
|
||||
| COGECO-21B20 | 11 | 2462 | 100 | High-freq illuminator |
|
||||
| HP M255 LaserJet | 5 | 2432 | 94 | Device fingerprinting target |
|
||||
| conclusion mesh | 3 | 2422 | 44 | Low-freq illuminator |
|
||||
| NETGEAR72 | 9 | 2452 | 42 | Mid-high illuminator |
|
||||
| NETGEAR72-Guest | 9 | 2452 | 42 | Co-channel illuminator |
|
||||
| COGECO-4321 | 11 | 2462 | 30 | Weak high-freq illuminator |
|
||||
| Innanen | 6 | 2437 | 19 | Weak center-band illuminator |
|
||||
|
||||
## Decision
|
||||
|
||||
Implement 5 multi-frequency-specific sensing applications, each as a standalone Node.js script in `scripts/`.
|
||||
|
||||
---
|
||||
|
||||
## Application 1: RF Tomographic Imaging
|
||||
|
||||
### Principle
|
||||
|
||||
Each WiFi channel "sees" through the room differently because multipath interference patterns are frequency-dependent. A 2 cm path length difference produces a null at 2432 MHz but constructive interference at 2412 MHz. With 6 channels x 2 nodes, we have 12 independent RF path observations through the room.
|
||||
|
||||
RF tomography back-projects attenuation along each transmitter-receiver path. Where paths overlap with high attenuation, there is an absorbing object (person, furniture, wall). Where paths show low attenuation, the space is clear.
|
||||
|
||||
### Algorithm
|
||||
|
||||
```
|
||||
For each CSI frame:
|
||||
1. Compute path attenuation = RSSI_free_space - RSSI_measured
|
||||
2. For each cell in a 10x10 room grid:
|
||||
a. Compute the cell's distance to the TX->RX line (perpendicular distance)
|
||||
b. Weight contribution by 1/distance (cells near the path contribute more)
|
||||
3. Accumulate weighted attenuation across all frames, channels, and node pairs
|
||||
4. Normalize: cells with high accumulated attenuation = absorbers (people/objects)
|
||||
```
|
||||
|
||||
Uses the Algebraic Reconstruction Technique (ART) for iterative refinement, or simple backprojection for real-time display.
|
||||
|
||||
### Resolution
|
||||
|
||||
- Theoretical: ~lambda/2 = 6 cm (at 2.4 GHz)
|
||||
- Practical with 2 nodes: ~20 cm (limited by node geometry)
|
||||
- Frequency diversity gain: sqrt(6) improvement over single-channel = ~2.4x
|
||||
|
||||
### Why Single-Channel Cannot Do This
|
||||
|
||||
Single-channel provides only 1 frequency observation per path. Frequency-selective fading means a single channel may show zero attenuation through a person (if the path happens to be at a constructive interference point). Multiple channels provide independent attenuation measurements through the same spatial path, enabling reliable detection.
|
||||
|
||||
### Script
|
||||
|
||||
`scripts/rf-tomography.js`
|
||||
|
||||
---
|
||||
|
||||
## Application 2: Passive Bistatic Radar
|
||||
|
||||
### Principle
|
||||
|
||||
Neighbor WiFi APs transmit continuously and uncontrollably. The ESP32 nodes capture CSI from these transmissions, which includes phase and amplitude modulated by objects in the room. Each neighbor AP acts as a free "illuminator of opportunity" at a known position and frequency.
|
||||
|
||||
This is the same principle used by military passive radar systems (e.g., the Ukrainian Kolchuga, Czech VERA-NG) that use FM radio and TV transmitters to detect aircraft without emitting any signals themselves. Here we use WiFi APs instead of broadcast towers, and detect people instead of aircraft.
|
||||
|
||||
### Algorithm
|
||||
|
||||
```
|
||||
For each neighbor AP (identified by BSSID/channel):
|
||||
1. Track CSI phase progression across consecutive frames
|
||||
2. Compute Doppler shift: fd = d(phase)/dt / (2*pi)
|
||||
- Positive Doppler = target moving toward the AP
|
||||
- Negative Doppler = target moving away
|
||||
3. Compute range from subcarrier phase slope:
|
||||
- tau = d(phase)/d(subcarrier_freq) / (2*pi)
|
||||
- range = c * tau (where c = speed of light)
|
||||
4. Build range-Doppler map per AP
|
||||
5. Fuse multi-static detections:
|
||||
- Each AP provides a range ellipse (locus of constant TX->target->RX delay)
|
||||
- Intersection of 3+ ellipses = target position
|
||||
```
|
||||
|
||||
### Multi-Static Geometry
|
||||
|
||||
With 3+ neighbor APs as transmitters and 2 ESP32 receivers, we have 6+ bistatic pairs. Each pair constrains the target to an ellipse. The intersection provides 2D position.
|
||||
|
||||
```
|
||||
AP1 (ch5) AP2 (ch11)
|
||||
\ /
|
||||
\ TARGET /
|
||||
\ /|\ /
|
||||
\ / | \ /
|
||||
ESP32_1 ---*--+--*--- ESP32_2
|
||||
/ \ | / \
|
||||
/ \|/ \
|
||||
/ TARGET \
|
||||
/ \
|
||||
AP3 (ch3) AP4 (ch9)
|
||||
```
|
||||
|
||||
### Why Single-Channel Cannot Do This
|
||||
|
||||
Single-channel only captures CSI from APs on that one channel. With channel 5, you see ruv.net and Cohen-Guest, but miss COGECO-21B20 (ch11), conclusion mesh (ch3), NETGEAR72 (ch9). Multi-frequency scanning captures illumination from all 9 APs across 6 channels, providing the geometric diversity needed for position triangulation.
|
||||
|
||||
### Script
|
||||
|
||||
`scripts/passive-radar.js`
|
||||
|
||||
---
|
||||
|
||||
## Application 3: Frequency-Selective Material Classification
|
||||
|
||||
### Principle
|
||||
|
||||
Different materials interact with 2.4 GHz WiFi signals differently, and critically, their absorption/reflection varies with frequency:
|
||||
|
||||
| Material | Attenuation Pattern | Frequency Dependence |
|
||||
|----------|--------------------|--------------------|
|
||||
| Metal | Total reflection, deep null | Frequency-flat (blocks all equally) |
|
||||
| Water/Human body | Strong absorption | Increases with frequency (dielectric loss ~ f^2) |
|
||||
| Wood | Mild attenuation | Increases with frequency (moisture content) |
|
||||
| Glass | Low attenuation | Nearly frequency-flat |
|
||||
| Drywall | Low-moderate attenuation | Slight frequency dependence |
|
||||
| Concrete | Moderate-high attenuation | Increases with frequency |
|
||||
|
||||
### Algorithm
|
||||
|
||||
```
|
||||
For each subcarrier index i across all channels:
|
||||
1. Measure attenuation A(i, ch) on each channel
|
||||
2. Compute frequency selectivity:
|
||||
- Flat ratio = std(A across channels) / mean(A across channels)
|
||||
- Slope = linear regression of A vs frequency
|
||||
3. Classify:
|
||||
- Flat ratio < 0.1 AND high attenuation -> Metal
|
||||
- Flat ratio < 0.1 AND low attenuation -> Glass/Air
|
||||
- Positive slope (A increases with freq) AND high A -> Water/Human
|
||||
- Positive slope AND moderate A -> Wood
|
||||
- High variance across channels -> Complex scatterer
|
||||
```
|
||||
|
||||
### Physics Basis
|
||||
|
||||
At 2.4 GHz, water's complex permittivity is epsilon_r = 77 - j10. The imaginary component (loss) increases with frequency within the WiFi band. Metal is a perfect conductor regardless of frequency. Glass (epsilon_r ~ 6 - j0.1) has negligible loss at all WiFi frequencies.
|
||||
|
||||
The 50 MHz span (2412-2462 MHz) is only ~2% of the carrier frequency, but this is sufficient to detect the frequency-dependent absorption signature of water-bearing materials (human body, wet wood, potted plants) versus frequency-flat materials (metal, glass).
|
||||
|
||||
### Why Single-Channel Cannot Do This
|
||||
|
||||
Material classification requires measuring how attenuation varies with frequency. A single channel provides only one frequency point -- there is no frequency axis to measure against. Multi-frequency scanning provides 6 frequency points spanning 50 MHz, enabling slope and variance computation.
|
||||
|
||||
### Script
|
||||
|
||||
`scripts/material-classifier.js`
|
||||
|
||||
---
|
||||
|
||||
## Application 4: Through-Wall Motion Detection
|
||||
|
||||
### Principle
|
||||
|
||||
Lower WiFi frequencies penetrate walls better than higher frequencies. At 2.4 GHz, wall attenuation for a standard drywall+stud partition is approximately:
|
||||
|
||||
| Channel | Freq (MHz) | Drywall Loss (dB) | Concrete Loss (dB) |
|
||||
|---------|------------|-------------------|-------------------|
|
||||
| 1 | 2412 | 2.5 | 8.0 |
|
||||
| 6 | 2437 | 2.6 | 8.3 |
|
||||
| 11 | 2462 | 2.7 | 8.6 |
|
||||
|
||||
The absolute differences are small (~0.2 dB), but with 6 channels we can:
|
||||
|
||||
1. **Baseline the wall's frequency-dependent attenuation profile** during a calibration period (no one behind the wall)
|
||||
2. **Detect changes above baseline** that indicate motion behind the wall
|
||||
3. **Weight lower channels more heavily** since they have better through-wall SNR
|
||||
4. **Cross-validate** across channels: real through-wall motion appears on all channels (with frequency-dependent amplitude), while interference/noise typically appears on only one channel
|
||||
|
||||
### Algorithm
|
||||
|
||||
```
|
||||
Calibration phase (60 seconds, no motion behind wall):
|
||||
For each channel ch:
|
||||
baseline_mean[ch] = mean(CSI amplitude over calibration)
|
||||
baseline_std[ch] = std(CSI amplitude over calibration)
|
||||
|
||||
Detection phase:
|
||||
For each frame on channel ch:
|
||||
1. Compute deviation = |current_amplitude - baseline_mean[ch]| / baseline_std[ch]
|
||||
2. Channel weight = f(penetration_quality[ch])
|
||||
3. Per-channel score = deviation * weight
|
||||
|
||||
Fused score = weighted sum across channels
|
||||
Alert if fused_score > threshold for N consecutive frames
|
||||
```
|
||||
|
||||
### Why Single-Channel Cannot Do This
|
||||
|
||||
Single-channel through-wall detection suffers from high false-positive rates because it cannot distinguish wall effects from motion. With multi-frequency, we can:
|
||||
|
||||
1. Characterize the wall's frequency response during calibration
|
||||
2. Subtract the wall effect per channel
|
||||
3. Cross-validate detections across channels (real motion is coherent across frequencies; noise is not)
|
||||
|
||||
The frequency diversity provides a ~2.4x improvement in detection SNR (sqrt(6) independent observations).
|
||||
|
||||
### Script
|
||||
|
||||
`scripts/through-wall-detector.js`
|
||||
|
||||
---
|
||||
|
||||
## Application 5: Device Fingerprinting via RF Emissions
|
||||
|
||||
### Principle
|
||||
|
||||
Every electronic device has unique RF characteristics visible in the WiFi spectrum. When a device transmits (or even when its internal oscillators radiate EMI), it modulates nearby WiFi signals in device-specific ways:
|
||||
|
||||
- **WiFi APs**: each AP has unique transmit power, phase noise, and clock drift characteristics
|
||||
- **Printers**: the HP M255 LaserJet creates specific subcarrier patterns when printing (motor EMI)
|
||||
- **Microwave ovens**: 2.45 GHz magnetron radiates across channels 8-11, creating distinctive wideband interference
|
||||
- **Bluetooth devices**: 2.4 GHz frequency-hopping creates transient spikes across channels
|
||||
|
||||
### Algorithm
|
||||
|
||||
```
|
||||
Learning phase:
|
||||
For each known device (from WiFi scan SSID/BSSID correlation):
|
||||
1. Record CSI patterns when device is active vs inactive
|
||||
2. Compute per-channel signature:
|
||||
- Mean amplitude profile across subcarriers
|
||||
- Variance profile (active devices increase variance on specific subcarriers)
|
||||
- Phase noise characteristics
|
||||
3. Store signature as device fingerprint
|
||||
|
||||
Detection phase:
|
||||
For each analysis window:
|
||||
1. Compute current CSI profile per channel
|
||||
2. Correlate against stored fingerprints
|
||||
3. Report device activity: "HP printer active (confidence 0.87)"
|
||||
```
|
||||
|
||||
### Multi-Frequency Advantage
|
||||
|
||||
Different devices affect different channels:
|
||||
|
||||
- HP printer (ch5): affects subcarriers 20-40 on channel 5 during print jobs
|
||||
- NETGEAR72 router (ch9): creates clock-drift correlated phase patterns on channel 9
|
||||
- Microwave: broadband interference strongest on channels 9-11
|
||||
|
||||
Single-channel sensing only sees devices that affect that one channel. Multi-frequency scanning observes the full 2412-2462 MHz band, detecting device activity regardless of which channel the device operates on.
|
||||
|
||||
### Script
|
||||
|
||||
`scripts/device-fingerprint.js`
|
||||
|
||||
---
|
||||
|
||||
## Implementation
|
||||
|
||||
### Shared Infrastructure
|
||||
|
||||
All 5 scripts share common infrastructure:
|
||||
|
||||
| Component | Detail |
|
||||
|-----------|--------|
|
||||
| Packet format | ADR-018 binary (UDP) or .csi.jsonl (replay) |
|
||||
| IQ parsing | `parseIqHex()` for JSONL, `parseCSIFrame()` for binary UDP |
|
||||
| Channel assignment | From binary freq field, or simulated round-robin for legacy JSONL |
|
||||
| Node positions | Configurable, default: Node 1 at (0,0), Node 2 at (3,0) meters |
|
||||
| Visualization | ASCII Unicode block characters and box drawing |
|
||||
|
||||
### Scripts
|
||||
|
||||
| Script | Application | Lines | Key Algorithm |
|
||||
|--------|------------|-------|---------------|
|
||||
| `scripts/rf-tomography.js` | RF Tomographic Imaging | ~500 | ART backprojection |
|
||||
| `scripts/passive-radar.js` | Passive Bistatic Radar | ~500 | Range-Doppler + multi-static fusion |
|
||||
| `scripts/material-classifier.js` | Material Classification | ~450 | Frequency-selective attenuation analysis |
|
||||
| `scripts/through-wall-detector.js` | Through-Wall Detection | ~400 | Baselined multi-channel anomaly detection |
|
||||
| `scripts/device-fingerprint.js` | Device Fingerprinting | ~450 | Per-channel signature correlation |
|
||||
|
||||
### Data Requirements
|
||||
|
||||
- **Live mode**: UDP port 5006, 2 ESP32 nodes channel-hopping per ADR-073
|
||||
- **Replay mode**: `--replay <file.csi.jsonl>` with overnight recordings
|
||||
- **Calibration**: through-wall detector requires 60s calibration with `--calibrate`
|
||||
|
||||
## Performance Targets
|
||||
|
||||
| Application | Latency | Update Rate | Accuracy Target |
|
||||
|-------------|---------|-------------|-----------------|
|
||||
| RF Tomography | <100ms per frame | 1 Hz image update | 20 cm spatial resolution |
|
||||
| Passive Radar | <200ms per frame | 2 Hz range-Doppler | 1 m range, 0.1 m/s velocity |
|
||||
| Material Classification | <500ms per window | 0.5 Hz classification | 70% correct material ID |
|
||||
| Through-Wall Detection | <100ms per frame | 2 Hz detection | 90% true positive, <10% false positive |
|
||||
| Device Fingerprinting | <1s per window | 0.2 Hz activity update | 80% correct device ID |
|
||||
|
||||
## Risks
|
||||
|
||||
### Limited Frequency Span
|
||||
|
||||
The 50 MHz span (2412-2462 MHz) is only 2% of the carrier frequency. Material classification accuracy depends on the attenuation slope being measurable within this narrow range. Mitigation: use long averaging windows (5-10 seconds) to improve SNR of frequency-dependent measurements.
|
||||
|
||||
### Node Geometry
|
||||
|
||||
2 nodes provide limited spatial diversity for tomographic imaging. The backprojection is essentially 1D along the node-to-node axis, with poor resolution perpendicular to it. Mitigation: neighbor APs provide additional geometric diversity for passive radar mode.
|
||||
|
||||
### Legacy Data Compatibility
|
||||
|
||||
Overnight recordings (`data/recordings/overnight-*.csi.jsonl`) were captured before multi-frequency scanning was deployed and lack channel/frequency fields. Scripts simulate channel assignment for replay. Full multi-frequency data requires re-recording with channel hopping enabled.
|
||||
|
||||
### Phase Calibration
|
||||
|
||||
Passive radar requires accurate phase tracking across consecutive frames. ESP32 CSI phase includes a random offset per channel hop that must be removed. Mitigation: use phase-difference between consecutive frames rather than absolute phase.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **5 GHz multi-frequency**: rejected -- no 5 GHz APs visible in environment, no free illuminators.
|
||||
2. **UWB (ultra-wideband)**: rejected -- ESP32-S3 does not support UWB. Would require additional hardware (DW1000/DW3000 modules).
|
||||
3. **Dedicated radar hardware**: rejected -- multi-frequency WiFi sensing achieves similar capabilities using existing infrastructure at zero additional cost.
|
||||
|
||||
## References
|
||||
|
||||
- Wilson, J. & Patwari, N. (2010). "Radio Tomographic Imaging with Wireless Networks." IEEE Trans. Mobile Computing.
|
||||
- Colone, F. et al. (2012). "WiFi-Based Passive Bistatic Radar: Data Processing Schemes and Experimental Results." IEEE Trans. Aerospace and Electronic Systems.
|
||||
- Adib, F. & Katabi, D. (2013). "See Through Walls with WiFi!" ACM SIGCOMM.
|
||||
- Banerjee, A. et al. (2014). "RF-based material identification using WiFi signals." ACM MobiCom.
|
||||
@@ -0,0 +1,115 @@
|
||||
# Architecture Decision Records
|
||||
|
||||
This folder contains 44 Architecture Decision Records (ADRs) that document every significant technical choice in the RuView / WiFi-DensePose project.
|
||||
|
||||
## Why ADRs?
|
||||
|
||||
Building a system that turns WiFi signals into human pose estimation involves hundreds of non-obvious decisions: which signal processing algorithms to use, how to bridge ESP32 firmware to a Rust pipeline, whether to run inference on-device or on a server, how to handle multi-person separation with limited subcarriers.
|
||||
|
||||
ADRs capture the **context**, **options considered**, **decision made**, and **consequences** for each of these choices. They serve three purposes:
|
||||
|
||||
1. **Institutional memory** — Six months from now, anyone (human or AI) can read *why* we chose IIR bandpass filters over FIR for vital sign extraction, not just see the code.
|
||||
|
||||
2. **AI-assisted development** — When an AI agent works on this codebase, ADRs give it the constraints and rationale it needs to make changes that align with the existing architecture. Without them, AI-generated code tends to drift — reinventing patterns that already exist, contradicting earlier decisions, or optimizing for the wrong tradeoffs.
|
||||
|
||||
3. **Review checkpoints** — Each ADR is a reviewable artifact. When a proposed change touches the architecture, the ADR forces the author to articulate tradeoffs *before* writing code, not after.
|
||||
|
||||
### ADRs and Domain-Driven Design
|
||||
|
||||
The project uses [Domain-Driven Design](../ddd/) (DDD) to organize code into bounded contexts — each with its own language, types, and responsibilities. ADRs and DDD work together:
|
||||
|
||||
- **ADRs define boundaries**: ADR-029 (RuvSense) established multistatic sensing as a separate bounded context from single-node CSI. ADR-042 (CHCI) defined a new aggregate root for coherent channel imaging.
|
||||
- **DDD models define the language**: The [RuvSense domain model](../ddd/ruvsense-domain-model.md) defines terms like "coherence gate", "dwell time", and "TDM slot" that ADRs reference precisely.
|
||||
- **Together they prevent drift**: An AI agent reading ADR-039 knows that edge processing tiers are configured via NVS keys, not compile-time flags — because the ADR says so. The DDD model tells it which aggregate owns that configuration.
|
||||
|
||||
### How ADRs are structured
|
||||
|
||||
Each ADR follows a consistent format:
|
||||
|
||||
- **Context** — What problem or gap prompted this decision
|
||||
- **Decision** — What we chose to do and how
|
||||
- **Consequences** — What improved, what got harder, and what risks remain
|
||||
- **References** — Related ADRs, papers, and code paths
|
||||
|
||||
Statuses: **Proposed** (under discussion), **Accepted** (approved and/or implemented), **Superseded** (replaced by a later ADR).
|
||||
|
||||
---
|
||||
|
||||
## ADR Index
|
||||
|
||||
### Hardware and firmware
|
||||
|
||||
| ADR | Title | Status |
|
||||
|-----|-------|--------|
|
||||
| [ADR-012](ADR-012-esp32-csi-sensor-mesh.md) | ESP32 CSI Sensor Mesh for Distributed Sensing | Accepted (partial) |
|
||||
| [ADR-018](ADR-018-esp32-dev-implementation.md) | ESP32 Development Implementation Path | Proposed |
|
||||
| [ADR-028](ADR-028-esp32-capability-audit.md) | ESP32 Capability Audit and Witness Record | Accepted |
|
||||
| [ADR-029](ADR-029-ruvsense-multistatic-sensing-mode.md) | RuvSense Multistatic Sensing Mode (TDM, channel hopping) | Proposed |
|
||||
| [ADR-032](ADR-032-multistatic-mesh-security-hardening.md) | Multistatic Mesh Security Hardening | Accepted |
|
||||
| [ADR-039](ADR-039-esp32-edge-intelligence.md) | ESP32-S3 Edge Intelligence Pipeline (on-device vitals) | Accepted (hardware-validated) |
|
||||
| [ADR-040](ADR-040-wasm-programmable-sensing.md) | WASM Programmable Sensing (Tier 3) | Accepted |
|
||||
| [ADR-041](ADR-041-wasm-module-collection.md) | WASM Module Collection (65 edge modules) | Accepted (hardware-validated) |
|
||||
| [ADR-044](ADR-044-provisioning-tool-enhancements.md) | Provisioning Tool Enhancements | Proposed |
|
||||
|
||||
### Signal processing and sensing
|
||||
|
||||
| ADR | Title | Status |
|
||||
|-----|-------|--------|
|
||||
| [ADR-013](ADR-013-feature-level-sensing-commodity-gear.md) | Feature-Level Sensing on Commodity Gear | Accepted |
|
||||
| [ADR-014](ADR-014-sota-signal-processing.md) | SOTA Signal Processing Algorithms | Accepted |
|
||||
| [ADR-021](ADR-021-vital-sign-detection-rvdna-pipeline.md) | Vital Sign Detection (breathing, heart rate) | Partial |
|
||||
| [ADR-030](ADR-030-ruvsense-persistent-field-model.md) | Persistent Field Model and Drift Detection | Proposed |
|
||||
| [ADR-033](ADR-033-crv-signal-line-sensing-integration.md) | CRV Signal Line Sensing Integration | Proposed |
|
||||
| [ADR-037](ADR-037-multi-person-pose-detection.md) | Multi-Person Pose Detection from Single ESP32 | Proposed |
|
||||
| [ADR-042](ADR-042-coherent-human-channel-imaging.md) | Coherent Human Channel Imaging (beyond CSI) | Proposed |
|
||||
|
||||
### Machine learning and training
|
||||
|
||||
| ADR | Title | Status |
|
||||
|-----|-------|--------|
|
||||
| [ADR-005](ADR-005-sona-self-learning-pose-estimation.md) | SONA Self-Learning for Pose Estimation | Partial |
|
||||
| [ADR-006](ADR-006-gnn-enhanced-csi-pattern-recognition.md) | GNN-Enhanced CSI Pattern Recognition | Partial |
|
||||
| [ADR-015](ADR-015-public-dataset-training-strategy.md) | Public Dataset Strategy (MM-Fi, Wi-Pose) | Accepted |
|
||||
| [ADR-016](ADR-016-ruvector-integration.md) | RuVector Training Pipeline Integration | Accepted |
|
||||
| [ADR-017](ADR-017-ruvector-signal-mat-integration.md) | RuVector Signal + MAT Integration | Proposed |
|
||||
| [ADR-020](ADR-020-rust-ruvector-ai-model-migration.md) | Migrate AI Inference to Rust (ONNX Runtime) | Accepted |
|
||||
| [ADR-023](ADR-023-trained-densepose-model-ruvector-pipeline.md) | Trained DensePose Model with RuVector Pipeline | Proposed |
|
||||
| [ADR-024](ADR-024-contrastive-csi-embedding-model.md) | Project AETHER: Contrastive CSI Embeddings | Required |
|
||||
| [ADR-027](ADR-027-cross-environment-domain-generalization.md) | Project MERIDIAN: Cross-Environment Generalization | Proposed |
|
||||
|
||||
### Platform and UI
|
||||
|
||||
| ADR | Title | Status |
|
||||
|-----|-------|--------|
|
||||
| [ADR-019](ADR-019-sensing-only-ui-mode.md) | Sensing-Only UI with Gaussian Splats | Accepted |
|
||||
| [ADR-022](ADR-022-windows-wifi-enhanced-fidelity-ruvector.md) | Windows WiFi Enhanced Fidelity (multi-BSSID) | Partial |
|
||||
| [ADR-025](ADR-025-macos-corewlan-wifi-sensing.md) | macOS CoreWLAN WiFi Sensing | Proposed |
|
||||
| [ADR-031](ADR-031-ruview-sensing-first-rf-mode.md) | RuView Sensing-First RF Mode | Proposed |
|
||||
| [ADR-034](ADR-034-expo-mobile-app.md) | Expo React Native Mobile App | Accepted |
|
||||
| [ADR-035](ADR-035-live-sensing-ui-accuracy.md) | Live Sensing UI Accuracy and Data Transparency | Accepted |
|
||||
| [ADR-036](ADR-036-rvf-training-pipeline-ui.md) | Training Pipeline UI Integration | Proposed |
|
||||
| [ADR-043](ADR-043-sensing-server-ui-api-completion.md) | Sensing Server UI API Completion (14 endpoints) | Accepted |
|
||||
|
||||
### Architecture and infrastructure
|
||||
|
||||
| ADR | Title | Status |
|
||||
|-----|-------|--------|
|
||||
| [ADR-001](ADR-001-wifi-mat-disaster-detection.md) | WiFi-Mat Disaster Detection Architecture | Accepted |
|
||||
| [ADR-002](ADR-002-ruvector-rvf-integration-strategy.md) | RuVector RVF Integration Strategy | Superseded |
|
||||
| [ADR-003](ADR-003-rvf-cognitive-containers-csi.md) | RVF Cognitive Containers for CSI | Proposed |
|
||||
| [ADR-004](ADR-004-hnsw-vector-search-fingerprinting.md) | HNSW Vector Search for Fingerprinting | Partial |
|
||||
| [ADR-007](ADR-007-post-quantum-cryptography-secure-sensing.md) | Post-Quantum Cryptography for Sensing | Proposed |
|
||||
| [ADR-008](ADR-008-distributed-consensus-multi-ap.md) | Distributed Consensus for Multi-AP | Proposed |
|
||||
| [ADR-009](ADR-009-rvf-wasm-runtime-edge-deployment.md) | RVF WASM Runtime for Edge Deployment | Proposed |
|
||||
| [ADR-010](ADR-010-witness-chains-audit-trail-integrity.md) | Witness Chains for Audit Trail Integrity | Proposed |
|
||||
| [ADR-011](ADR-011-python-proof-of-reality-mock-elimination.md) | Proof-of-Reality and Mock Elimination | Proposed |
|
||||
| [ADR-026](ADR-026-survivor-track-lifecycle.md) | Survivor Track Lifecycle (MAT crate) | Accepted |
|
||||
| [ADR-038](ADR-038-sublinear-goal-oriented-action-planning.md) | Sublinear GOAP for Roadmap Optimization | Proposed |
|
||||
|
||||
---
|
||||
|
||||
## Related
|
||||
|
||||
- [DDD Domain Models](../ddd/) — Bounded context definitions, aggregate roots, and ubiquitous language
|
||||
- [User Guide](../user-guide.md) — Setup, API reference, and hardware instructions
|
||||
- [Build Guide](../build-guide.md) — Building from source
|
||||
@@ -0,0 +1,34 @@
|
||||
# Domain Models
|
||||
|
||||
This folder contains Domain-Driven Design (DDD) specifications for each major subsystem in RuView.
|
||||
|
||||
DDD organizes the codebase around the problem being solved — not around technical layers. Each *bounded context* owns its own data, rules, and language. Contexts communicate through domain events, not by sharing mutable state. This makes the system easier to reason about, test, and extend — whether you're a person or an AI agent.
|
||||
|
||||
## Models
|
||||
|
||||
| Model | What it covers | Bounded Contexts |
|
||||
|-------|---------------|------------------|
|
||||
| [RuvSense](ruvsense-domain-model.md) | Multistatic WiFi sensing, pose tracking, vital signs, edge intelligence | 7 contexts: Sensing, Coherence, Tracking, Field Model, Longitudinal, Spatial Identity, Edge Intelligence |
|
||||
| [Signal Processing](signal-processing-domain-model.md) | SOTA signal processing: phase cleaning, feature extraction, motion analysis | 3 contexts: CSI Preprocessing, Feature Extraction, Motion Analysis |
|
||||
| [Training Pipeline](training-pipeline-domain-model.md) | ML training: datasets, model architecture, embeddings, domain generalization | 4 contexts: Dataset Management, Model Architecture, Training Orchestration, Embedding & Transfer |
|
||||
| [Hardware Platform](hardware-platform-domain-model.md) | ESP32 firmware, edge intelligence, WASM runtime, aggregation, provisioning | 5 contexts: Sensor Node, Edge Processing, WASM Runtime, Aggregation, Provisioning |
|
||||
| [Sensing Server](sensing-server-domain-model.md) | Single-binary Axum server: CSI ingestion, model management, recording, training, visualization | 5 contexts: CSI Ingestion, Model Management, CSI Recording, Training Pipeline, Visualization |
|
||||
| [WiFi-Mat](wifi-mat-domain-model.md) | Disaster response: survivor detection, START triage, mass casualty assessment | 3 contexts: Detection, Localization, Alerting |
|
||||
| [CHCI](chci-domain-model.md) | Coherent Human Channel Imaging: sub-millimeter body surface reconstruction | 3 contexts: Sounding, Channel Estimation, Imaging |
|
||||
|
||||
## How to read these
|
||||
|
||||
Each model defines:
|
||||
|
||||
- **Ubiquitous Language** — Terms with precise meanings used in both code and conversation
|
||||
- **Bounded Contexts** — Independent subsystems with clear responsibilities and boundaries
|
||||
- **Aggregates** — Clusters of objects that enforce business rules (e.g., a PoseTrack owns its keypoints)
|
||||
- **Value Objects** — Immutable data with meaning (e.g., a CoherenceScore is not just a float)
|
||||
- **Domain Events** — Things that happened that other contexts may care about
|
||||
- **Invariants** — Rules that must always be true (e.g., "drift alert requires >2sigma for >3 days")
|
||||
- **Anti-Corruption Layers** — Adapters that translate between contexts without leaking internals
|
||||
|
||||
## Related
|
||||
|
||||
- [Architecture Decision Records](../adr/README.md) — Why each technical choice was made
|
||||
- [User Guide](../user-guide.md) — Setup and API reference
|
||||
@@ -0,0 +1,926 @@
|
||||
# Coherent Human Channel Imaging (CHCI) Domain Model
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
### Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **Coherent Human Channel Imaging (CHCI)** | A purpose-built RF sensing protocol that uses phase-locked sounding, multi-band fusion, and cognitive waveform adaptation to reconstruct human body surfaces and physiological motion at sub-millimeter resolution |
|
||||
| **Sounding Frame** | A deterministic OFDM transmission (NDP or custom burst) with known pilot structure, transmitted at fixed cadence for channel measurement — as opposed to passive CSI extracted from data traffic |
|
||||
| **Phase Coherence** | The property of multiple radio nodes sharing a common phase reference, enabling complex-valued channel measurements without per-node LO drift correction |
|
||||
| **Reference Clock** | A shared oscillator (TCXO + PLL) distributed to all CHCI nodes via coaxial cable, providing both 40 MHz timing reference and in-band phase reference signal |
|
||||
| **Cognitive Waveform** | A sounding waveform whose parameters (cadence, bandwidth, band selection, power, subcarrier subset) adapt in real-time based on the current scene state inferred from the body model |
|
||||
| **Diffraction Tomography** | Coherent reconstruction of body surface geometry from complex-valued channel responses across multiple node pairs and frequency bands — produces surface contours rather than volumetric opacity |
|
||||
| **Sensing Mode** | One of six operational states (IDLE, ALERT, ACTIVE, VITAL, GESTURE, SLEEP) that determine waveform parameters and processing pipeline configuration |
|
||||
| **Micro-Burst** | A very short (4–20 μs) deterministic OFDM symbol transmitted at high cadence (1–5 kHz) for maximizing Doppler resolution without full 802.11 frame overhead |
|
||||
| **Multi-Band Fusion** | Simultaneous sounding at 2.4 GHz and 5 GHz (optionally 6 GHz), fused as projections of the same latent motion field using body model priors as constraints |
|
||||
| **Displacement Floor** | The minimum detectable surface displacement at a given range, determined by phase noise, coherent averaging depth, and antenna count: δ_min = λ/(4π) × σ_φ/√(N_ant × N_avg) |
|
||||
| **Channel Contrast** | The ratio of complex channel response with human present to the empty-room reference response — the input to diffraction tomography |
|
||||
| **Coherence Delta** | The change in phase coherence metric between consecutive observation windows — the trigger signal for cognitive waveform transitions |
|
||||
| **NDP** | Null Data PPDU — an 802.11bf-standard sounding frame containing only preamble and training fields, no data payload |
|
||||
| **Sensing Availability Window (SAW)** | An 802.11bf-defined time interval during which NDP sounding exchanges are permitted between sensing initiator and responder |
|
||||
| **Body Model Prior** | Geometric constraints derived from known human body dimensions (segment lengths, joint angle limits) used to regularize cross-band fusion and tomographic reconstruction |
|
||||
| **Phase Reference Signal** | A continuous-wave tone at the operating band center frequency, distributed alongside the 40 MHz clock, enabling all nodes to measure and compensate residual phase offset |
|
||||
|
||||
---
|
||||
|
||||
## Bounded Contexts
|
||||
|
||||
### 1. Waveform Generation Context
|
||||
|
||||
**Responsibility**: Generating, scheduling, and transmitting deterministic sounding waveforms across all CHCI nodes.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Waveform Generation Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────┐ ┌───────────────┐ ┌──────────────┐ │
|
||||
│ │ NDP Sounding │ │ Micro-Burst │ │ Chirp │ │
|
||||
│ │ Generator │ │ Generator │ │ Generator │ │
|
||||
│ │ (802.11bf) │ │ (Custom OFDM) │ │ (Multi-BW) │ │
|
||||
│ └───────┬───────┘ └───────┬───────┘ └──────┬───────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────────┬───────┴────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Sounding │ │
|
||||
│ │ Scheduler │ ← Cadence, band, power from │
|
||||
│ │ (Aggregate Root) │ Cognitive Engine │
|
||||
│ └────────┬─────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────┴──────────┐ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ TX Chain │ │ TX Chain │ │
|
||||
│ │ (2.4 GHz) │ │ (5 GHz) │ │
|
||||
│ └──────────────┘ └──────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ SoundingFrameTransmitted { band, timestamp, seq_id } │
|
||||
│ BurstSequenceCompleted { burst_count, duration } │
|
||||
│ WaveformConfigChanged { old_mode, new_mode } │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `SoundingScheduler` (Aggregate Root) — Orchestrates sounding frame transmission across nodes and bands according to the current waveform configuration
|
||||
|
||||
**Entities:**
|
||||
- `SoundingFrame` — A single NDP or micro-burst transmission with sequence ID, band, timestamp, and pilot structure
|
||||
- `BurstSequence` — An ordered set of micro-bursts within one observation window, used for coherent Doppler integration
|
||||
- `WaveformConfig` — The current waveform parameter set (cadence, bandwidth, band selection, power level, subcarrier mask)
|
||||
|
||||
**Value Objects:**
|
||||
- `SoundingCadence` — Transmission rate in Hz (1–5000), constrained by regulatory duty cycle limits
|
||||
- `BandSelection` — Set of active bands {2.4 GHz, 5 GHz, 6 GHz} for current mode
|
||||
- `SubcarrierMask` — Bit vector selecting active subcarriers for focused sensing (vital mode uses optimal subset)
|
||||
- `BurstDuration` — Single burst length in microseconds (4–20 μs)
|
||||
- `DutyCycle` — Computed duty cycle percentage, must not exceed regulatory limit (ETSI: 10 ms max burst)
|
||||
|
||||
**Domain Services:**
|
||||
- `RegulatoryComplianceChecker` — Validates that any waveform configuration satisfies FCC Part 15.247 and ETSI EN 300 328 constraints before applying
|
||||
- `BandCoordinator` — Manages time-division or simultaneous multi-band sounding to avoid self-interference
|
||||
|
||||
---
|
||||
|
||||
### 2. Clock Synchronization Context
|
||||
|
||||
**Responsibility**: Distributing and maintaining phase-coherent timing across all CHCI nodes in the sensing mesh.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Clock Synchronization Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────┐ │
|
||||
│ │ Reference │ │
|
||||
│ │ Clock Module │ ← TCXO (40 MHz, ±0.5 ppm) │
|
||||
│ │ (Aggregate │ │
|
||||
│ │ Root) │ │
|
||||
│ └───────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────┴────────┐ │
|
||||
│ │ PLL Synthesizer│ ← SI5351A: generates 40 MHz clock │
|
||||
│ │ │ + 2.4/5 GHz CW phase reference │
|
||||
│ └───────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌─────┼─────────────────┐ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌─────┐ ┌─────┐ ┌─────┐ │
|
||||
│ │Node1│ │Node2│ ... │NodeN│ │
|
||||
│ │Phase│ │Phase│ │Phase│ │
|
||||
│ │Lock │ │Lock │ │Lock │ │
|
||||
│ └──┬──┘ └──┬──┘ └──┬──┘ │
|
||||
│ │ │ │ │
|
||||
│ └───────┼──────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Phase Calibration │ ← Measures residual offset │
|
||||
│ │ Service │ per node at startup │
|
||||
│ └──────────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ ClockLockAcquired { node_id, offset_ppm } │
|
||||
│ PhaseDriftDetected { node_id, drift_deg_per_min } │
|
||||
│ CalibrationCompleted { residual_offsets: Vec<f64> } │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `ReferenceClockModule` (Aggregate Root) — The single source of timing truth for the entire CHCI mesh
|
||||
|
||||
**Entities:**
|
||||
- `NodePhaseLock` — Per-node state tracking lock status, residual offset, and drift rate
|
||||
- `CalibrationSession` — A timed procedure that measures and records per-node phase offsets under static conditions
|
||||
|
||||
**Value Objects:**
|
||||
- `PhaseOffset` — Residual phase offset in degrees after clock distribution, per node per subcarrier
|
||||
- `DriftRate` — Phase drift in degrees per minute, must remain below threshold (0.05°/min for heartbeat sensing)
|
||||
- `LockStatus` — Enum {Acquiring, Locked, Drifting, Lost} indicating current synchronization state
|
||||
|
||||
**Domain Services:**
|
||||
- `PhaseCalibrationService` — Runs startup and periodic calibration routines; replaces statistical LO estimation in current `phase_align.rs`
|
||||
- `DriftMonitor` — Continuous background service that detects when any node exceeds drift threshold and triggers recalibration
|
||||
|
||||
**Invariants:**
|
||||
- All nodes must achieve `Locked` status before CHCI sensing begins
|
||||
- Phase variance per subcarrier must remain ≤ 0.5° RMS over any 10-minute window
|
||||
- If any node transitions to `Lost`, system falls back to statistical phase correction (legacy mode)
|
||||
|
||||
---
|
||||
|
||||
### 3. Coherent Signal Processing Context
|
||||
|
||||
**Responsibility**: Processing raw coherent CSI into body-surface representations using diffraction tomography and multi-band fusion.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ Coherent Signal Processing Context │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────┐ ┌───────────────┐ ┌──────────────────┐ │
|
||||
│ │ Coherent CSI │ │ Reference │ │ Calibration │ │
|
||||
│ │ Stream │ │ Channel │ │ Store │ │
|
||||
│ │ (per node │ │ (empty room) │ │ (per deployment) │ │
|
||||
│ │ per band) │ │ │ │ │ │
|
||||
│ └───────┬───────┘ └───────┬───────┘ └────────┬─────────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────────┬───────┴─────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────────────┐ │
|
||||
│ │ Channel Contrast │ │
|
||||
│ │ Computer │ │
|
||||
│ │ H_c = H_meas / H_ref │ │
|
||||
│ └───────────┬───────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────┴──────────┐ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌──────────────────┐ ┌──────────────────┐ │
|
||||
│ │ Diffraction │ │ Multi-Band │ │
|
||||
│ │ Tomography │ │ Coherent Fusion │ │
|
||||
│ │ Engine │ │ │ │
|
||||
│ │ (Aggregate Root) │ │ Body model priors │ │
|
||||
│ │ │ │ as soft │ │
|
||||
│ │ Complex │ │ constraints │ │
|
||||
│ │ permittivity │ │ │ │
|
||||
│ │ contrast per │ │ Cross-band phase │ │
|
||||
│ │ voxel │ │ alignment │ │
|
||||
│ └────────┬─────────┘ └────────┬─────────┘ │
|
||||
│ │ │ │
|
||||
│ └──────────┬──────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Body Surface │──▶ DensePose UV Mapping │
|
||||
│ │ Reconstruction │ │
|
||||
│ └──────────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ VoxelGridUpdated { grid_dims, resolution_cm, timestamp } │
|
||||
│ BodySurfaceReconstructed { n_vertices, confidence } │
|
||||
│ CoherenceDegradation { node_id, band, severity } │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `DiffractionTomographyEngine` (Aggregate Root) — Reconstructs 3D body surface geometry from coherent channel contrast measurements across all node pairs and frequency bands
|
||||
|
||||
**Entities:**
|
||||
- `CoherentCsiFrame` — A single coherent channel measurement: complex-valued H(f) per subcarrier, with phase-lock metadata, node ID, band, sequence ID, and timestamp
|
||||
- `ReferenceChannel` — The empty-room complex channel response per link per band, used as the denominator in channel contrast computation
|
||||
- `VoxelGrid` — 3D grid of complex permittivity contrast values, the output of diffraction tomography
|
||||
- `BodySurface` — Extracted iso-surface from voxel grid, represented as triangulated mesh or point cloud
|
||||
|
||||
**Value Objects:**
|
||||
- `ChannelContrast` — Complex ratio H_measured/H_reference per subcarrier per link — the fundamental input to tomography
|
||||
- `SubcarrierResponse` — Complex-valued (amplitude + phase) channel response at a single subcarrier frequency
|
||||
- `VoxelCoordinate` — (x, y, z) position in room coordinate frame with associated complex permittivity value
|
||||
- `SurfaceNormal` — Orientation vector at each surface vertex, derived from permittivity gradient
|
||||
- `CoherenceMetric` — Complex-valued coherence score (magnitude + phase) replacing the current real-valued Z-score
|
||||
|
||||
**Domain Services:**
|
||||
- `ChannelContrastComputer` — Divides measured channel by reference to isolate human-induced perturbation
|
||||
- `MultiBandFuser` — Aligns phase across bands using body model priors and combines into unified spectral response
|
||||
- `SurfaceExtractor` — Applies marching cubes or similar iso-surface algorithm to permittivity contrast grid
|
||||
|
||||
**RuVector Integration:**
|
||||
- `ruvector-attention` → Cross-band attention weights for frequency fusion (extends `CrossViewpointAttention`)
|
||||
- `ruvector-solver` → Sparse reconstruction for under-determined tomographic inversions
|
||||
- `ruvector-temporal-tensor` → Temporal coherence of surface reconstructions across frames
|
||||
|
||||
---
|
||||
|
||||
### 4. Cognitive Waveform Context
|
||||
|
||||
**Responsibility**: Adapting the sensing waveform in real-time based on scene state, optimizing the tradeoff between sensing fidelity and power consumption.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Cognitive Waveform Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────────────────────────────────────────────┐ │
|
||||
│ │ Scene State Observer │ │
|
||||
│ │ │ │
|
||||
│ │ Body Model ──▶ ┌──────────────┐ │ │
|
||||
│ │ │ Coherence │ │ │
|
||||
│ │ Coherence ──▶│ Delta │──▶ Mode Transition │ │
|
||||
│ │ Metrics │ Analyzer │ Signal │ │
|
||||
│ │ └──────────────┘ │ │
|
||||
│ │ Motion ──▶ │ │
|
||||
│ │ Classifier │ │
|
||||
│ └───────────────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────────────┐ │
|
||||
│ │ Sensing Mode │ │
|
||||
│ │ State Machine │ │
|
||||
│ │ (Aggregate Root) │ │
|
||||
│ │ │ │
|
||||
│ │ IDLE ──▶ ALERT ──▶ ACTIVE │
|
||||
│ │ ╱ │ ╲ │
|
||||
│ │ VITAL GESTURE SLEEP │
|
||||
│ │ │
|
||||
│ └───────────┬───────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────────────┐ │
|
||||
│ │ Waveform Parameter │ │
|
||||
│ │ Computer │ │
|
||||
│ │ │──▶ WaveformConfig │
|
||||
│ │ Mode → {cadence, │ (to Waveform │
|
||||
│ │ bandwidth, bands, │ Generation Context) │
|
||||
│ │ power, subcarriers} │ │
|
||||
│ └───────────────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ SensingModeChanged { from, to, trigger_reason } │
|
||||
│ PowerBudgetAdjusted { new_budget_mw, mode } │
|
||||
│ SubcarrierSubsetOptimized { selected: Vec<u16>, criterion }│
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `SensingModeStateMachine` (Aggregate Root) — Manages transitions between six sensing modes based on coherence delta, motion classification, and body model state
|
||||
|
||||
**Entities:**
|
||||
- `SensingMode` — One of {IDLE, ALERT, ACTIVE, VITAL, GESTURE, SLEEP} with associated waveform parameter set
|
||||
- `ModeTransition` — A state change event with trigger reason, timestamp, and hysteresis counter
|
||||
- `PowerBudget` — Per-mode power allocation constraining cadence and TX power
|
||||
|
||||
**Value Objects:**
|
||||
- `CoherenceDelta` — Magnitude of coherence change between consecutive observation windows — the primary mode transition trigger
|
||||
- `MotionClassification` — Enum {Static, Breathing, Walking, Gesturing, Falling} derived from micro-Doppler signature
|
||||
- `ModeHysteresis` — Counter preventing rapid mode oscillation: requires N consecutive trigger events before transition (default N=3)
|
||||
- `OptimalSubcarrierSet` — The subset of subcarriers with highest SNR for vital sign extraction, computed from recent channel statistics
|
||||
|
||||
**Domain Services:**
|
||||
- `SceneStateObserver` — Fuses body model output, coherence metrics, and motion classifier into a unified scene state descriptor
|
||||
- `ModeTransitionEvaluator` — Applies hysteresis and priority rules to determine if a mode change should occur
|
||||
- `SubcarrierSelector` — Identifies optimal subcarrier subset for vital mode using Fisher information criterion or SNR ranking
|
||||
- `PowerManager` — Computes TX power and duty cycle to stay within regulatory and battery constraints per mode
|
||||
|
||||
**Invariants:**
|
||||
- IDLE mode must be entered after 30 seconds of no detection (configurable)
|
||||
- Mode transitions must satisfy hysteresis: ≥3 consecutive trigger events
|
||||
- Power budget must never exceed regulatory limit (20 dBm EIRP at 2.4 GHz)
|
||||
- Subcarrier subset in VITAL mode must include ≥16 subcarriers for statistical reliability
|
||||
|
||||
---
|
||||
|
||||
### 5. Displacement Measurement Context
|
||||
|
||||
**Responsibility**: Extracting sub-millimeter physiological displacement (breathing, heartbeat, tremor) from coherent phase time series.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Displacement Measurement Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────┐ │
|
||||
│ │ Phase Time │ ← Coherent CSI phase per subcarrier │
|
||||
│ │ Series Buffer │ per link, at sounding cadence │
|
||||
│ └──────┬───────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Phase-to- │ │
|
||||
│ │ Displacement │ │
|
||||
│ │ Converter │ │
|
||||
│ │ δ = λΔφ / (4π) │ │
|
||||
│ └──────┬────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────┴──────────────────────────┐ │
|
||||
│ │ │ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌──────────────────┐ ┌──────────────────┐ │
|
||||
│ │ Respiratory │ │ Cardiac │ │
|
||||
│ │ Analyzer │ │ Analyzer │ │
|
||||
│ │ (Aggregate Root) │ │ │ │
|
||||
│ │ │ │ Bandpass: │ │
|
||||
│ │ Bandpass: │ │ 0.8–3.0 Hz │ │
|
||||
│ │ 0.1–0.6 Hz │ │ (48–180 BPM) │ │
|
||||
│ │ (6–36 BPM) │ │ │ │
|
||||
│ │ │ │ Harmonic cancel │ │
|
||||
│ │ Amplitude: 4–12mm │ │ (remove respir. │ │
|
||||
│ │ │ │ harmonics) │ │
|
||||
│ └────────┬──────────┘ │ │ │
|
||||
│ │ │ Amplitude: │ │
|
||||
│ │ │ 0.2–0.5 mm │ │
|
||||
│ │ └────────┬─────────┘ │
|
||||
│ │ │ │
|
||||
│ └──────────┬───────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Vital Signs │ │
|
||||
│ │ Fusion │──▶ VitalSignReport │
|
||||
│ │ (multi-link, │ │
|
||||
│ │ multi-band) │ │
|
||||
│ └──────────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ BreathingRateEstimated { bpm, confidence, method } │
|
||||
│ HeartRateEstimated { bpm, confidence, hrv_ms } │
|
||||
│ ApneaEventDetected { duration_s, severity } │
|
||||
│ DisplacementAnomaly { max_displacement_mm, location } │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `RespiratoryAnalyzer` (Aggregate Root) — Extracts breathing rate and pattern from 0.1–0.6 Hz displacement band
|
||||
|
||||
**Entities:**
|
||||
- `PhaseTimeSeries` — Windowed buffer of unwrapped phase values per subcarrier per link, at sounding cadence
|
||||
- `DisplacementTimeSeries` — Converted from phase: δ(t) = λΔφ(t) / (4π), represents physical surface displacement in mm
|
||||
- `VitalSignReport` — Fused output containing breathing rate, heart rate, HRV, confidence scores, and anomaly flags
|
||||
|
||||
**Value Objects:**
|
||||
- `PhaseUnwrapped` — Continuous (unwrapped) phase in radians, free from 2π ambiguity
|
||||
- `DisplacementSample` — Single displacement value in mm with timestamp and confidence
|
||||
- `BreathingRate` — BPM value (6–36 range) with confidence score
|
||||
- `HeartRate` — BPM value (48–180 range) with confidence score and HRV interval
|
||||
- `ApneaEvent` — Duration, severity, and confidence of detected breathing cessation
|
||||
|
||||
**Domain Services:**
|
||||
- `PhaseUnwrapper` — Continuous phase unwrapping with outlier rejection; critical for displacement conversion
|
||||
- `RespiratoryHarmonicCanceller` — Removes breathing harmonics from cardiac band to isolate heartbeat signal
|
||||
- `MultilinkFuser` — Combines displacement estimates across node pairs using SNR-weighted averaging
|
||||
- `AnomalyDetector` — Flags displacement patterns inconsistent with normal physiology (fall, seizure, cardiac arrest)
|
||||
|
||||
**Invariants:**
|
||||
- Phase unwrapping must maintain continuity: |Δφ| < π between consecutive samples
|
||||
- Displacement floor must be validated against acceptance metric (AT-2: ≤ 0.1 mm at 2 m)
|
||||
- Heart rate estimation requires minimum 10 seconds of stable data (cardiac analyzer warmup)
|
||||
- Multi-link fusion must use ≥2 independent links for confidence scoring
|
||||
|
||||
---
|
||||
|
||||
### 6. Regulatory Compliance Context
|
||||
|
||||
**Responsibility**: Ensuring all CHCI transmissions comply with applicable ISM band regulations across deployment jurisdictions.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Regulatory Compliance Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────┐ ┌───────────────┐ ┌──────────────┐ │
|
||||
│ │ FCC Part 15 │ │ ETSI EN │ │ 802.11bf │ │
|
||||
│ │ Rules │ │ 300 328 │ │ Compliance │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ - 30 dBm max │ │ - 20 dBm EIRP│ │ - NDP format │ │
|
||||
│ │ - Digital mod │ │ - LBT or 10ms │ │ - SAW window │ │
|
||||
│ │ - Spread │ │ burst max │ │ - SMS setup │ │
|
||||
│ │ spectrum │ │ - Duty cycle │ │ │ │
|
||||
│ └───────┬───────┘ └───────┬───────┘ └──────┬───────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────────┬───────┴────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Compliance │ │
|
||||
│ │ Validator │ │
|
||||
│ │ (Aggregate Root) │ │
|
||||
│ │ │ │
|
||||
│ │ Validates every │ │
|
||||
│ │ WaveformConfig │ │
|
||||
│ │ before TX │ │
|
||||
│ └────────┬─────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────┐ │
|
||||
│ │ Jurisdiction │ │
|
||||
│ │ Registry │ │
|
||||
│ │ │ │
|
||||
│ │ US → FCC │ │
|
||||
│ │ EU → ETSI │ │
|
||||
│ │ JP → ARIB │ │
|
||||
│ │ ... │ │
|
||||
│ └──────────────────┘ │
|
||||
│ │
|
||||
│ Events emitted: │
|
||||
│ ComplianceCheckPassed { jurisdiction, config_hash } │
|
||||
│ ComplianceViolation { rule, parameter, value, limit } │
|
||||
│ JurisdictionChanged { from, to } │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `ComplianceValidator` (Aggregate Root) — Gate that must approve every waveform configuration before transmission is permitted
|
||||
|
||||
**Entities:**
|
||||
- `JurisdictionProfile` — Complete set of regulatory constraints for a given region (FCC, ETSI, ARIB, etc.)
|
||||
- `ComplianceRecord` — Audit trail of compliance checks with timestamps and configuration hashes
|
||||
|
||||
**Value Objects:**
|
||||
- `MaxEIRP` — Maximum effective isotropic radiated power in dBm, per band per jurisdiction
|
||||
- `MaxBurstDuration` — Maximum continuous transmission time (ETSI: 10 ms)
|
||||
- `MinIdleTime` — Minimum idle period between bursts
|
||||
- `ModulationType` — Must be digital modulation (OFDM qualifies) or spread spectrum for FCC
|
||||
- `DutyCycleLimit` — Maximum percentage of time occupied by transmissions
|
||||
|
||||
**Invariants:**
|
||||
- No transmission shall occur without a passing `ComplianceCheckPassed` event
|
||||
- Duty cycle must be recalculated and validated on every cadence change
|
||||
- Jurisdiction must be set during deployment configuration; default is most restrictive (ETSI)
|
||||
|
||||
---
|
||||
|
||||
## Core Domain Entities
|
||||
|
||||
### CoherentCsiFrame (Entity)
|
||||
|
||||
```rust
|
||||
pub struct CoherentCsiFrame {
|
||||
/// Unique sequence identifier for this sounding frame
|
||||
seq_id: u64,
|
||||
/// Node that received this frame
|
||||
rx_node_id: NodeId,
|
||||
/// Node that transmitted this frame (known from sounding schedule)
|
||||
tx_node_id: NodeId,
|
||||
/// Frequency band: Band2_4GHz, Band5GHz, Band6GHz
|
||||
band: FrequencyBand,
|
||||
/// UTC timestamp with microsecond precision
|
||||
timestamp_us: u64,
|
||||
/// Complex channel response per subcarrier: (amplitude, phase) pairs
|
||||
subcarrier_responses: Vec<Complex64>,
|
||||
/// Phase lock status at time of capture
|
||||
phase_lock: LockStatus,
|
||||
/// Residual phase offset from calibration (degrees)
|
||||
residual_offset_deg: f64,
|
||||
/// Signal-to-noise ratio estimate (dB)
|
||||
snr_db: f32,
|
||||
/// Sounding mode that produced this frame
|
||||
source_mode: SoundingMode,
|
||||
}
|
||||
```
|
||||
|
||||
**Invariants:**
|
||||
- `phase_lock` must be `Locked` for frame to be used in coherent processing
|
||||
- `subcarrier_responses.len()` must match expected count for `band` and bandwidth (56 for 20 MHz)
|
||||
- `snr_db` must be ≥ 10 dB for frame to contribute to displacement estimation
|
||||
- `timestamp_us` must be monotonically increasing per `rx_node_id`
|
||||
|
||||
### WaveformConfig (Value Object)
|
||||
|
||||
```rust
|
||||
pub struct WaveformConfig {
|
||||
/// Active sensing mode
|
||||
mode: SensingMode,
|
||||
/// Sounding cadence in Hz
|
||||
cadence_hz: f64,
|
||||
/// Active frequency bands
|
||||
bands: BandSet,
|
||||
/// Bandwidth per band
|
||||
bandwidth_mhz: u8,
|
||||
/// Transmit power in dBm
|
||||
tx_power_dbm: f32,
|
||||
/// Subcarrier mask (None = all subcarriers active)
|
||||
subcarrier_mask: Option<BitVec>,
|
||||
/// Burst duration in microseconds
|
||||
burst_duration_us: u16,
|
||||
/// Number of symbols per burst
|
||||
symbols_per_burst: u8,
|
||||
/// Computed duty cycle (must pass compliance check)
|
||||
duty_cycle_pct: f64,
|
||||
}
|
||||
```
|
||||
|
||||
**Invariants:**
|
||||
- `cadence_hz` must be ≥ 1.0 and ≤ 5000.0
|
||||
- `duty_cycle_pct` must not exceed jurisdiction limit (ETSI: derived from 10 ms burst max)
|
||||
- `tx_power_dbm` must not exceed jurisdiction max EIRP
|
||||
- `bandwidth_mhz` must be one of {20, 40, 80}
|
||||
- `burst_duration_us` must be ≥ 4 (single OFDM symbol + CP)
|
||||
|
||||
### SensingMode (Value Object)
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SensingMode {
|
||||
/// 1 Hz, single band, presence detection only
|
||||
Idle,
|
||||
/// 10 Hz, dual band, coarse tracking
|
||||
Alert,
|
||||
/// 50-200 Hz, all bands, full DensePose + vitals
|
||||
Active,
|
||||
/// 100 Hz, optimal subcarrier subset, breathing + HR + HRV
|
||||
Vital,
|
||||
/// 200 Hz, full band, DTW gesture classification
|
||||
Gesture,
|
||||
/// 20 Hz, single band, apnea/movement/stage detection
|
||||
Sleep,
|
||||
}
|
||||
|
||||
impl SensingMode {
|
||||
pub fn default_config(&self) -> WaveformConfig {
|
||||
match self {
|
||||
Self::Idle => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 1.0,
|
||||
bands: BandSet::single(Band::Band2_4GHz),
|
||||
bandwidth_mhz: 20,
|
||||
tx_power_dbm: 10.0,
|
||||
subcarrier_mask: None,
|
||||
burst_duration_us: 4,
|
||||
symbols_per_burst: 1,
|
||||
duty_cycle_pct: 0.0004,
|
||||
},
|
||||
Self::Alert => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 10.0,
|
||||
bands: BandSet::dual(Band::Band2_4GHz, Band::Band5GHz),
|
||||
bandwidth_mhz: 20,
|
||||
tx_power_dbm: 15.0,
|
||||
subcarrier_mask: None,
|
||||
burst_duration_us: 8,
|
||||
symbols_per_burst: 2,
|
||||
duty_cycle_pct: 0.008,
|
||||
},
|
||||
Self::Active => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 100.0,
|
||||
bands: BandSet::all(),
|
||||
bandwidth_mhz: 40,
|
||||
tx_power_dbm: 20.0,
|
||||
subcarrier_mask: None,
|
||||
burst_duration_us: 16,
|
||||
symbols_per_burst: 4,
|
||||
duty_cycle_pct: 0.16,
|
||||
},
|
||||
Self::Vital => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 100.0,
|
||||
bands: BandSet::dual(Band::Band2_4GHz, Band::Band5GHz),
|
||||
bandwidth_mhz: 20,
|
||||
tx_power_dbm: 18.0,
|
||||
subcarrier_mask: Some(optimal_vital_subcarriers()),
|
||||
burst_duration_us: 8,
|
||||
symbols_per_burst: 2,
|
||||
duty_cycle_pct: 0.08,
|
||||
},
|
||||
Self::Gesture => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 200.0,
|
||||
bands: BandSet::all(),
|
||||
bandwidth_mhz: 40,
|
||||
tx_power_dbm: 20.0,
|
||||
subcarrier_mask: None,
|
||||
burst_duration_us: 16,
|
||||
symbols_per_burst: 4,
|
||||
duty_cycle_pct: 0.32,
|
||||
},
|
||||
Self::Sleep => WaveformConfig {
|
||||
mode: *self,
|
||||
cadence_hz: 20.0,
|
||||
bands: BandSet::single(Band::Band2_4GHz),
|
||||
bandwidth_mhz: 20,
|
||||
tx_power_dbm: 12.0,
|
||||
subcarrier_mask: None,
|
||||
burst_duration_us: 4,
|
||||
symbols_per_burst: 1,
|
||||
duty_cycle_pct: 0.008,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### VitalSignReport (Value Object)
|
||||
|
||||
```rust
|
||||
pub struct VitalSignReport {
|
||||
/// Timestamp of this report
|
||||
timestamp_us: u64,
|
||||
/// Breathing rate in BPM (None if not measurable)
|
||||
breathing_bpm: Option<f64>,
|
||||
/// Breathing confidence [0.0, 1.0]
|
||||
breathing_confidence: f64,
|
||||
/// Heart rate in BPM (None if not measurable — requires CHCI coherent mode)
|
||||
heart_rate_bpm: Option<f64>,
|
||||
/// Heart rate confidence [0.0, 1.0]
|
||||
heart_rate_confidence: f64,
|
||||
/// Heart rate variability: RMSSD in milliseconds
|
||||
hrv_rmssd_ms: Option<f64>,
|
||||
/// Detected anomalies
|
||||
anomalies: Vec<VitalAnomaly>,
|
||||
/// Number of independent links contributing to this estimate
|
||||
contributing_links: u16,
|
||||
/// Sensing mode that produced this report
|
||||
source_mode: SensingMode,
|
||||
}
|
||||
|
||||
pub enum VitalAnomaly {
|
||||
Apnea { duration_s: f64, severity: Severity },
|
||||
Tachycardia { bpm: f64 },
|
||||
Bradycardia { bpm: f64 },
|
||||
IrregularRhythm { irregularity_score: f64 },
|
||||
FallDetected { impact_g: f64 },
|
||||
NoMotion { duration_s: f64 },
|
||||
}
|
||||
```
|
||||
|
||||
### NodeId and FrequencyBand (Value Objects)
|
||||
|
||||
```rust
|
||||
/// Unique identifier for a CHCI node in the sensing mesh
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct NodeId(pub u8);
|
||||
|
||||
/// Operating frequency band
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FrequencyBand {
|
||||
/// 2.4 GHz ISM band (2400-2483.5 MHz), λ = 12.5 cm
|
||||
Band2_4GHz,
|
||||
/// 5 GHz UNII band (5150-5850 MHz), λ = 6.0 cm
|
||||
Band5GHz,
|
||||
/// 6 GHz band (5925-7125 MHz), λ = 5.0 cm, WiFi 6E
|
||||
Band6GHz,
|
||||
}
|
||||
|
||||
impl FrequencyBand {
|
||||
pub fn wavelength_m(&self) -> f64 {
|
||||
match self {
|
||||
Self::Band2_4GHz => 0.125,
|
||||
Self::Band5GHz => 0.060,
|
||||
Self::Band6GHz => 0.050,
|
||||
}
|
||||
}
|
||||
|
||||
/// Displacement per radian of phase change: λ/(4π)
|
||||
pub fn displacement_per_radian_mm(&self) -> f64 {
|
||||
self.wavelength_m() * 1000.0 / (4.0 * std::f64::consts::PI)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Domain Events
|
||||
|
||||
### Waveform Events
|
||||
|
||||
```rust
|
||||
pub enum WaveformEvent {
|
||||
/// A sounding frame was transmitted
|
||||
SoundingFrameTransmitted {
|
||||
seq_id: u64,
|
||||
tx_node: NodeId,
|
||||
band: FrequencyBand,
|
||||
timestamp_us: u64,
|
||||
},
|
||||
/// A burst sequence completed (micro-burst mode)
|
||||
BurstSequenceCompleted {
|
||||
burst_count: u32,
|
||||
total_duration_us: u64,
|
||||
},
|
||||
/// Waveform configuration changed (mode transition)
|
||||
WaveformConfigChanged {
|
||||
old_mode: SensingMode,
|
||||
new_mode: SensingMode,
|
||||
trigger: ModeTransitionTrigger,
|
||||
},
|
||||
}
|
||||
|
||||
pub enum ModeTransitionTrigger {
|
||||
CoherenceDeltaThreshold { delta: f64 },
|
||||
PersonDetected { confidence: f64 },
|
||||
PersonLost { absence_duration_s: f64 },
|
||||
PoseClassification { pose: PoseClass },
|
||||
MotionSpike { magnitude: f64 },
|
||||
Manual,
|
||||
}
|
||||
```
|
||||
|
||||
### Clock Events
|
||||
|
||||
```rust
|
||||
pub enum ClockEvent {
|
||||
/// A node achieved phase lock
|
||||
ClockLockAcquired {
|
||||
node_id: NodeId,
|
||||
residual_offset_deg: f64,
|
||||
},
|
||||
/// Phase drift detected on a node
|
||||
PhaseDriftDetected {
|
||||
node_id: NodeId,
|
||||
drift_deg_per_min: f64,
|
||||
},
|
||||
/// Phase lock lost on a node — triggers fallback to statistical correction
|
||||
ClockLockLost {
|
||||
node_id: NodeId,
|
||||
reason: LockLossReason,
|
||||
},
|
||||
/// Calibration procedure completed
|
||||
CalibrationCompleted {
|
||||
residual_offsets: Vec<(NodeId, f64)>,
|
||||
max_residual_deg: f64,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Measurement Events
|
||||
|
||||
```rust
|
||||
pub enum MeasurementEvent {
|
||||
/// Body surface reconstructed from diffraction tomography
|
||||
BodySurfaceReconstructed {
|
||||
n_vertices: u32,
|
||||
resolution_cm: f64,
|
||||
confidence: f64,
|
||||
timestamp_us: u64,
|
||||
},
|
||||
/// Vital signs estimated
|
||||
VitalSignsUpdated {
|
||||
report: VitalSignReport,
|
||||
},
|
||||
/// Displacement anomaly detected
|
||||
DisplacementAnomaly {
|
||||
max_displacement_mm: f64,
|
||||
anomaly_type: VitalAnomaly,
|
||||
},
|
||||
/// Coherence degradation on a link (may trigger recalibration)
|
||||
CoherenceDegradation {
|
||||
tx_node: NodeId,
|
||||
rx_node: NodeId,
|
||||
band: FrequencyBand,
|
||||
severity: Severity,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Context Map
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ CHCI Context Map │
|
||||
│ │
|
||||
│ ┌────────────────┐ ┌────────────────┐ │
|
||||
│ │ Waveform │ ◀───── │ Cognitive │ │
|
||||
│ │ Generation │ config │ Waveform │ │
|
||||
│ │ Context │ │ Context │ │
|
||||
│ └───────┬────────┘ └───────▲────────┘ │
|
||||
│ │ │ │
|
||||
│ │ sounding │ scene state │
|
||||
│ │ frames │ feedback │
|
||||
│ ▼ │ │
|
||||
│ ┌────────────────┐ ┌───────┴────────┐ │
|
||||
│ │ Clock │ phase │ Coherent │ │
|
||||
│ │ Synchro- │ lock ──▶│ Signal │ │
|
||||
│ │ nization │ status │ Processing │ │
|
||||
│ │ Context │ │ Context │ │
|
||||
│ └────────────────┘ └───────┬────────┘ │
|
||||
│ │ │
|
||||
│ body surface, │
|
||||
│ coherence metrics │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌────────────────┐ │
|
||||
│ │ Displacement │ │
|
||||
│ │ Measurement │ │
|
||||
│ │ Context │ │
|
||||
│ └────────────────┘ │
|
||||
│ │
|
||||
│ ┌────────────────┐ │
|
||||
│ │ Regulatory │ ◀── validates all WaveformConfig before TX │
|
||||
│ │ Compliance │ │
|
||||
│ │ Context │ │
|
||||
│ └────────────────┘ │
|
||||
│ │
|
||||
│ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │
|
||||
│ Integration with existing WiFi-DensePose bounded contexts: │
|
||||
│ │
|
||||
│ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ │
|
||||
│ │ RuvSense │ │ RuVector │ │ DensePose │ │
|
||||
│ │ Multistatic │ │ Cross-View │ │ Body Model │ │
|
||||
│ │ (ADR-029) │ │ Fusion │ │ (Core) │ │
|
||||
│ └────────────────┘ └────────────────┘ └────────────────┘ │
|
||||
│ │
|
||||
│ CHCI Signal Processing feeds directly into existing │
|
||||
│ RuvSense/RuVector/DensePose pipeline — coherent CSI │
|
||||
│ replaces incoherent CSI as input, same output interface │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Anti-Corruption Layers
|
||||
|
||||
| Boundary | Direction | Mechanism |
|
||||
|----------|-----------|-----------|
|
||||
| CHCI Signal Processing → RuvSense | Downstream | `CoherentCsiFrame` adapts to existing `CsiFrame` trait via `IntoLegacyCsi` adapter — existing pipeline works unmodified |
|
||||
| Cognitive Waveform → ADR-039 Edge Tiers | Bidirectional | Sensing modes map to edge tiers: IDLE→Tier0, ACTIVE→Tier1, VITAL→Tier2. Shared `EdgeConfig` value object |
|
||||
| Clock Synchronization → Hardware | Downstream | `ClockDriver` trait abstracts SI5351A hardware specifics; mock implementation for testing |
|
||||
| Regulatory Compliance → All TX Contexts | Upstream | Compliance Validator acts as a policy gateway — no transmission without passing check |
|
||||
|
||||
---
|
||||
|
||||
## Integration with Existing Codebase
|
||||
|
||||
### Modified Modules
|
||||
|
||||
| File | Current | CHCI Change |
|
||||
|------|---------|-------------|
|
||||
| `signal/src/ruvsense/phase_align.rs` | Statistical LO offset estimation via circular mean | Add `SharedClockAligner` path: when `phase_lock == Locked`, skip statistical estimation, apply only residual calibration offset |
|
||||
| `signal/src/ruvsense/multiband.rs` | Independent per-channel fusion | Add `CoherentCrossBandFuser`: phase-aligns across bands using body model priors before fusion |
|
||||
| `signal/src/ruvsense/coherence.rs` | Z-score coherence scoring (real-valued) | Add `ComplexCoherenceMetric`: phasor-domain coherence using both magnitude and phase information |
|
||||
| `signal/src/ruvsense/tomography.rs` | Amplitude-only ISTA L1 solver | Add `DiffractionTomographyEngine`: complex-valued reconstruction using channel contrast |
|
||||
| `signal/src/ruvsense/coherence_gate.rs` | Accept/Reject gate decisions | Add cognitive waveform feedback: gate decisions emit `CoherenceDelta` events to mode state machine |
|
||||
| `signal/src/ruvsense/multistatic.rs` | Attention-weighted fusion | Add clock synchronization status as fusion weight modifier |
|
||||
| `hardware/src/esp32/` | TDM protocol, channel hopping | Add NDP sounding mode, reference clock driver, phase reference input |
|
||||
| `ruvector/src/viewpoint/attention.rs` | CrossViewpointAttention | Extend to cross-band attention with frequency-dependent geometric bias |
|
||||
|
||||
### New Crate: `wifi-densepose-chci`
|
||||
|
||||
```
|
||||
wifi-densepose-chci/
|
||||
├── src/
|
||||
│ ├── lib.rs # Crate root, re-exports
|
||||
│ ├── waveform/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── ndp_generator.rs # 802.11bf NDP sounding frame generation
|
||||
│ │ ├── burst_generator.rs # Micro-burst OFDM symbol generation
|
||||
│ │ ├── scheduler.rs # Sounding schedule orchestration
|
||||
│ │ └── compliance.rs # Regulatory compliance validation
|
||||
│ ├── clock/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── reference.rs # Reference clock module abstraction
|
||||
│ │ ├── pll_driver.rs # SI5351A PLL synthesizer driver
|
||||
│ │ ├── calibration.rs # Phase calibration procedures
|
||||
│ │ └── drift_monitor.rs # Continuous drift detection
|
||||
│ ├── cognitive/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── mode.rs # SensingMode enum and transitions
|
||||
│ │ ├── state_machine.rs # Mode state machine with hysteresis
|
||||
│ │ ├── scene_observer.rs # Scene state fusion from body model + coherence
|
||||
│ │ ├── subcarrier_select.rs # Optimal subcarrier subset for vital mode
|
||||
│ │ └── power_manager.rs # Power budget per mode
|
||||
│ ├── tomography/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── contrast.rs # Channel contrast computation
|
||||
│ │ ├── diffraction.rs # Coherent diffraction tomography engine
|
||||
│ │ └── surface.rs # Iso-surface extraction (marching cubes)
|
||||
│ ├── displacement/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── phase_to_disp.rs # Phase-to-displacement conversion
|
||||
│ │ ├── respiratory.rs # Breathing rate analyzer
|
||||
│ │ ├── cardiac.rs # Heart rate + HRV analyzer
|
||||
│ │ └── anomaly.rs # Vital sign anomaly detection
|
||||
│ └── types.rs # Shared types (NodeId, FrequencyBand, etc.)
|
||||
├── Cargo.toml
|
||||
└── tests/
|
||||
├── integration/
|
||||
│ ├── acceptance_tests.rs # AT-1 through AT-8
|
||||
│ └── mode_transitions.rs # Cognitive state machine tests
|
||||
└── unit/
|
||||
├── compliance_tests.rs
|
||||
├── displacement_tests.rs
|
||||
└── tomography_tests.rs
|
||||
```
|
||||
@@ -0,0 +1,648 @@
|
||||
# Deployment Platform Domain Model
|
||||
|
||||
The Deployment Platform domain covers everything from cross-compiling the sensing server for ARM targets to managing TV box appliances running Armbian: provisioning devices, deploying binaries, configuring kiosk displays, and coordinating multi-room installations. It bridges the gap between the Sensing Server domain (which produces the binary) and the physical hardware it runs on.
|
||||
|
||||
This document defines the system using [Domain-Driven Design](https://martinfowler.com/bliki/DomainDrivenDesign.html) (DDD): bounded contexts that own their data and rules, aggregate roots that enforce invariants, value objects that carry meaning, and domain events that connect everything.
|
||||
|
||||
**Bounded Contexts:**
|
||||
|
||||
| # | Context | Responsibility | Key ADRs | Code |
|
||||
|---|---------|----------------|----------|------|
|
||||
| 1 | [Appliance Management](#1-appliance-management-context) | Device inventory, provisioning, health monitoring, OTA updates for TV box deployments | [ADR-046](../adr/ADR-046-android-tv-box-armbian-deployment.md) | `scripts/deploy/`, `config/armbian/` |
|
||||
| 2 | [Cross-Compilation](#2-cross-compilation-context) | Build pipeline for aarch64, binary packaging, CI/CD release artifacts | [ADR-046](../adr/ADR-046-android-tv-box-armbian-deployment.md) | `.github/workflows/`, `Cross.toml` |
|
||||
| 3 | [Display Kiosk](#3-display-kiosk-context) | HDMI output management, Chromium kiosk mode, screen rotation, auto-start | [ADR-046](../adr/ADR-046-android-tv-box-armbian-deployment.md) | `config/armbian/kiosk/` |
|
||||
| 4 | [WiFi CSI Bridge](#4-wifi-csi-bridge-context) | Custom WiFi driver CSI extraction, protocol translation to ESP32 binary format | [ADR-046](../adr/ADR-046-android-tv-box-armbian-deployment.md) | `tools/csi-bridge/` |
|
||||
| 5 | [Network Topology](#5-network-topology-context) | ESP32 mesh ↔ TV box connectivity, dedicated AP mode, multi-room routing | [ADR-046](../adr/ADR-046-android-tv-box-armbian-deployment.md), [ADR-012](../adr/ADR-012-esp32-csi-sensor-mesh.md) | `config/armbian/network/` |
|
||||
|
||||
---
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
### Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **Appliance** | A TV box running Armbian with the sensing server deployed, treated as a managed device in the fleet |
|
||||
| **Fleet** | The set of all appliances across a multi-room or multi-site installation |
|
||||
| **Deployment Package** | A self-contained archive containing the sensing-server binary, systemd unit, configuration, and setup script for a target architecture |
|
||||
| **Kiosk Mode** | Chromium running in full-screen, no-UI mode pointing at `localhost:3000`, auto-started by systemd on HDMI-connected appliances |
|
||||
| **CSI Bridge** | A userspace daemon that reads CSI data from a patched WiFi driver and re-encodes it as ESP32-compatible UDP frames for the sensing server |
|
||||
| **Dedicated AP** | An optional `hostapd`-managed WiFi access point on the TV box that creates an isolated network for ESP32 nodes |
|
||||
| **OTA Update** | Over-the-air binary replacement: download new sensing-server binary, validate checksum, swap via atomic rename, restart service |
|
||||
| **Reference Device** | A TV box model that has been tested and validated for Armbian + sensing-server deployment (e.g., T95 Max+ / S905X3) |
|
||||
| **Provisioning** | First-time setup of an appliance: flash Armbian to SD, deploy package, configure WiFi, start services |
|
||||
| **Health Beacon** | Periodic JSON payload sent by each appliance to a central coordinator (if multi-room) containing uptime, CPU temp, memory usage, inference latency, connected ESP32 count |
|
||||
|
||||
---
|
||||
|
||||
## Bounded Contexts
|
||||
|
||||
### 1. Appliance Management Context
|
||||
|
||||
**Responsibility:** Track deployed TV box appliances, provision new devices, monitor health, and coordinate OTA updates across the fleet.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Appliance Management Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Device | | Provisioning | |
|
||||
| | Registry | | Service | |
|
||||
| | (fleet state) | | (first-time | |
|
||||
| | | | setup) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Health Monitor | |
|
||||
| | (beacon receiver,| |
|
||||
| | thermal alerts, | |
|
||||
| | connectivity) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | OTA Updater | |
|
||||
| | (binary swap, | |
|
||||
| | rollback, | |
|
||||
| | checksum verify)| |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
|
||||
```rust
|
||||
/// Aggregate Root: A managed TV box appliance in the fleet.
|
||||
/// Identified by MAC address of the primary Ethernet interface.
|
||||
pub struct Appliance {
|
||||
/// Unique device identifier (Ethernet MAC address).
|
||||
pub device_id: DeviceId,
|
||||
/// Human-readable name (e.g., "living-room", "bedroom-1").
|
||||
pub name: String,
|
||||
/// Hardware model (e.g., "T95 Max+ S905X3").
|
||||
pub hardware_model: HardwareModel,
|
||||
/// Current deployment state.
|
||||
pub state: ApplianceState,
|
||||
/// Installed sensing-server version.
|
||||
pub server_version: SemanticVersion,
|
||||
/// Network configuration.
|
||||
pub network: NetworkConfig,
|
||||
/// Last received health beacon.
|
||||
pub last_health: Option<HealthBeacon>,
|
||||
/// Provisioning timestamp.
|
||||
pub provisioned_at: DateTime<Utc>,
|
||||
/// Connected ESP32 node IDs (from last beacon).
|
||||
pub connected_nodes: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Lifecycle states for an appliance.
|
||||
pub enum ApplianceState {
|
||||
/// SD card prepared, not yet booted.
|
||||
Provisioned,
|
||||
/// Booted and running, health beacons received.
|
||||
Online,
|
||||
/// No health beacon for >5 minutes.
|
||||
Unreachable,
|
||||
/// OTA update in progress.
|
||||
Updating,
|
||||
/// Manual maintenance / stopped.
|
||||
Offline,
|
||||
/// Thermal throttling or hardware issue detected.
|
||||
Degraded,
|
||||
}
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Hardware model specification for a TV box.
|
||||
pub struct HardwareModel {
|
||||
/// Marketing name (e.g., "T95 Max+").
|
||||
pub name: String,
|
||||
/// SoC identifier (e.g., "Amlogic S905X3").
|
||||
pub soc: String,
|
||||
/// WiFi chipset (e.g., "RTL8822CS").
|
||||
pub wifi_chipset: String,
|
||||
/// Total RAM in MB.
|
||||
pub ram_mb: u32,
|
||||
/// eMMC storage in GB.
|
||||
pub emmc_gb: u32,
|
||||
/// Whether CSI bridge is supported for this WiFi chipset.
|
||||
pub csi_bridge_supported: bool,
|
||||
/// Armbian device tree name (e.g., "meson-sm1-sei610").
|
||||
pub armbian_dtb: String,
|
||||
}
|
||||
|
||||
/// Periodic health report from an appliance.
|
||||
pub struct HealthBeacon {
|
||||
pub device_id: DeviceId,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub uptime_secs: u64,
|
||||
pub cpu_temp_celsius: f32,
|
||||
pub cpu_usage_percent: f32,
|
||||
pub memory_used_mb: u32,
|
||||
pub memory_total_mb: u32,
|
||||
pub disk_used_percent: f32,
|
||||
pub inference_latency_ms: f32,
|
||||
pub connected_esp32_nodes: Vec<u8>,
|
||||
pub server_version: SemanticVersion,
|
||||
pub csi_frames_per_sec: f32,
|
||||
pub websocket_clients: u32,
|
||||
}
|
||||
|
||||
/// Network configuration for an appliance.
|
||||
pub struct NetworkConfig {
|
||||
/// Primary IP address (Ethernet or WiFi client).
|
||||
pub ip_address: IpAddr,
|
||||
/// Whether the appliance runs a dedicated AP for ESP32 nodes.
|
||||
pub dedicated_ap: Option<DedicatedApConfig>,
|
||||
/// UDP port for ESP32 CSI reception.
|
||||
pub csi_udp_port: u16, // default: 5005
|
||||
/// HTTP port for sensing server.
|
||||
pub http_port: u16, // default: 3000
|
||||
}
|
||||
|
||||
/// Configuration for a dedicated WiFi AP hosted by the appliance.
|
||||
pub struct DedicatedApConfig {
|
||||
/// SSID for the ESP32 mesh network.
|
||||
pub ssid: String,
|
||||
/// WPA2 passphrase.
|
||||
pub passphrase: String,
|
||||
/// Channel (1-11 for 2.4 GHz).
|
||||
pub channel: u8,
|
||||
/// DHCP range for connected ESP32 nodes.
|
||||
pub dhcp_range: (IpAddr, IpAddr),
|
||||
}
|
||||
|
||||
/// Unique device identifier (Ethernet MAC).
|
||||
pub struct DeviceId(pub [u8; 6]);
|
||||
|
||||
/// Semantic version for tracking installed software.
|
||||
pub struct SemanticVersion {
|
||||
pub major: u16,
|
||||
pub minor: u16,
|
||||
pub patch: u16,
|
||||
pub pre: Option<String>,
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `ProvisioningService` — Generates Armbian SD card image with pre-configured deployment package, WiFi credentials, and systemd units
|
||||
- `HealthMonitorService` — Listens for UDP health beacons from fleet appliances, triggers alerts on thermal throttling (>80°C), unreachable (>5 min), or high memory usage (>90%)
|
||||
- `OtaUpdateService` — Downloads new binary from release URL, verifies SHA-256 checksum, performs atomic swap (`rename(new, current)`), restarts systemd service, rolls back if health beacon fails within 60s
|
||||
|
||||
**Invariants:**
|
||||
- Device ID (MAC address) is immutable after provisioning
|
||||
- OTA update refuses to proceed if current CPU temperature >75°C (thermal headroom)
|
||||
- Rollback is automatic if no healthy beacon is received within 60 seconds of restart
|
||||
- Dedicated AP SSID must not match the upstream WiFi SSID
|
||||
|
||||
---
|
||||
|
||||
### 2. Cross-Compilation Context
|
||||
|
||||
**Responsibility:** Build the sensing-server binary for ARM64 targets, package deployment archives, and manage CI/CD release artifacts.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Cross-Compilation Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Cross.toml | | GitHub Actions| |
|
||||
| | (target cfg) | | CI Matrix | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Build Pipeline | |
|
||||
| | (cross build | |
|
||||
| | --target | |
|
||||
| | aarch64-unknown-| |
|
||||
| | linux-gnu) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Binary Packager | |
|
||||
| | (strip, compress,|---> .tar.gz artifact |
|
||||
| | bundle assets, | |
|
||||
| | systemd units) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// A packaged deployment archive for a target platform.
|
||||
pub struct DeploymentPackage {
|
||||
/// Target triple (e.g., "aarch64-unknown-linux-gnu").
|
||||
pub target: String,
|
||||
/// Sensing server binary (stripped).
|
||||
pub binary: PathBuf,
|
||||
/// Binary size in bytes.
|
||||
pub binary_size: u64,
|
||||
/// SHA-256 checksum of the binary.
|
||||
pub checksum: String,
|
||||
/// Systemd service unit file.
|
||||
pub service_unit: String,
|
||||
/// Static web UI assets directory.
|
||||
pub ui_assets: PathBuf,
|
||||
/// Armbian configuration files (kiosk, network, etc.).
|
||||
pub config_files: Vec<PathBuf>,
|
||||
/// Setup script (runs on first boot).
|
||||
pub setup_script: PathBuf,
|
||||
/// Version being packaged.
|
||||
pub version: SemanticVersion,
|
||||
}
|
||||
|
||||
/// Build target specification.
|
||||
pub struct BuildTarget {
|
||||
/// Rust target triple.
|
||||
pub triple: String,
|
||||
/// CPU architecture description.
|
||||
pub arch: String,
|
||||
/// Whether NEON SIMD is available.
|
||||
pub has_neon: bool,
|
||||
/// Cross-compilation Docker image.
|
||||
pub cross_image: String,
|
||||
/// Binary size limit in bytes.
|
||||
pub size_limit: u64,
|
||||
}
|
||||
```
|
||||
|
||||
**Supported Targets:**
|
||||
|
||||
| Target Triple | Architecture | Use Case | Size Limit |
|
||||
|---------------|-------------|----------|------------|
|
||||
| `x86_64-unknown-linux-gnu` | x86-64 | PC/laptop (existing) | 30 MB |
|
||||
| `aarch64-unknown-linux-gnu` | ARM64 | TV box (Armbian) | 15 MB |
|
||||
| `armv7-unknown-linux-gnueabihf` | ARMv7 | Older TV boxes (32-bit) | 12 MB |
|
||||
| `x86_64-pc-windows-msvc` | x86-64 | Windows (existing) | 30 MB |
|
||||
|
||||
**Invariants:**
|
||||
- Stripped binary must be under size limit for target
|
||||
- SHA-256 checksum is computed and included in every deployment package
|
||||
- UI assets are embedded in binary via `include_dir!` or bundled alongside
|
||||
- No native GPU dependencies — CPU-only inference (candle or ONNX Runtime)
|
||||
|
||||
---
|
||||
|
||||
### 3. Display Kiosk Context
|
||||
|
||||
**Responsibility:** Manage HDMI output on TV box appliances, running Chromium in kiosk mode to display the sensing dashboard full-screen on boot.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Display Kiosk Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | systemd | | Chromium | |
|
||||
| | autologin + | | Kiosk Launch | |
|
||||
| | X11/Wayland | | (full-screen, | |
|
||||
| | session | | no-UI bars) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Display Manager | |
|
||||
| | (resolution, | |
|
||||
| | rotation, | |
|
||||
| | overscan, | |
|
||||
| | sleep/wake) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Display configuration for kiosk mode.
|
||||
pub struct KioskConfig {
|
||||
/// URL to display (default: "http://localhost:3000").
|
||||
pub url: String,
|
||||
/// Screen rotation in degrees (0, 90, 180, 270).
|
||||
pub rotation: u16,
|
||||
/// Whether to hide the mouse cursor.
|
||||
pub hide_cursor: bool,
|
||||
/// Auto-refresh interval in seconds (0 = disabled).
|
||||
pub auto_refresh_secs: u32,
|
||||
/// Display sleep schedule (e.g., off 23:00-06:00).
|
||||
pub sleep_schedule: Option<SleepSchedule>,
|
||||
/// Overscan compensation percentage (0-10).
|
||||
pub overscan_percent: u8,
|
||||
}
|
||||
|
||||
/// Sleep schedule for display power management.
|
||||
pub struct SleepSchedule {
|
||||
/// Time to turn display off (HH:MM local time).
|
||||
pub sleep_time: String,
|
||||
/// Time to turn display on (HH:MM local time).
|
||||
pub wake_time: String,
|
||||
}
|
||||
```
|
||||
|
||||
**Invariants:**
|
||||
- Chromium kiosk starts only after sensing-server systemd unit is `active`
|
||||
- If Chromium crashes, systemd restarts it within 5 seconds (`Restart=always`)
|
||||
- Display sleep/wake uses CEC commands (HDMI-CEC) to control TV power when available
|
||||
- No browser UI elements are visible (address bar, scrollbars, etc.)
|
||||
|
||||
---
|
||||
|
||||
### 4. WiFi CSI Bridge Context
|
||||
|
||||
**Responsibility:** Extract CSI data from patched WiFi drivers on the TV box and translate it into ESP32-compatible binary frames for the sensing server. This is the Phase 2 custom firmware path.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| WiFi CSI Bridge Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Patched WiFi | | CSI Reader | |
|
||||
| | Driver | | (Netlink / | |
|
||||
| | (kernel space)| | procfs / | |
|
||||
| | CSI hooks | | UDP socket) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Protocol | |
|
||||
| | Translator | |
|
||||
| | (chipset CSI → | |
|
||||
| | ESP32 binary | |
|
||||
| | 0xC5100001) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | UDP Sender | |
|
||||
| | (localhost:5005) |---> sensing-server |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Raw CSI extraction from a WiFi chipset.
|
||||
pub struct ChipsetCsiFrame {
|
||||
/// Source chipset type.
|
||||
pub chipset: WifiChipset,
|
||||
/// Timestamp of extraction (kernel monotonic clock).
|
||||
pub timestamp_us: u64,
|
||||
/// Number of subcarriers (varies by chipset and bandwidth).
|
||||
pub n_subcarriers: u16,
|
||||
/// Number of spatial streams / antennas.
|
||||
pub n_streams: u8,
|
||||
/// Channel frequency in MHz.
|
||||
pub freq_mhz: u16,
|
||||
/// Bandwidth (20/40/80/160 MHz).
|
||||
pub bandwidth_mhz: u16,
|
||||
/// RSSI in dBm.
|
||||
pub rssi_dbm: i8,
|
||||
/// Noise floor estimate in dBm.
|
||||
pub noise_floor_dbm: i8,
|
||||
/// Complex CSI values (I/Q pairs) per subcarrier per stream.
|
||||
pub csi_matrix: Vec<Complex<f32>>,
|
||||
/// Source MAC address (BSSID of the AP being measured).
|
||||
pub source_mac: [u8; 6],
|
||||
}
|
||||
|
||||
/// Supported WiFi chipsets for CSI extraction.
|
||||
pub enum WifiChipset {
|
||||
/// Broadcom BCM43455 via Nexmon CSI patches.
|
||||
BroadcomBcm43455,
|
||||
/// Realtek RTL8822CS via modified rtw88 driver.
|
||||
RealtekRtl8822cs,
|
||||
/// MediaTek MT7661 via mt76 driver modification.
|
||||
MediatekMt7661,
|
||||
}
|
||||
|
||||
/// Translated frame in ESP32 binary protocol (ADR-018).
|
||||
pub struct Esp32CompatFrame {
|
||||
/// Magic: 0xC5100001
|
||||
pub magic: u32,
|
||||
/// Virtual node ID assigned to this WiFi interface.
|
||||
pub node_id: u8,
|
||||
/// Number of antennas / spatial streams.
|
||||
pub n_antennas: u8,
|
||||
/// Number of subcarriers (resampled to match ESP32 format).
|
||||
pub n_subcarriers: u8,
|
||||
/// Frequency in MHz.
|
||||
pub freq_mhz: u16,
|
||||
/// Sequence number (monotonic counter).
|
||||
pub sequence: u32,
|
||||
/// RSSI in dBm.
|
||||
pub rssi: i8,
|
||||
/// Noise floor in dBm.
|
||||
pub noise_floor: i8,
|
||||
/// Amplitude values (extracted from complex CSI).
|
||||
pub amplitudes: Vec<f32>,
|
||||
/// Phase values (extracted from complex CSI).
|
||||
pub phases: Vec<f32>,
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `CsiExtractionService` — Reads raw CSI from patched driver via Netlink socket (BCM43455), procfs (RTL8822CS), or UDP (MT7661)
|
||||
- `SubcarrierResamplerService` — Resamples chipset-specific subcarrier counts to match ESP32 format (e.g., 256 → 128 via decimation or interpolation)
|
||||
- `ProtocolTranslatorService` — Converts `ChipsetCsiFrame` to `Esp32CompatFrame` with ADR-018 binary encoding
|
||||
- `CalibrationService` — Compensates for chipset-specific phase offsets, antenna spacing, and gain differences relative to ESP32 CSI
|
||||
|
||||
**Invariants:**
|
||||
- Bridge assigns virtual `node_id` in range 200-254 (reserved for non-ESP32 sources) to avoid collision with physical ESP32 node IDs (1-199)
|
||||
- Subcarrier resampling preserves frequency ordering (lowest to highest)
|
||||
- Phase values are unwrapped before encoding (continuous, not wrapped to ±π)
|
||||
- Bridge daemon starts only if a compatible patched driver is detected at boot
|
||||
|
||||
---
|
||||
|
||||
### 5. Network Topology Context
|
||||
|
||||
**Responsibility:** Manage network connectivity between ESP32 sensor nodes and TV box appliances, including optional dedicated AP mode and multi-room routing.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Network Topology Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | hostapd | | DHCP Server | |
|
||||
| | (dedicated AP | | (dnsmasq for | |
|
||||
| | for ESP32 | | ESP32 nodes) | |
|
||||
| | mesh) | | | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Topology Manager | |
|
||||
| | (node discovery, | |
|
||||
| | IP assignment, | |
|
||||
| | route config) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Firewall Rules | |
|
||||
| | (iptables/nft: | |
|
||||
| | allow UDP 5005, | |
|
||||
| | block external | |
|
||||
| | access to ESP32 | |
|
||||
| | subnet) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Network topology for a single-room deployment.
|
||||
pub struct RoomTopology {
|
||||
/// Appliance acting as the aggregator.
|
||||
pub appliance: DeviceId,
|
||||
/// Whether the appliance runs a dedicated AP.
|
||||
pub dedicated_ap: bool,
|
||||
/// Connected ESP32 nodes with their assigned IPs.
|
||||
pub nodes: Vec<EspNodeConnection>,
|
||||
/// Upstream network interface (Ethernet or WiFi client).
|
||||
pub uplink_interface: String,
|
||||
/// Sensing network interface (dedicated AP or same as uplink).
|
||||
pub sensing_interface: String,
|
||||
}
|
||||
|
||||
/// An ESP32 node's network connection to the appliance.
|
||||
pub struct EspNodeConnection {
|
||||
/// ESP32 node ID (from firmware NVS).
|
||||
pub node_id: u8,
|
||||
/// MAC address of the ESP32.
|
||||
pub mac: [u8; 6],
|
||||
/// Assigned IP address (via DHCP or static).
|
||||
pub ip: IpAddr,
|
||||
/// Last CSI frame received timestamp.
|
||||
pub last_seen: DateTime<Utc>,
|
||||
/// Average CSI frames per second from this node.
|
||||
pub fps: f32,
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `DedicatedApService` — Configures `hostapd` to create a WPA2 AP on the TV box's WiFi interface, assigns DHCP range via `dnsmasq`, sets up IP forwarding
|
||||
- `NodeDiscoveryService` — Monitors UDP port 5005 for new ESP32 node IDs, registers them in the topology, alerts on node departure (no frames for >30s)
|
||||
- `FirewallService` — Configures `nftables`/`iptables` to isolate the ESP32 subnet from the upstream LAN, allowing only UDP 5005 inbound and HTTP 3000 outbound
|
||||
|
||||
**Invariants:**
|
||||
- Dedicated AP uses a separate WiFi interface or virtual interface (not the uplink)
|
||||
- ESP32 subnet is isolated from upstream LAN by default (firewall rules)
|
||||
- If dedicated AP is disabled, ESP32 nodes must be on the same LAN subnet as the appliance
|
||||
- Node discovery does not require mDNS or any discovery protocol — ESP32 nodes are configured with the appliance's IP via NVS provisioning (ADR-044)
|
||||
|
||||
---
|
||||
|
||||
## Domain Events
|
||||
|
||||
| Event | Published By | Consumed By | Payload |
|
||||
|-------|-------------|-------------|---------|
|
||||
| `ApplianceProvisioned` | Appliance Mgmt | Fleet Dashboard | `{ device_id, name, hardware_model, ip }` |
|
||||
| `ApplianceOnline` | Appliance Mgmt | Fleet Dashboard | `{ device_id, server_version, uptime }` |
|
||||
| `ApplianceUnreachable` | Appliance Mgmt | Fleet Dashboard, Alerting | `{ device_id, last_seen, reason }` |
|
||||
| `ApplianceDegraded` | Appliance Mgmt | Fleet Dashboard, Alerting | `{ device_id, cpu_temp, reason }` |
|
||||
| `OtaUpdateStarted` | Appliance Mgmt | Fleet Dashboard | `{ device_id, from_version, to_version }` |
|
||||
| `OtaUpdateCompleted` | Appliance Mgmt | Fleet Dashboard | `{ device_id, new_version, duration_secs }` |
|
||||
| `OtaUpdateRolledBack` | Appliance Mgmt | Fleet Dashboard, Alerting | `{ device_id, attempted_version, rollback_version, reason }` |
|
||||
| `BinaryBuilt` | Cross-Compilation | Release Pipeline | `{ target, version, binary_size, checksum }` |
|
||||
| `DeploymentPackageCreated` | Cross-Compilation | Appliance Mgmt | `{ target, version, package_url }` |
|
||||
| `KioskStarted` | Display Kiosk | Appliance Mgmt | `{ device_id, url, resolution }` |
|
||||
| `KioskCrashed` | Display Kiosk | Appliance Mgmt | `{ device_id, exit_code, restart_count }` |
|
||||
| `CsiBridgeStarted` | WiFi CSI Bridge | Appliance Mgmt, Sensing Server | `{ device_id, chipset, virtual_node_id }` |
|
||||
| `CsiBridgeFailed` | WiFi CSI Bridge | Appliance Mgmt | `{ device_id, chipset, error }` |
|
||||
| `EspNodeDiscovered` | Network Topology | Appliance Mgmt | `{ appliance_id, node_id, mac, ip }` |
|
||||
| `EspNodeLost` | Network Topology | Appliance Mgmt, Alerting | `{ appliance_id, node_id, last_seen }` |
|
||||
| `DedicatedApStarted` | Network Topology | Appliance Mgmt | `{ appliance_id, ssid, channel }` |
|
||||
|
||||
---
|
||||
|
||||
## Context Map
|
||||
|
||||
```
|
||||
+-------------------+ +---------------------+
|
||||
| Appliance |--------->| Fleet Dashboard |
|
||||
| Management | events | (external UI for |
|
||||
| (fleet state) | -------> | multi-room mgmt) |
|
||||
+--------+----------+ +---------------------+
|
||||
|
|
||||
| provisions, monitors
|
||||
v
|
||||
+-------------------+ +---------------------+
|
||||
| Cross-Compilation |--------->| GitHub Releases |
|
||||
| (build pipeline) | uploads | (binary artifacts) |
|
||||
+-------------------+ +---------------------+
|
||||
|
|
||||
| provides binary
|
||||
v
|
||||
+-------------------+ +---------------------+
|
||||
| Display Kiosk |--------->| Sensing Server |
|
||||
| (Chromium on | loads | (upstream domain, |
|
||||
| HDMI output) | UI from | produces web UI) |
|
||||
+-------------------+ +----------+----------+
|
||||
^
|
||||
+-------------------+ |
|
||||
| WiFi CSI Bridge |-----UDP 5005------>|
|
||||
| (patched driver) | ESP32 compat |
|
||||
+-------------------+ frames |
|
||||
|
|
||||
+-------------------+ |
|
||||
| Network Topology |-----UDP 5005------>|
|
||||
| (ESP32 mesh | ESP32 frames |
|
||||
| connectivity) | |
|
||||
+-------------------+ |
|
||||
```
|
||||
|
||||
**Relationships:**
|
||||
|
||||
| Upstream | Downstream | Relationship | Mechanism |
|
||||
|----------|-----------|--------------|-----------|
|
||||
| Cross-Compilation | Appliance Mgmt | Supplier-Consumer | Build produces binary; Appliance Mgmt deploys it |
|
||||
| Appliance Mgmt | Display Kiosk | Customer-Supplier | Appliance Mgmt starts kiosk after server is healthy |
|
||||
| WiFi CSI Bridge | Sensing Server (external) | Conformist | Bridge adapts its output to match ESP32 binary protocol (ADR-018) |
|
||||
| Network Topology | Sensing Server (external) | Shared Kernel | Both depend on UDP port 5005 and ESP32 node ID scheme |
|
||||
| Appliance Mgmt | Network Topology | Customer-Supplier | Appliance config determines whether dedicated AP is enabled |
|
||||
|
||||
---
|
||||
|
||||
## Anti-Corruption Layers
|
||||
|
||||
### ESP32 Protocol ACL (CSI Bridge)
|
||||
|
||||
The WiFi CSI Bridge translates chipset-specific CSI formats (Nexmon, rtw88, mt76) into the ESP32 binary protocol (ADR-018). The sensing server never knows whether frames came from a real ESP32 or a TV box WiFi chipset. Virtual node IDs (200-254) prevent collision with physical ESP32 IDs but are otherwise treated identically by the ingestion context.
|
||||
|
||||
### Armbian Platform ACL
|
||||
|
||||
Appliance Management abstracts over Armbian specifics (device tree names, boot configuration, dtb overlays) through the `HardwareModel` value object. Higher-level contexts (Cross-Compilation, Display Kiosk) depend only on the target triple (`aarch64-unknown-linux-gnu`) and systemd service interface, not on Amlogic/Allwinner/Rockchip kernel specifics.
|
||||
|
||||
### Fleet Coordination ACL
|
||||
|
||||
For multi-room deployments, each appliance is self-contained (runs its own sensing server, display, and network). The fleet dashboard reads health beacons but never controls individual appliances directly. OTA updates are pulled by each appliance (not pushed), maintaining the appliance as the authority over its own state.
|
||||
|
||||
---
|
||||
|
||||
## Related
|
||||
|
||||
- [ADR-046: Android TV Box / Armbian Deployment](../adr/ADR-046-android-tv-box-armbian-deployment.md) — Primary architectural decision
|
||||
- [ADR-012: ESP32 CSI Sensor Mesh](../adr/ADR-012-esp32-csi-sensor-mesh.md) — ESP32 mesh network design
|
||||
- [ADR-018: Dev Implementation](../adr/ADR-018-dev-implementation.md) — ESP32 binary CSI protocol
|
||||
- [ADR-039: Edge Intelligence](../adr/ADR-039-esp32-edge-intelligence.md) — On-device processing tiers
|
||||
- [ADR-044: Provisioning Tool](../adr/ADR-044-provisioning-tool-enhancements.md) — NVS provisioning for ESP32 nodes
|
||||
- [Hardware Platform Domain Model](hardware-platform-domain-model.md) — Upstream domain (ESP32 hardware)
|
||||
- [Sensing Server Domain Model](sensing-server-domain-model.md) — Upstream domain (server software)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,32 @@
|
||||
# RuvSense Domain Model
|
||||
|
||||
RuvSense is the multistatic WiFi sensing subsystem of RuView. It turns raw radio signals from multiple ESP32 sensors into tracked human poses, vital signs, and spatial awareness — all without cameras.
|
||||
|
||||
This document defines the system using [Domain-Driven Design](https://martinfowler.com/bliki/DomainDrivenDesign.html) (DDD): bounded contexts that own their data and rules, aggregate roots that enforce invariants, value objects that carry meaning, and domain events that connect everything. The goal is to make the system's structure match the physics it models — so that anyone reading the code (or an AI agent modifying it) understands *why* each piece exists, not just *what* it does.
|
||||
|
||||
**Bounded Contexts:**
|
||||
|
||||
| # | Context | Responsibility | Key ADRs | Code |
|
||||
|---|---------|----------------|----------|------|
|
||||
| 1 | [Multistatic Sensing](#1-multistatic-sensing-context) | Collect and fuse CSI from multiple nodes and channels | [ADR-029](../adr/ADR-029-ruvsense-multistatic-sensing-mode.md) | `signal/src/ruvsense/{multiband,phase_align,multistatic}.rs` |
|
||||
| 2 | [Coherence](#2-coherence-context) | Monitor signal quality, gate bad data | [ADR-029](../adr/ADR-029-ruvsense-multistatic-sensing-mode.md) | `signal/src/ruvsense/{coherence,coherence_gate}.rs` |
|
||||
| 3 | [Pose Tracking](#3-pose-tracking-context) | Track people as persistent skeletons with re-ID | [ADR-024](../adr/ADR-024-contrastive-csi-embedding-model.md), [ADR-037](../adr/ADR-037-multi-person-pose-detection.md) | `signal/src/ruvsense/pose_tracker.rs` |
|
||||
| 4 | [Field Model](#4-field-model-context) | Learn room baselines, extract body perturbations | [ADR-030](../adr/ADR-030-ruvsense-persistent-field-model.md) | `signal/src/ruvsense/{field_model,tomography}.rs` |
|
||||
| 5 | [Longitudinal Monitoring](#5-longitudinal-monitoring-context) | Track health trends over days/weeks | [ADR-030](../adr/ADR-030-ruvsense-persistent-field-model.md) | `signal/src/ruvsense/longitudinal.rs` |
|
||||
| 6 | [Spatial Identity](#6-spatial-identity-context) | Cross-room tracking via environment fingerprints | [ADR-030](../adr/ADR-030-ruvsense-persistent-field-model.md) | `signal/src/ruvsense/cross_room.rs` |
|
||||
| 7 | [Edge Intelligence](#7-edge-intelligence-context) | On-device sensing (no server needed) | [ADR-039](../adr/ADR-039-esp32-edge-intelligence.md), [ADR-040](../adr/ADR-040-wasm-programmable-sensing.md) | `firmware/esp32-csi-node/main/edge_processing.c` |
|
||||
|
||||
All code paths shown are relative to `rust-port/wifi-densepose-rs/crates/wifi-densepose-` unless otherwise noted.
|
||||
|
||||
---
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
### Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **Sensing Cycle** | One complete TDMA round (all nodes TX once): 50ms at 20 Hz |
|
||||
| **Sensing Cycle** | One complete TDMA round (all nodes TX once): ~35ms at 28.5 Hz (measured) |
|
||||
| **Link** | A single TX-RX pair; with N nodes there are N×(N-1) directed links |
|
||||
| **Multi-Band Frame** | Fused CSI from one node hopping across multiple channels in one dwell cycle |
|
||||
| **Fused Sensing Frame** | Aggregated observation from all nodes at one sensing cycle, ready for inference |
|
||||
@@ -15,6 +35,8 @@
|
||||
| **Pose Track** | A temporally persistent per-person 17-keypoint trajectory with Kalman state |
|
||||
| **Track Lifecycle** | State machine: Tentative → Active → Lost → Terminated |
|
||||
| **Re-ID Embedding** | 128-dim AETHER contrastive vector encoding body identity |
|
||||
| **Edge Tier** | Processing level on the ESP32: 0 = raw passthrough, 1 = signal cleanup, 2 = vitals, 3 = WASM modules |
|
||||
| **WASM Module** | A small program compiled to WebAssembly that runs on the ESP32 for custom on-device sensing |
|
||||
| **Node** | An ESP32-S3 device acting as both TX and RX in the multistatic mesh |
|
||||
| **Aggregator** | Central device (ESP32/RPi/x86) that collects CSI from all nodes and runs fusion |
|
||||
| **Sensing Schedule** | TDMA slot assignment: which node transmits when |
|
||||
@@ -194,7 +216,7 @@
|
||||
**Domain Services:**
|
||||
- `PersonSeparationService` — Min-cut partitioning of cross-link correlation graph
|
||||
- `TrackAssignmentService` — Bipartite matching of detections to existing tracks
|
||||
- `KalmanPredictionService` — Predict step at 20 Hz (decoupled from measurement rate)
|
||||
- `KalmanPredictionService` — Predict step at 28 Hz (decoupled from measurement rate)
|
||||
- `KalmanUpdateService` — Gated measurement update (subject to coherence gate)
|
||||
- `EmbeddingIdentifierService` — AETHER cosine similarity for re-ID
|
||||
|
||||
@@ -575,7 +597,7 @@ pub trait MeshRepository {
|
||||
### Multistatic Sensing
|
||||
- At least 2 nodes must be active for multistatic fusion (fallback to single-node mode otherwise)
|
||||
- Channel hop sequence must contain at least 1 non-overlapping channel
|
||||
- TDMA cycle period must be ≤50ms for 20 Hz output
|
||||
- TDMA cycle period must be ≤50ms for 28 Hz output
|
||||
- Guard interval must be ≥2× clock drift budget (≥1ms for 50ms cycle)
|
||||
|
||||
### Coherence
|
||||
@@ -1005,7 +1027,7 @@ pub trait SpatialIdentityRepository {
|
||||
### Extended Invariants
|
||||
|
||||
#### Field Model
|
||||
- Baseline calibration requires ≥10 minutes of empty-room CSI (≥12,000 frames at 20 Hz)
|
||||
- Baseline calibration requires ≥10 minutes of empty-room CSI (≥12,000 frames at 28 Hz)
|
||||
- Environmental modes capped at K=5 (more modes overfit to noise)
|
||||
- Tomographic inversion only valid with ≥8 links (4 nodes minimum)
|
||||
- Baseline expires after 24 hours if not refreshed during quiet period
|
||||
@@ -1025,3 +1047,154 @@ pub trait SpatialIdentityRepository {
|
||||
- Transition graph is append-only (immutable audit trail)
|
||||
- No image data stored — only 128-dim embeddings and structural events
|
||||
- Maximum 100 rooms indexed per deployment (HNSW scaling constraint)
|
||||
|
||||
---
|
||||
|
||||
## Part III: Edge Intelligence Bounded Context (ADR-039, ADR-040, ADR-041)
|
||||
|
||||
### 7. Edge Intelligence Context
|
||||
|
||||
**Responsibility:** Run signal processing and sensing algorithms directly on the ESP32-S3, without requiring a server. The node detects presence, measures breathing and heart rate, alerts on falls, and runs custom WASM modules — all locally with instant response.
|
||||
|
||||
This is the only bounded context that runs on the microcontroller rather than the aggregator. It operates independently: the server is optional for visualization, but the ESP32 handles real-time sensing on its own.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ Edge Intelligence Context │
|
||||
│ (runs on ESP32-S3, Core 1) │
|
||||
├──────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌───────────────┐ ┌───────────────┐ │
|
||||
│ │ Phase │ │ Welford │ │
|
||||
│ │ Extractor │ │ Variance │ │
|
||||
│ │ (I/Q → φ, │ │ Tracker │ │
|
||||
│ │ unwrap) │ │ (per-subk) │ │
|
||||
│ └───────┬───────┘ └───────┬───────┘ │
|
||||
│ │ │ │
|
||||
│ └────────┬───────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌────────────────┐ │
|
||||
│ │ Top-K Select │ │
|
||||
│ │ + Bandpass │ │
|
||||
│ │ (breathing: │ │
|
||||
│ │ 0.1-0.5 Hz, │ │
|
||||
│ │ HR: 0.8-2 Hz) │ │
|
||||
│ └────────┬───────┘ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────┼─────────────┐ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │Presence│ │ Vitals │ │ Fall │ │
|
||||
│ │Detector│ │ (BPM via │ │ Detector │ │
|
||||
│ │(motion │ │ zero- │ │ (phase │ │
|
||||
│ │ energy)│ │ crossing)│ │ accel) │ │
|
||||
│ └────┬───┘ └────┬─────┘ └────┬─────┘ │
|
||||
│ └───────────┼──────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌────────────────┐ │
|
||||
│ │ Vitals Packet │──▶ UDP 32-byte (0xC5110002) │
|
||||
│ │ Assembler │ at 1 Hz to aggregator │
|
||||
│ └────────┬───────┘ │
|
||||
│ │ │
|
||||
│ ┌────────▼───────┐ │
|
||||
│ │ WASM3 Runtime │ │
|
||||
│ │ (Tier 3: hot- │──▶ Custom module outputs │
|
||||
│ │ loadable │ │
|
||||
│ │ modules) │ │
|
||||
│ └────────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
- `EdgeProcessingState` (Aggregate Root) — Holds all per-subcarrier state, filter history, and detection flags
|
||||
|
||||
**Value Objects:**
|
||||
- `VitalsPacket` — 32-byte UDP packet: presence, motion, breathing BPM, heart rate BPM, confidence, fall flag, occupancy
|
||||
- `EdgeTier` — Off (0) / BasicSignal (1) / FullVitals (2) / WasmExtended (3)
|
||||
- `PresenceState` — Empty / Present / Moving
|
||||
- `BandpassOutput` — Filtered signal in breathing or heart rate band
|
||||
- `FallAlert` — Phase acceleration exceeding configurable threshold
|
||||
|
||||
**Entities:**
|
||||
- `WasmModule` — A loaded WASM binary with its own memory arena (160 KB), frame budget (10 ms), and timer interval
|
||||
|
||||
**Domain Services:**
|
||||
- `PhaseExtractionService` — Converts raw I/Q to unwrapped phase per subcarrier
|
||||
- `VarianceTrackingService` — Welford running stats for subcarrier selection
|
||||
- `TopKSelectionService` — Picks highest-variance subcarriers for downstream analysis
|
||||
- `BandpassFilterService` — Biquad IIR filters for breathing (0.1-0.5 Hz) and heart rate (0.8-2.0 Hz)
|
||||
- `PresenceDetectionService` — Adaptive threshold calibration (3-sigma over 1200-frame window)
|
||||
- `VitalSignService` — Zero-crossing BPM estimation from filtered phase signals
|
||||
- `FallDetectionService` — Phase acceleration exceeding threshold triggers alert
|
||||
- `WasmRuntimeService` — WASM3 interpreter: load, execute, and sandbox custom modules
|
||||
|
||||
**NVS Configuration (runtime, no reflash needed):**
|
||||
|
||||
| Key | Type | Default | Purpose |
|
||||
|-----|------|---------|---------|
|
||||
| `edge_tier` | u8 | 0 | Processing tier (0/1/2/3) |
|
||||
| `pres_thresh` | u16 | 0 | Presence threshold (0 = auto-calibrate) |
|
||||
| `fall_thresh` | u16 | 2000 | Fall detection threshold (rad/s^2 x 1000) |
|
||||
| `vital_win` | u16 | 256 | Phase history window (frames) |
|
||||
| `vital_int` | u16 | 1000 | Vitals packet interval (ms) |
|
||||
| `subk_count` | u8 | 8 | Top-K subcarrier count |
|
||||
| `wasm_max` | u8 | 4 | Max concurrent WASM modules |
|
||||
| `wasm_verify` | u8 | 0 | Require Ed25519 signature for uploads |
|
||||
|
||||
**Implementation files:**
|
||||
- `firmware/esp32-csi-node/main/edge_processing.c` — DSP pipeline (~750 lines)
|
||||
- `firmware/esp32-csi-node/main/edge_processing.h` — Types and API
|
||||
- `firmware/esp32-csi-node/main/nvs_config.c` — NVS key reader (20 keys)
|
||||
- `firmware/esp32-csi-node/provision.py` — CLI provisioning tool
|
||||
|
||||
**Invariants:**
|
||||
- Edge processing runs on Core 1; WiFi and CSI callbacks run on Core 0 (no contention)
|
||||
- CSI data flows from Core 0 to Core 1 via a lock-free SPSC ring buffer
|
||||
- UDP sends are rate-limited to 50 Hz to prevent lwIP buffer exhaustion (Issue #127)
|
||||
- ENOMEM backoff suppresses sends for 100 ms if lwIP runs out of packet buffers
|
||||
- WASM modules are sandboxed: 160 KB arena, 10 ms frame budget, no direct hardware access
|
||||
- Tier changes via NVS take effect on next reboot — no hot-reconfiguration of the DSP pipeline
|
||||
- Fall detection threshold should be tuned per deployment (default 2000 causes false positives in static environments)
|
||||
|
||||
**Domain Events:**
|
||||
```rust
|
||||
pub enum EdgeEvent {
|
||||
/// Presence state changed
|
||||
PresenceChanged {
|
||||
node_id: u8,
|
||||
state: PresenceState, // Empty / Present / Moving
|
||||
motion_energy: f32,
|
||||
timestamp_ms: u32,
|
||||
},
|
||||
|
||||
/// Fall detected on-device
|
||||
FallDetected {
|
||||
node_id: u8,
|
||||
acceleration: f32, // rad/s^2
|
||||
timestamp_ms: u32,
|
||||
},
|
||||
|
||||
/// Vitals packet emitted
|
||||
VitalsEmitted {
|
||||
node_id: u8,
|
||||
breathing_bpm: f32,
|
||||
heart_rate_bpm: f32,
|
||||
confidence: f32,
|
||||
timestamp_ms: u32,
|
||||
},
|
||||
|
||||
/// WASM module loaded or failed
|
||||
WasmModuleLoaded {
|
||||
slot: u8,
|
||||
module_name: String,
|
||||
success: bool,
|
||||
timestamp_ms: u32,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**Relationship to other contexts:**
|
||||
- Edge Intelligence → Multistatic Sensing: **Alternative** (edge runs on-device; multistatic runs on aggregator — same physics, different compute location)
|
||||
- Edge Intelligence → Pose Tracking: **Upstream** (edge provides presence/vitals; aggregator can skip detection if edge already confirmed occupancy)
|
||||
- Edge Intelligence → Coherence: **Simplified** (edge uses simple variance thresholds instead of full coherence gating)
|
||||
|
||||
@@ -0,0 +1,842 @@
|
||||
# Sensing Server Domain Model
|
||||
|
||||
The Sensing Server is the single-binary deployment surface of WiFi-DensePose. It receives raw CSI frames from ESP32 nodes, processes them into sensing features, streams live data to a web UI, and provides a self-contained workflow for recording data, training models, and running inference -- all without external dependencies.
|
||||
|
||||
This document defines the system using [Domain-Driven Design](https://martinfowler.com/bliki/DomainDrivenDesign.html) (DDD): bounded contexts that own their data and rules, aggregate roots that enforce invariants, value objects that carry meaning, and domain events that connect everything. The server is implemented as a single Axum binary (`wifi-densepose-sensing-server`) with all state managed through `Arc<RwLock<AppStateInner>>`.
|
||||
|
||||
**Bounded Contexts:**
|
||||
|
||||
| # | Context | Responsibility | Key ADRs | Code |
|
||||
|---|---------|----------------|----------|------|
|
||||
| 1 | [CSI Ingestion](#1-csi-ingestion-context) | Receive, decode, and feature-extract CSI frames from ESP32 UDP | [ADR-019](../adr/ADR-019-sensing-only-ui-mode.md), [ADR-035](../adr/ADR-035-live-sensing-ui-accuracy.md) | `sensing-server/src/main.rs` |
|
||||
| 2 | [Model Management](#2-model-management-context) | Load, unload, list RVF models; LoRA profile activation | [ADR-043](../adr/ADR-043-sensing-server-ui-api-completion.md) | `sensing-server/src/model_manager.rs` |
|
||||
| 3 | [CSI Recording](#3-csi-recording-context) | Record CSI frames to .jsonl files, manage recording sessions | [ADR-043](../adr/ADR-043-sensing-server-ui-api-completion.md) | `sensing-server/src/recording.rs` |
|
||||
| 4 | [Training Pipeline](#4-training-pipeline-context) | Background training runs, progress streaming, contrastive pretraining | [ADR-043](../adr/ADR-043-sensing-server-ui-api-completion.md) | `sensing-server/src/training_api.rs` |
|
||||
| 5 | [Visualization](#5-visualization-context) | WebSocket streaming to web UI, Gaussian splat rendering, data transparency | [ADR-019](../adr/ADR-019-sensing-only-ui-mode.md), [ADR-035](../adr/ADR-035-live-sensing-ui-accuracy.md) | `ui/` |
|
||||
|
||||
All code paths shown are relative to `rust-port/wifi-densepose-rs/crates/wifi-densepose-` unless otherwise noted.
|
||||
|
||||
---
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
### Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **Sensing Update** | A complete JSON message broadcast to WebSocket clients each tick, containing node data, features, classification, signal field, and optional vital signs |
|
||||
| **Tick** | One processing cycle of the sensing loop (default 100ms = 10 fps, configurable via `--tick-ms`) |
|
||||
| **Data Source** | Origin of CSI data: `esp32` (UDP port 5005), `wifi` (Windows RSSI), `simulated` (synthetic), or `auto` (try ESP32 then fall back) |
|
||||
| **RVF Model** | A `.rvf` container file holding trained weights, manifest metadata, optional LoRA adapters, and vital sign configuration |
|
||||
| **LoRA Profile** | A lightweight adapter applied on top of a base RVF model for environment-specific fine-tuning without retraining the full model |
|
||||
| **Recording Session** | A period during which CSI frames are appended to a `.csi.jsonl` file, identified by a session ID and optional activity label |
|
||||
| **Training Run** | A background task that loads recorded CSI data, extracts features, trains a regularised linear model, and exports a `.rvf` container |
|
||||
| **Frame History** | A circular buffer of the last 100 CSI amplitude vectors used for temporal analysis (sliding-window variance, Goertzel breathing estimation) |
|
||||
| **Goertzel Filter** | A frequency-domain estimator applied to the frame history to detect breathing rate (0.1--0.5 Hz) via a 9-candidate filter bank |
|
||||
| **Signal Field** | A 20x1x20 grid of interpolated signal intensity values rendered as Gaussian splats in the UI |
|
||||
| **Pose Source** | Whether pose keypoints are `signal_derived` (analytical from CSI features) or `model_inference` (from a loaded RVF model) |
|
||||
| **Progressive Loader** | A two-layer model loading strategy: Layer A loads instantly for basic inference, Layer B loads in background for full accuracy |
|
||||
| **Sensing-Only Mode** | UI mode when the DensePose backend is unavailable; suppresses DensePose tabs, shows only sensing and signal visualization |
|
||||
| **AppStateInner** | The single shared state struct holding all server state, accessed via `Arc<RwLock<AppStateInner>>` |
|
||||
| **PCK Score** | Percentage of Correct Keypoints -- the primary accuracy metric for pose estimation models |
|
||||
| **Contrastive Pretraining** | Self-supervised training on unlabeled CSI data that learns signal representations before supervised fine-tuning (ADR-024) |
|
||||
|
||||
---
|
||||
|
||||
## Bounded Contexts
|
||||
|
||||
### 1. CSI Ingestion Context
|
||||
|
||||
**Responsibility:** Receive raw CSI frames from ESP32 nodes via UDP (port 5005), decode the binary protocol, extract temporal and frequency-domain features, and produce a `SensingUpdate` each tick.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| CSI Ingestion Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | UDP Listener | | Data Source | |
|
||||
| | (port 5005) | | Selector | |
|
||||
| | Esp32Frame | | (auto/esp32/ | |
|
||||
| | parser | | wifi/sim) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Frame History | |
|
||||
| | Buffer | |
|
||||
| | (VecDeque<Vec>, | |
|
||||
| | 100 frames) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Feature | |
|
||||
| | Extractor | |
|
||||
| | (Welford stats, | |
|
||||
| | Goertzel FFT, | |
|
||||
| | L2 motion) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Vital Sign | |
|
||||
| | Detector |---> SensingUpdate |
|
||||
| | (HR, RR, | |
|
||||
| | breathing) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
|
||||
```rust
|
||||
/// Aggregate Root: The central shared state of the sensing server.
|
||||
/// All mutations go through RwLock. All handler functions receive
|
||||
/// State<Arc<RwLock<AppStateInner>>>.
|
||||
pub struct AppStateInner {
|
||||
/// Most recent sensing update broadcast to clients.
|
||||
latest_update: Option<SensingUpdate>,
|
||||
/// RSSI history for sparkline display.
|
||||
rssi_history: VecDeque<f64>,
|
||||
/// Circular buffer of recent CSI amplitude vectors (100 frames).
|
||||
frame_history: VecDeque<Vec<f64>>,
|
||||
/// Monotonic tick counter.
|
||||
tick: u64,
|
||||
/// Active data source identifier ("esp32", "wifi", "simulated").
|
||||
source: String,
|
||||
/// Broadcast channel for WebSocket fan-out.
|
||||
tx: broadcast::Sender<String>,
|
||||
/// Vital sign detector instance.
|
||||
vital_detector: VitalSignDetector,
|
||||
/// Most recent vital signs reading.
|
||||
latest_vitals: VitalSigns,
|
||||
/// Smoothed person count (EMA) for hysteresis.
|
||||
smoothed_person_score: f64,
|
||||
// ... model, recording, training fields (see other contexts)
|
||||
}
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// A complete sensing update broadcast to WebSocket clients each tick.
|
||||
pub struct SensingUpdate {
|
||||
pub msg_type: String, // always "sensing_update"
|
||||
pub timestamp: f64, // Unix timestamp with ms precision
|
||||
pub source: String, // "esp32" | "wifi" | "simulated"
|
||||
pub tick: u64, // monotonic tick counter
|
||||
pub nodes: Vec<NodeInfo>, // per-node CSI data
|
||||
pub features: FeatureInfo, // extracted signal features
|
||||
pub classification: ClassificationInfo,
|
||||
pub signal_field: SignalField,
|
||||
pub vital_signs: Option<VitalSigns>,
|
||||
pub persons: Option<Vec<PersonDetection>>,
|
||||
pub estimated_persons: Option<usize>,
|
||||
}
|
||||
|
||||
/// Per-node CSI data received from one ESP32.
|
||||
pub struct NodeInfo {
|
||||
pub node_id: u8,
|
||||
pub rssi_dbm: f64,
|
||||
pub position: [f64; 3],
|
||||
pub amplitude: Vec<f64>,
|
||||
pub subcarrier_count: usize,
|
||||
}
|
||||
|
||||
/// Extracted signal features from the frame history buffer.
|
||||
pub struct FeatureInfo {
|
||||
pub mean_rssi: f64,
|
||||
pub variance: f64,
|
||||
pub motion_band_power: f64,
|
||||
pub breathing_band_power: f64,
|
||||
pub dominant_freq_hz: f64,
|
||||
pub change_points: usize,
|
||||
pub spectral_power: f64,
|
||||
}
|
||||
|
||||
/// Motion classification derived from features.
|
||||
pub struct ClassificationInfo {
|
||||
pub motion_level: String, // "empty" | "static" | "active"
|
||||
pub presence: bool,
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
/// Interpolated signal field for Gaussian splat visualization.
|
||||
pub struct SignalField {
|
||||
pub grid_size: [usize; 3], // [20, 1, 20]
|
||||
pub values: Vec<f64>,
|
||||
}
|
||||
|
||||
/// ESP32 binary CSI frame (ADR-018 protocol, 20-byte header).
|
||||
pub struct Esp32Frame {
|
||||
pub magic: u32, // 0xC5100001
|
||||
pub node_id: u8,
|
||||
pub n_antennas: u8,
|
||||
pub n_subcarriers: u8,
|
||||
pub freq_mhz: u16,
|
||||
pub sequence: u32,
|
||||
pub rssi: i8,
|
||||
pub noise_floor: i8,
|
||||
pub amplitudes: Vec<f64>,
|
||||
pub phases: Vec<f64>,
|
||||
}
|
||||
|
||||
/// Data source selection enum.
|
||||
pub enum DataSource {
|
||||
Esp32Udp, // Real ESP32 CSI via UDP port 5005
|
||||
WindowsRssi, // Windows WiFi RSSI via netsh
|
||||
Simulated, // Synthetic sine-wave data
|
||||
Auto, // Try ESP32, fall back to Windows, then simulated
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `FeatureExtractionService` -- Computes temporal variance (Welford), Goertzel breathing estimation (9-band filter bank), L2 frame-to-frame motion score, SNR-based signal quality
|
||||
- `VitalSignDetectionService` -- Estimates breathing rate, heart rate, and confidence from CSI phase history
|
||||
- `DataSourceSelectionService` -- Probes UDP port 5005 for ESP32 frames; falls back through Windows RSSI then simulation
|
||||
|
||||
**Invariants:**
|
||||
- Frame history buffer never exceeds 100 entries (oldest dropped on push)
|
||||
- Goertzel breathing estimate requires 3x SNR above noise to be reported
|
||||
- Source type is determined once at startup and does not change during runtime
|
||||
|
||||
---
|
||||
|
||||
### 2. Model Management Context
|
||||
|
||||
**Responsibility:** Discover `.rvf` model files from `data/models/`, load weights into memory for inference, manage the active model lifecycle, and support LoRA profile activation.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Model Management Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Model Scanner | | RVF Reader | |
|
||||
| | (data/models/ | | (parse .rvf | |
|
||||
| | *.rvf enum) | | manifest) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Model Registry | |
|
||||
| | (Vec<ModelInfo>) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Model Loader | |
|
||||
| | (RvfReader -> |---> LoadedModelState |
|
||||
| | weights, | |
|
||||
| | LoRA profiles) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | LoRA Activator | |
|
||||
| | (profile switch) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
|
||||
```rust
|
||||
/// Aggregate Root: Runtime state for a loaded RVF model.
|
||||
/// At most one LoadedModelState exists at any time.
|
||||
pub struct LoadedModelState {
|
||||
/// Model identifier (derived from filename without .rvf extension).
|
||||
pub model_id: String,
|
||||
/// Original filename on disk.
|
||||
pub filename: String,
|
||||
/// Version string from the RVF manifest.
|
||||
pub version: String,
|
||||
/// Description from the RVF manifest.
|
||||
pub description: String,
|
||||
/// LoRA profiles available in this model.
|
||||
pub lora_profiles: Vec<String>,
|
||||
/// Currently active LoRA profile (if any).
|
||||
pub active_lora_profile: Option<String>,
|
||||
/// Model weights (f32 parameters).
|
||||
pub weights: Vec<f32>,
|
||||
/// Number of frames processed since load.
|
||||
pub frames_processed: u64,
|
||||
/// Cumulative inference time for avg calculation.
|
||||
pub total_inference_ms: f64,
|
||||
/// When the model was loaded.
|
||||
pub loaded_at: Instant,
|
||||
}
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Summary information for a model discovered on disk.
|
||||
pub struct ModelInfo {
|
||||
pub id: String,
|
||||
pub filename: String,
|
||||
pub version: String,
|
||||
pub description: String,
|
||||
pub size_bytes: u64,
|
||||
pub created_at: String,
|
||||
pub pck_score: Option<f64>,
|
||||
pub has_quantization: bool,
|
||||
pub lora_profiles: Vec<String>,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
/// Information about the currently loaded model with runtime stats.
|
||||
pub struct ActiveModelInfo {
|
||||
pub model_id: String,
|
||||
pub filename: String,
|
||||
pub version: String,
|
||||
pub description: String,
|
||||
pub avg_inference_ms: f64,
|
||||
pub frames_processed: u64,
|
||||
pub pose_source: String, // "model_inference"
|
||||
pub lora_profiles: Vec<String>,
|
||||
pub active_lora_profile: Option<String>,
|
||||
}
|
||||
|
||||
/// Request to load a model by ID.
|
||||
pub struct LoadModelRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
/// Request to activate a LoRA profile.
|
||||
pub struct ActivateLoraRequest {
|
||||
pub model_id: String,
|
||||
pub profile_name: String,
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `ModelScanService` -- Scans `data/models/` at startup for `.rvf` files, parses each with `RvfReader` to extract manifest metadata
|
||||
- `ModelLoadService` -- Reads model weights from an RVF container into memory, sets `model_loaded = true`
|
||||
- `LoraActivationService` -- Switches the active LoRA adapter on a loaded model without full reload
|
||||
|
||||
**Invariants:**
|
||||
- Only one model can be loaded at a time; loading a new model implicitly unloads the previous one
|
||||
- A model must be loaded before a LoRA profile can be activated
|
||||
- The `active_lora_profile` must be one of the model's declared `lora_profiles`
|
||||
- Model deletion is refused if the model is currently loaded (must unload first)
|
||||
- `data/models/` directory is created at startup if it does not exist
|
||||
|
||||
---
|
||||
|
||||
### 3. CSI Recording Context
|
||||
|
||||
**Responsibility:** Capture CSI frames to `.csi.jsonl` files during active recording sessions, manage session lifecycle, and provide download/delete operations on stored recordings.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| CSI Recording Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Start/Stop | | Auto-Stop | |
|
||||
| | Controller | | Timer | |
|
||||
| | (REST API) | | (duration_ | |
|
||||
| | | | secs check) | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Recording State | |
|
||||
| | (session_id, | |
|
||||
| | frame_count, | |
|
||||
| | file_path) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Frame Writer | |
|
||||
| | (maybe_record_ |---> .csi.jsonl file |
|
||||
| | frame on each | |
|
||||
| | tick) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Metadata Writer | |
|
||||
| | (.meta.json on | |
|
||||
| | stop) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
|
||||
```rust
|
||||
/// Aggregate Root: Runtime state for the active CSI recording session.
|
||||
/// At most one RecordingState can be active at any time.
|
||||
pub struct RecordingState {
|
||||
/// Whether a recording is currently active.
|
||||
pub active: bool,
|
||||
/// Session ID of the active recording.
|
||||
pub session_id: String,
|
||||
/// Session display name.
|
||||
pub session_name: String,
|
||||
/// Optional label / activity tag (e.g., "walking", "standing").
|
||||
pub label: Option<String>,
|
||||
/// Path to the JSONL file being written.
|
||||
pub file_path: PathBuf,
|
||||
/// Number of frames written so far.
|
||||
pub frame_count: u64,
|
||||
/// When the recording started (monotonic clock).
|
||||
pub start_time: Instant,
|
||||
/// ISO-8601 start timestamp for metadata.
|
||||
pub started_at: String,
|
||||
/// Optional auto-stop duration in seconds.
|
||||
pub duration_secs: Option<u64>,
|
||||
}
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Metadata for a completed or active recording session.
|
||||
pub struct RecordingSession {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub started_at: String,
|
||||
pub ended_at: Option<String>,
|
||||
pub frame_count: u64,
|
||||
pub file_size_bytes: u64,
|
||||
pub file_path: String,
|
||||
}
|
||||
|
||||
/// A single recorded CSI frame line (JSONL format).
|
||||
pub struct RecordedFrame {
|
||||
pub timestamp: f64,
|
||||
pub subcarriers: Vec<f64>,
|
||||
pub rssi: f64,
|
||||
pub noise_floor: f64,
|
||||
pub features: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Request to start a new recording session.
|
||||
pub struct StartRecordingRequest {
|
||||
pub session_name: String,
|
||||
pub label: Option<String>,
|
||||
pub duration_secs: Option<u64>,
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `RecordingLifecycleService` -- Creates a new `.csi.jsonl` file, generates session ID, manages start/stop transitions
|
||||
- `FrameWriterService` -- Called on each tick via `maybe_record_frame()`, appends a `RecordedFrame` JSON line to the active file
|
||||
- `AutoStopService` -- Checks elapsed time against `duration_secs` on each tick; triggers stop when exceeded
|
||||
- `RecordingScanService` -- Enumerates `data/recordings/` for `.csi.jsonl` files and reads companion `.meta.json` for session metadata
|
||||
|
||||
**Invariants:**
|
||||
- Only one recording session can be active at a time; starting a new recording while one is active returns HTTP 409 Conflict
|
||||
- Recording with `duration_secs` set auto-stops after the specified elapsed time
|
||||
- A `.meta.json` companion file is written when a recording stops, capturing final frame count and duration
|
||||
- `data/recordings/` directory is created at startup if it does not exist
|
||||
- Frame writer acquires a read lock on `AppStateInner` per tick; stop acquires a write lock
|
||||
|
||||
---
|
||||
|
||||
### 4. Training Pipeline Context
|
||||
|
||||
**Responsibility:** Run background training against recorded CSI data, stream epoch-level progress via WebSocket, and export trained models as `.rvf` containers. Supports supervised training, contrastive pretraining (ADR-024), and LoRA fine-tuning.
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Training Pipeline Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | Training API | | WebSocket | |
|
||||
| | (start/stop/ | | Progress | |
|
||||
| | status) | | Streamer | |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | ^ |
|
||||
| v | |
|
||||
| +-------------------+ | |
|
||||
| | Training | | |
|
||||
| | Orchestrator +--------+ |
|
||||
| | (tokio::spawn) | broadcast::Sender |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Feature | |
|
||||
| | Extractor | |
|
||||
| | (subcarrier var, | |
|
||||
| | Goertzel power, | |
|
||||
| | temporal grad) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | Gradient Descent | |
|
||||
| | Trainer | |
|
||||
| | (batch SGD, |---> TrainingProgress |
|
||||
| | early stopping, | |
|
||||
| | warmup) | |
|
||||
| +--------+----------+ |
|
||||
| v |
|
||||
| +-------------------+ |
|
||||
| | RVF Exporter | |
|
||||
| | (RvfBuilder -> |---> data/models/*.rvf |
|
||||
| | .rvf container) | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates:**
|
||||
|
||||
```rust
|
||||
/// Aggregate Root: Runtime training state stored in AppStateInner.
|
||||
/// At most one training run can be active at any time.
|
||||
pub struct TrainingState {
|
||||
/// Current status snapshot.
|
||||
pub status: TrainingStatus,
|
||||
/// Handle to the background training task (for cancellation).
|
||||
pub task_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Current training status (returned by GET /api/v1/train/status).
|
||||
pub struct TrainingStatus {
|
||||
pub active: bool,
|
||||
pub epoch: u32,
|
||||
pub total_epochs: u32,
|
||||
pub train_loss: f64,
|
||||
pub val_pck: f64, // Percentage of Correct Keypoints
|
||||
pub val_oks: f64, // Object Keypoint Similarity
|
||||
pub lr: f64, // current learning rate
|
||||
pub best_pck: f64,
|
||||
pub best_epoch: u32,
|
||||
pub patience_remaining: u32,
|
||||
pub eta_secs: Option<u64>,
|
||||
pub phase: String, // "idle" | "training" | "complete" | "failed"
|
||||
}
|
||||
|
||||
/// Progress update sent over WebSocket to connected UI clients.
|
||||
pub struct TrainingProgress {
|
||||
pub epoch: u32,
|
||||
pub batch: u32,
|
||||
pub total_batches: u32,
|
||||
pub train_loss: f64,
|
||||
pub val_pck: f64,
|
||||
pub val_oks: f64,
|
||||
pub lr: f64,
|
||||
pub phase: String,
|
||||
}
|
||||
|
||||
/// Training configuration submitted with a start request.
|
||||
pub struct TrainingConfig {
|
||||
pub epochs: u32, // default: 100
|
||||
pub batch_size: u32, // default: 8
|
||||
pub learning_rate: f64, // default: 0.001
|
||||
pub weight_decay: f64, // default: 1e-4
|
||||
pub early_stopping_patience: u32, // default: 20
|
||||
pub warmup_epochs: u32, // default: 5
|
||||
pub pretrained_rvf: Option<String>,
|
||||
pub lora_profile: Option<String>,
|
||||
}
|
||||
|
||||
/// Request to start supervised training.
|
||||
pub struct StartTrainingRequest {
|
||||
pub dataset_ids: Vec<String>, // recording session IDs
|
||||
pub config: TrainingConfig,
|
||||
}
|
||||
|
||||
/// Request to start contrastive pretraining (ADR-024).
|
||||
pub struct PretrainRequest {
|
||||
pub dataset_ids: Vec<String>,
|
||||
pub epochs: u32, // default: 50
|
||||
pub lr: f64, // default: 0.001
|
||||
}
|
||||
|
||||
/// Request to start LoRA fine-tuning.
|
||||
pub struct LoraTrainRequest {
|
||||
pub base_model_id: String,
|
||||
pub dataset_ids: Vec<String>,
|
||||
pub profile_name: String,
|
||||
pub rank: u8, // default: 8
|
||||
pub epochs: u32, // default: 30
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `TrainingOrchestrationService` -- Spawns a background `tokio::task`, loads recorded frames, runs feature extraction, executes gradient descent with early stopping and warmup
|
||||
- `FeatureExtractionService` -- Computes per-subcarrier sliding-window variance, temporal gradients, Goertzel frequency-domain power across 9 bands, and 3 global scalar features (mean amplitude, std, motion score)
|
||||
- `ProgressBroadcastService` -- Sends `TrainingProgress` messages through a `broadcast::Sender` channel that WebSocket handlers subscribe to
|
||||
- `RvfExportService` -- Uses `RvfBuilder` to write the best checkpoint as a `.rvf` container to `data/models/`
|
||||
|
||||
**Invariants:**
|
||||
- Only one training run can be active at a time; starting training while one is running returns HTTP 409 Conflict
|
||||
- Training requires at least one recording with a minimum frame count before starting
|
||||
- Early stopping halts training after `patience` epochs with no improvement in `val_pck`
|
||||
- Learning rate warmup ramps linearly from 0 to `learning_rate` over `warmup_epochs`
|
||||
- On completion, the best model (by `val_pck`) is automatically exported as `.rvf`
|
||||
- Training status phase transitions: `idle` -> `training` -> `complete` | `failed` -> `idle`
|
||||
- Stopping an active training run aborts the background task via `JoinHandle::abort()` and resets phase to `idle`
|
||||
|
||||
---
|
||||
|
||||
### 5. Visualization Context
|
||||
|
||||
**Responsibility:** Stream sensing data to web UI clients via WebSocket, render Gaussian splat visualizations, display data source transparency indicators, and manage UI mode (full vs. sensing-only).
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Visualization Context |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ +----------------+ |
|
||||
| | WebSocket | | Sensing | |
|
||||
| | Hub | | Service (JS) | |
|
||||
| | (/ws/sensing) | | (client-side | |
|
||||
| | broadcast:: | | reconnect + | |
|
||||
| | Receiver | | sim fallback)| |
|
||||
| +-------+--------+ +-------+--------+ |
|
||||
| | | |
|
||||
| +----------+----------+ |
|
||||
| v |
|
||||
| +----------------------------------------------+ |
|
||||
| | UI Components | |
|
||||
| | | |
|
||||
| | +----------+ +----------+ +----------+ | |
|
||||
| | | Sensing | | Live | | Models | | |
|
||||
| | | Tab | | Demo Tab | | Tab | | |
|
||||
| | | (splats) | | (pose) | | (manage) | | |
|
||||
| | +----------+ +----------+ +----------+ | |
|
||||
| | +----------+ +----------+ | |
|
||||
| | | Recording| | Training | | |
|
||||
| | | Tab | | Tab | | |
|
||||
| | | (capture)| | (train) | | |
|
||||
| | +----------+ +----------+ | |
|
||||
| +----------------------------------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Value Objects:**
|
||||
|
||||
```rust
|
||||
/// Data source indicator shown in the UI (ADR-035).
|
||||
pub enum DataSourceIndicator {
|
||||
LiveEsp32, // Green banner: "LIVE - ESP32"
|
||||
Reconnecting, // Yellow banner: "RECONNECTING..."
|
||||
Simulated, // Red banner: "SIMULATED DATA"
|
||||
}
|
||||
|
||||
/// Pose estimation mode badge (ADR-035).
|
||||
pub enum EstimationMode {
|
||||
SignalDerived, // Green badge: analytical pose from CSI features
|
||||
ModelInference, // Blue badge: neural network inference from loaded RVF
|
||||
}
|
||||
|
||||
/// Render mode for pose visualization (ADR-035).
|
||||
pub enum RenderMode {
|
||||
Skeleton, // Green lines connecting joints + red keypoint dots
|
||||
Keypoints, // Large colored dots with glow and labels
|
||||
Heatmap, // Gaussian radial blobs per keypoint, faint skeleton overlay
|
||||
Dense, // Body region segmentation with colored filled polygons
|
||||
}
|
||||
```
|
||||
|
||||
**Domain Services:**
|
||||
- `WebSocketBroadcastService` -- Subscribes to `broadcast::Sender<String>`, forwards each `SensingUpdate` JSON to all connected WebSocket clients
|
||||
- `SensingServiceJS` -- Client-side JavaScript that manages WebSocket connection, tracks `dataSource` state, falls back to simulation after 5 failed reconnect attempts (~30s delay)
|
||||
- `GaussianSplatRenderer` -- Custom GLSL `ShaderMaterial` rendering point-cloud splats on a 20x20 floor grid, colored by signal intensity
|
||||
- `PoseRenderer` -- Renders skeleton, keypoints, heatmap, or dense body segmentation modes
|
||||
- `BackendDetector` -- Auto-detects whether the full DensePose backend is available; sets `sensingOnlyMode = true` if unreachable
|
||||
|
||||
**Invariants:**
|
||||
- WebSocket sensing service is started on application init, not lazily on tab visit (ADR-043 fix)
|
||||
- Simulation fallback is delayed to 5 failed reconnect attempts (~30 seconds) to avoid premature synthetic data
|
||||
- `pose_source` field is passed through data conversion so the Estimation Mode badge displays correctly
|
||||
- Dashboard and Live Demo tabs read `sensingService.dataSource` at load time -- the service must already be connected
|
||||
|
||||
---
|
||||
|
||||
## Domain Events
|
||||
|
||||
| Event | Published By | Consumed By | Payload |
|
||||
|-------|-------------|-------------|---------|
|
||||
| `ServerStarted` | CSI Ingestion | Visualization | `{ http_port, udp_port, source_type }` |
|
||||
| `CsiFrameIngested` | CSI Ingestion | Recording, Visualization | `{ source, node_id, subcarrier_count, tick }` |
|
||||
| `SensingUpdateBroadcast` | CSI Ingestion | Visualization (WebSocket) | Full `SensingUpdate` JSON |
|
||||
| `ModelLoaded` | Model Management | CSI Ingestion (inference path) | `{ model_id, weight_count, version }` |
|
||||
| `ModelUnloaded` | Model Management | CSI Ingestion | `{ model_id }` |
|
||||
| `LoraProfileActivated` | Model Management | CSI Ingestion | `{ model_id, profile_name }` |
|
||||
| `RecordingStarted` | Recording | Visualization | `{ session_id, session_name, file_path }` |
|
||||
| `RecordingStopped` | Recording | Visualization | `{ session_id, frame_count, duration_secs }` |
|
||||
| `TrainingStarted` | Training Pipeline | Visualization | `{ run_id, config, recording_ids }` |
|
||||
| `TrainingEpochComplete` | Training Pipeline | Visualization (WebSocket) | `{ epoch, total_epochs, train_loss, val_pck, lr }` |
|
||||
| `TrainingComplete` | Training Pipeline | Model Management, Visualization | `{ run_id, final_pck, model_path }` |
|
||||
| `TrainingFailed` | Training Pipeline | Visualization | `{ run_id, error_message }` |
|
||||
| `WebSocketClientConnected` | Visualization | -- | `{ endpoint, client_addr }` |
|
||||
| `WebSocketClientDisconnected` | Visualization | -- | `{ endpoint, client_addr }` |
|
||||
|
||||
In the current implementation, events are realized through two mechanisms:
|
||||
1. **`broadcast::Sender<String>`** for WebSocket fan-out of sensing updates
|
||||
2. **`broadcast::Sender<TrainingProgress>`** for training progress streaming
|
||||
3. **State mutations via RwLock** where other contexts read state changes on their next tick
|
||||
|
||||
---
|
||||
|
||||
## Context Map
|
||||
|
||||
```
|
||||
+-------------------+ +---------------------+
|
||||
| CSI Ingestion |--------->| Visualization |
|
||||
| (produces | publish | (WebSocket |
|
||||
| SensingUpdate) | -------> | consumers) |
|
||||
+--------+----------+ +----------+----------+
|
||||
| |
|
||||
| maybe_record_frame() | reads dataSource
|
||||
v |
|
||||
+-------------------+ |
|
||||
| CSI Recording | |
|
||||
| (hooks into | |
|
||||
| tick loop) | |
|
||||
+--------+----------+ |
|
||||
| |
|
||||
| provides dataset_ids |
|
||||
v |
|
||||
+-------------------+ +----------+----------+
|
||||
| Training Pipeline |--------->| Model Management |
|
||||
| (reads .jsonl, | exports | (loads .rvf for |
|
||||
| trains model) | .rvf --> | inference) |
|
||||
+-------------------+ +----------+----------+
|
||||
|
|
||||
| model weights
|
||||
v
|
||||
+----------+----------+
|
||||
| CSI Ingestion |
|
||||
| (inference path |
|
||||
| uses loaded model)|
|
||||
+----------------------+
|
||||
```
|
||||
|
||||
**Relationships:**
|
||||
|
||||
| Upstream | Downstream | Relationship | Mechanism |
|
||||
|----------|-----------|--------------|-----------|
|
||||
| CSI Ingestion | Visualization | Published Language | `broadcast::Sender<String>` with `SensingUpdate` JSON schema |
|
||||
| CSI Ingestion | CSI Recording | Shared Kernel | `maybe_record_frame()` called from the ingestion tick loop |
|
||||
| CSI Recording | Training Pipeline | Conformist | Training reads `.csi.jsonl` files produced by recording; no negotiation on format |
|
||||
| Training Pipeline | Model Management | Supplier-Consumer | Training exports `.rvf` to `data/models/`; Model Management scans and loads |
|
||||
| Model Management | CSI Ingestion | Shared Kernel | Loaded weights stored in `AppStateInner`; ingestion reads them for inference |
|
||||
| Training Pipeline | Visualization | Published Language | `broadcast::Sender<TrainingProgress>` with progress JSON schema |
|
||||
|
||||
---
|
||||
|
||||
## Anti-Corruption Layers
|
||||
|
||||
### ESP32 Binary Protocol ACL
|
||||
|
||||
The ESP32 sends CSI frames using a compact binary protocol (ADR-018): 20-byte header with magic `0xC5100001`, followed by amplitude and phase arrays. The `Esp32Frame` parser in the ingestion context decodes this binary format into domain value objects (`NodeInfo`, amplitude/phase vectors) before any downstream processing. No other context handles raw UDP bytes.
|
||||
|
||||
### RVF Container ACL
|
||||
|
||||
The `.rvf` container format encapsulates model weights, manifest metadata, vital sign configuration, and optional LoRA adapters. The `RvfReader` and `RvfBuilder` types in the `rvf_container` module provide the anti-corruption layer between the on-disk binary format and the domain types (`ModelInfo`, `LoadedModelState`). The training pipeline writes through `RvfBuilder`; the model management context reads through `RvfReader`.
|
||||
|
||||
### Sensing-Only Mode ACL (Client-Side)
|
||||
|
||||
When the DensePose backend (port 8000) is unreachable, the client-side `BackendDetector` sets `sensingOnlyMode = true`. The `ApiService.request()` method short-circuits all requests to the DensePose backend, returning empty responses instead of `ERR_CONNECTION_REFUSED`. This prevents DensePose-specific concerns from leaking into the sensing UI.
|
||||
|
||||
### JSONL Recording Format ACL
|
||||
|
||||
CSI frames are recorded as newline-delimited JSON (`.csi.jsonl`). The `RecordedFrame` struct defines the schema: `{timestamp, subcarriers, rssi, noise_floor, features}`. The training pipeline reads through this schema, extracting subcarrier arrays for feature computation. If the internal sensing representation changes, only the `maybe_record_frame()` serializer needs updating -- the training pipeline depends only on the `RecordedFrame` contract.
|
||||
|
||||
---
|
||||
|
||||
## REST API Surface
|
||||
|
||||
All endpoints share `AppStateInner` via `Arc<RwLock<AppStateInner>>`.
|
||||
|
||||
### CSI Ingestion & Sensing
|
||||
|
||||
| Method | Path | Context | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| GET | `/api/v1/sensing/latest` | Ingestion | Latest sensing update |
|
||||
| WS | `/ws/sensing` | Visualization | Streaming sensing updates |
|
||||
|
||||
### Model Management
|
||||
|
||||
| Method | Path | Context | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| GET | `/api/v1/models` | Model Mgmt | List all discovered `.rvf` models |
|
||||
| GET | `/api/v1/models/:id` | Model Mgmt | Detailed info for a specific model |
|
||||
| GET | `/api/v1/models/active` | Model Mgmt | Active model with runtime stats |
|
||||
| POST | `/api/v1/models/load` | Model Mgmt | Load model weights into memory |
|
||||
| POST | `/api/v1/models/unload` | Model Mgmt | Unload the active model |
|
||||
| DELETE | `/api/v1/models/:id` | Model Mgmt | Delete a model file from disk |
|
||||
| GET | `/api/v1/models/lora/profiles` | Model Mgmt | List LoRA profiles for active model |
|
||||
| POST | `/api/v1/models/lora/activate` | Model Mgmt | Activate a LoRA adapter |
|
||||
|
||||
### CSI Recording
|
||||
|
||||
| Method | Path | Context | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| POST | `/api/v1/recording/start` | Recording | Start a new recording session |
|
||||
| POST | `/api/v1/recording/stop` | Recording | Stop the active recording |
|
||||
| GET | `/api/v1/recording/list` | Recording | List all recording sessions |
|
||||
| GET | `/api/v1/recording/download/:id` | Recording | Download a `.csi.jsonl` file |
|
||||
| DELETE | `/api/v1/recording/:id` | Recording | Delete a recording |
|
||||
|
||||
### Training Pipeline
|
||||
|
||||
| Method | Path | Context | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| POST | `/api/v1/train/start` | Training | Start supervised training |
|
||||
| POST | `/api/v1/train/stop` | Training | Stop the active training run |
|
||||
| GET | `/api/v1/train/status` | Training | Current training phase and metrics |
|
||||
| POST | `/api/v1/train/pretrain` | Training | Start contrastive pretraining |
|
||||
| POST | `/api/v1/train/lora` | Training | Start LoRA fine-tuning |
|
||||
| WS | `/ws/train/progress` | Training | Streaming training progress |
|
||||
|
||||
---
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
data/
|
||||
+-- models/ # RVF model files
|
||||
| +-- wifi-densepose-v1.rvf # Trained model container
|
||||
| +-- wifi-densepose-field-v2.rvf # Environment-calibrated model
|
||||
+-- recordings/ # CSI recording sessions
|
||||
+-- walking-20260303_140000.csi.jsonl # Raw CSI frames (JSONL)
|
||||
+-- walking-20260303_140000.csi.meta.json # Session metadata
|
||||
+-- standing-20260303_141500.csi.jsonl
|
||||
+-- standing-20260303_141500.csi.meta.json
|
||||
|
||||
crates/wifi-densepose-sensing-server/
|
||||
+-- src/
|
||||
+-- main.rs # Server entry, CLI args, AppStateInner, sensing loop
|
||||
+-- model_manager.rs # Model Management bounded context
|
||||
+-- recording.rs # CSI Recording bounded context
|
||||
+-- training_api.rs # Training Pipeline bounded context
|
||||
+-- rvf_container.rs # RVF format ACL (RvfReader, RvfBuilder)
|
||||
+-- rvf_pipeline.rs # Progressive loader for model inference
|
||||
+-- vital_signs.rs # Vital sign detection from CSI phase
|
||||
+-- dataset.rs # Dataset loading for training
|
||||
+-- trainer.rs # Core training loop implementation
|
||||
+-- embedding.rs # Contrastive embedding extraction
|
||||
+-- graph_transformer.rs # Graph transformer architecture
|
||||
+-- sona.rs # SONA self-optimizing profile
|
||||
+-- sparse_inference.rs # Sparse inference engine
|
||||
+-- lib.rs # Public module re-exports
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Related
|
||||
|
||||
- [ADR-019: Sensing-Only UI Mode](../adr/ADR-019-sensing-only-ui-mode.md) -- Decoupled sensing UI, Gaussian splats, Python WebSocket bridge
|
||||
- [ADR-035: Live Sensing UI Accuracy](../adr/ADR-035-live-sensing-ui-accuracy.md) -- Data transparency, Goertzel breathing estimation, signal-responsive pose
|
||||
- [ADR-043: Sensing Server UI API Completion](../adr/ADR-043-sensing-server-ui-api-completion.md) -- Model, recording, training endpoints; single-binary deployment
|
||||
- [RuvSense Domain Model](ruvsense-domain-model.md) -- Upstream signal processing domain (multistatic sensing, coherence, tracking)
|
||||
- [WiFi-Mat Domain Model](wifi-mat-domain-model.md) -- Downstream disaster response domain
|
||||
@@ -0,0 +1,663 @@
|
||||
# Signal Processing Domain Model
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
Based on ADR-014 (SOTA Signal Processing) and the `wifi-densepose-signal` crate.
|
||||
|
||||
### Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **CsiFrame** | A single CSI measurement: amplitude + phase per antenna per subcarrier at one timestamp |
|
||||
| **Conjugate Multiplication** | `H_ref[k] * conj(H_target[k])` — cancels CFO/SFO/PDD, isolating environment-induced phase |
|
||||
| **CSI Ratio** | The complex result of conjugate multiplication between two antenna streams |
|
||||
| **Hampel Filter** | Running median +/- scaled MAD outlier detector; resists up to 50% contamination |
|
||||
| **Phase Sanitization** | Pipeline of unwrapping, outlier removal, smoothing, and noise filtering on raw CSI phase |
|
||||
| **Spectrogram** | 2D time-frequency matrix from STFT, standard CNN input for WiFi activity recognition |
|
||||
| **Subcarrier Sensitivity** | Variance ratio (motion var / static var) ranking how responsive a subcarrier is to motion |
|
||||
| **Body Velocity Profile (BVP)** | Doppler-derived velocity x time 2D matrix; domain-independent motion representation |
|
||||
| **Fresnel Zone** | Ellipsoidal region between TX and RX where signal reflection/diffraction occurs |
|
||||
| **Breathing Estimate** | BPM + amplitude + confidence derived from Fresnel zone boundary crossings |
|
||||
| **Motion Score** | Composite (0.0-1.0) from variance, correlation, phase, and optional Doppler components |
|
||||
| **Presence State** | Binary detection result: human present/absent with smoothed confidence |
|
||||
| **Calibration** | Recording baseline variance during a known-empty period for adaptive detection |
|
||||
|
||||
---
|
||||
|
||||
## Bounded Contexts
|
||||
|
||||
### 1. CSI Preprocessing Context
|
||||
|
||||
**Responsibility**: Produce clean, hardware-artifact-free CSI data from raw measurements.
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------+
|
||||
| CSI Preprocessing Context |
|
||||
+-----------------------------------------------------------+
|
||||
| |
|
||||
| +--------------+ +--------------+ +------------+ |
|
||||
| | Conjugate | | Hampel | | Phase | |
|
||||
| | Multiplication| | Filter | | Sanitizer | |
|
||||
| +------+-------+ +------+-------+ +-----+------+ |
|
||||
| | | | |
|
||||
| v v v |
|
||||
| +------+-------+ +------+-------+ +-----+------+ |
|
||||
| | CsiRatio | | HampelResult | | Sanitized | |
|
||||
| | (clean phase)| |(outlier-free)| | Phase | |
|
||||
| +--------------+ +--------------+ +------------+ |
|
||||
| | | | |
|
||||
| +-------------------+------------------+ |
|
||||
| | |
|
||||
| v |
|
||||
| +-------+--------+ |
|
||||
| | CsiProcessor |--> CleanedCsiData |
|
||||
| +----------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates**: `CsiProcessor` (Aggregate Root)
|
||||
|
||||
**Value Objects**: `CsiData`, `CsiRatio`, `HampelResult`, `HampelConfig`, `PhaseSanitizerConfig`
|
||||
|
||||
**Domain Services**: `CsiPreprocessor`, `PhaseSanitizer`
|
||||
|
||||
---
|
||||
|
||||
### 2. Feature Extraction Context
|
||||
|
||||
**Responsibility**: Transform clean CSI data into ML-ready feature representations.
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------+
|
||||
| Feature Extraction Context |
|
||||
+-----------------------------------------------------------+
|
||||
| |
|
||||
| +--------------+ +--------------+ +------------+ |
|
||||
| | STFT | | Subcarrier | | Doppler | |
|
||||
| | Spectrogram | | Selection | | BVP Engine | |
|
||||
| +------+-------+ +------+-------+ +-----+------+ |
|
||||
| | | | |
|
||||
| v v v |
|
||||
| +------+-------+ +------+-------+ +-----+------+ |
|
||||
| | Spectrogram | | Subcarrier | | BodyVel | |
|
||||
| | (2D TF) | | Selection | | Profile | |
|
||||
| +--------------+ +--------------+ +------------+ |
|
||||
| | | | |
|
||||
| +-------------------+------------------+ |
|
||||
| | |
|
||||
| v |
|
||||
| +----------+----------+ |
|
||||
| | FeatureExtractor |--> CsiFeatures |
|
||||
| +---------------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates**: `FeatureExtractor` (Aggregate Root)
|
||||
|
||||
**Value Objects**: `Spectrogram`, `SubcarrierSelection`, `BodyVelocityProfile`, `CsiFeatures`
|
||||
|
||||
**Domain Services**: `SpectrogramConfig`, `SubcarrierSelectionConfig`, `BvpConfig`
|
||||
|
||||
---
|
||||
|
||||
### 3. Motion Analysis Context
|
||||
|
||||
**Responsibility**: Detect and classify human motion and vital signs from CSI features.
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------+
|
||||
| Motion Analysis Context |
|
||||
+-----------------------------------------------------------+
|
||||
| |
|
||||
| +--------------+ +--------------+ |
|
||||
| | Motion | | Fresnel | |
|
||||
| | Detector | | Breathing | |
|
||||
| +------+-------+ +------+-------+ |
|
||||
| | | |
|
||||
| v v |
|
||||
| +------+-------+ +------+-------+ |
|
||||
| | MotionScore | | Breathing | |
|
||||
| |+ Detection | | Estimate | |
|
||||
| +--------------+ +--------------+ |
|
||||
| | | |
|
||||
| +-------------------+ |
|
||||
| | |
|
||||
| v |
|
||||
| +--------+--------+ |
|
||||
| | HumanDetection |--> PresenceState |
|
||||
| | Result | |
|
||||
| +-----------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Aggregates**: `MotionDetector` (Aggregate Root)
|
||||
|
||||
**Value Objects**: `MotionScore`, `MotionAnalysis`, `HumanDetectionResult`, `BreathingEstimate`, `FresnelGeometry`
|
||||
|
||||
**Domain Services**: `FresnelBreathingEstimator`
|
||||
|
||||
---
|
||||
|
||||
## Aggregates
|
||||
|
||||
### CsiProcessor (CSI Preprocessing Root)
|
||||
|
||||
```rust
|
||||
pub struct CsiProcessor {
|
||||
config: CsiProcessorConfig,
|
||||
preprocessor: CsiPreprocessor,
|
||||
history: VecDeque<CsiData>,
|
||||
previous_detection_confidence: f64,
|
||||
statistics: ProcessingStatistics,
|
||||
}
|
||||
|
||||
impl CsiProcessor {
|
||||
/// Create with validated configuration
|
||||
pub fn new(config: CsiProcessorConfig) -> Result<Self, CsiProcessorError>;
|
||||
|
||||
/// Full preprocessing pipeline: noise removal -> windowing -> normalization
|
||||
pub fn preprocess(&self, csi_data: &CsiData) -> Result<CsiData, CsiProcessorError>;
|
||||
|
||||
/// Maintain temporal history for downstream feature extraction
|
||||
pub fn add_to_history(&mut self, csi_data: CsiData);
|
||||
|
||||
/// Apply exponential moving average to detection confidence
|
||||
pub fn apply_temporal_smoothing(&mut self, raw_confidence: f64) -> f64;
|
||||
}
|
||||
```
|
||||
|
||||
### FeatureExtractor (Feature Extraction Root)
|
||||
|
||||
```rust
|
||||
pub struct FeatureExtractor {
|
||||
config: FeatureExtractorConfig,
|
||||
}
|
||||
|
||||
impl FeatureExtractor {
|
||||
/// Extract all feature types from a single CsiData snapshot
|
||||
pub fn extract(&self, csi_data: &CsiData) -> CsiFeatures;
|
||||
}
|
||||
```
|
||||
|
||||
### MotionDetector (Motion Analysis Root)
|
||||
|
||||
```rust
|
||||
pub struct MotionDetector {
|
||||
config: MotionDetectorConfig,
|
||||
previous_confidence: f64,
|
||||
motion_history: VecDeque<MotionScore>,
|
||||
baseline_variance: Option<f64>,
|
||||
}
|
||||
|
||||
impl MotionDetector {
|
||||
/// Analyze motion from extracted features
|
||||
pub fn analyze_motion(&self, features: &CsiFeatures) -> MotionAnalysis;
|
||||
|
||||
/// Full detection pipeline: analyze -> score -> smooth -> threshold
|
||||
pub fn detect_human(&mut self, features: &CsiFeatures) -> HumanDetectionResult;
|
||||
|
||||
/// Record baseline variance for adaptive detection
|
||||
pub fn calibrate(&mut self, features: &CsiFeatures);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Value Objects
|
||||
|
||||
### CsiData
|
||||
|
||||
```rust
|
||||
pub struct CsiData {
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub amplitude: Array2<f64>, // (num_antennas x num_subcarriers)
|
||||
pub phase: Array2<f64>, // (num_antennas x num_subcarriers), radians
|
||||
pub frequency: f64, // center frequency in Hz
|
||||
pub bandwidth: f64, // bandwidth in Hz
|
||||
pub num_subcarriers: usize,
|
||||
pub num_antennas: usize,
|
||||
pub snr: f64, // signal-to-noise ratio in dB
|
||||
pub metadata: CsiMetadata,
|
||||
}
|
||||
```
|
||||
|
||||
### Spectrogram
|
||||
|
||||
```rust
|
||||
pub struct Spectrogram {
|
||||
pub data: Array2<f64>, // (n_freq x n_time) power/magnitude
|
||||
pub n_freq: usize, // frequency bins (window_size/2 + 1)
|
||||
pub n_time: usize, // time frames
|
||||
pub freq_resolution: f64, // Hz per bin
|
||||
pub time_resolution: f64, // seconds per frame
|
||||
}
|
||||
```
|
||||
|
||||
### SubcarrierSelection
|
||||
|
||||
```rust
|
||||
pub struct SubcarrierSelection {
|
||||
pub selected_indices: Vec<usize>, // ranked by sensitivity, descending
|
||||
pub sensitivity_scores: Vec<f64>, // variance ratio for ALL subcarriers
|
||||
pub selected_data: Option<Array2<f64>>, // filtered matrix (optional)
|
||||
}
|
||||
```
|
||||
|
||||
### BodyVelocityProfile
|
||||
|
||||
```rust
|
||||
pub struct BodyVelocityProfile {
|
||||
pub data: Array2<f64>, // (n_velocity_bins x n_time_frames)
|
||||
pub velocity_bins: Vec<f64>, // velocity value for each row (m/s)
|
||||
pub n_time: usize,
|
||||
pub time_resolution: f64, // seconds per frame
|
||||
pub velocity_resolution: f64, // m/s per bin
|
||||
}
|
||||
```
|
||||
|
||||
### BreathingEstimate
|
||||
|
||||
```rust
|
||||
pub struct BreathingEstimate {
|
||||
pub rate_bpm: f64, // breaths per minute
|
||||
pub confidence: f64, // combined confidence (0.0-1.0)
|
||||
pub period_seconds: f64, // estimated breathing period
|
||||
pub autocorrelation_peak: f64, // periodicity quality
|
||||
pub fresnel_confidence: f64, // Fresnel model match
|
||||
pub amplitude_variation: f64, // observed amplitude variation
|
||||
}
|
||||
```
|
||||
|
||||
### MotionScore
|
||||
|
||||
```rust
|
||||
pub struct MotionScore {
|
||||
pub total: f64, // weighted composite (0.0-1.0)
|
||||
pub variance_component: f64,
|
||||
pub correlation_component: f64,
|
||||
pub phase_component: f64,
|
||||
pub doppler_component: Option<f64>,
|
||||
}
|
||||
```
|
||||
|
||||
### HampelResult
|
||||
|
||||
```rust
|
||||
pub struct HampelResult {
|
||||
pub filtered: Vec<f64>, // outliers replaced with local median
|
||||
pub outlier_indices: Vec<usize>,
|
||||
pub medians: Vec<f64>, // local median at each sample
|
||||
pub sigma_estimates: Vec<f64>, // estimated local sigma at each sample
|
||||
}
|
||||
```
|
||||
|
||||
### FresnelGeometry
|
||||
|
||||
```rust
|
||||
pub struct FresnelGeometry {
|
||||
pub d_tx_body: f64, // TX to body distance (meters)
|
||||
pub d_body_rx: f64, // body to RX distance (meters)
|
||||
pub frequency: f64, // carrier frequency (Hz)
|
||||
}
|
||||
|
||||
impl FresnelGeometry {
|
||||
pub fn wavelength(&self) -> f64;
|
||||
pub fn fresnel_radius(&self, n: u32) -> f64;
|
||||
pub fn phase_change(&self, displacement_m: f64) -> f64;
|
||||
pub fn expected_amplitude_variation(&self, displacement_m: f64) -> f64;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Domain Events
|
||||
|
||||
### Preprocessing Events
|
||||
|
||||
```rust
|
||||
pub enum PreprocessingEvent {
|
||||
/// Raw CSI frame cleaned through the full pipeline
|
||||
FrameCleaned {
|
||||
timestamp: DateTime<Utc>,
|
||||
num_antennas: usize,
|
||||
num_subcarriers: usize,
|
||||
noise_filtered: bool,
|
||||
windowed: bool,
|
||||
normalized: bool,
|
||||
},
|
||||
|
||||
/// Outliers detected and replaced by Hampel filter
|
||||
OutliersDetected {
|
||||
subcarrier_indices: Vec<usize>,
|
||||
replacement_values: Vec<f64>,
|
||||
contamination_ratio: f64,
|
||||
},
|
||||
|
||||
/// Phase sanitization completed
|
||||
PhaseSanitized {
|
||||
method: UnwrappingMethod,
|
||||
outliers_removed: usize,
|
||||
smoothing_applied: bool,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Feature Extraction Events
|
||||
|
||||
```rust
|
||||
pub enum FeatureExtractionEvent {
|
||||
/// Spectrogram computed from temporal CSI stream
|
||||
SpectrogramGenerated {
|
||||
n_time: usize,
|
||||
n_freq: usize,
|
||||
window_size: usize,
|
||||
window_fn: WindowFunction,
|
||||
},
|
||||
|
||||
/// Top-K sensitive subcarriers selected
|
||||
SubcarriersSelected {
|
||||
top_k_indices: Vec<usize>,
|
||||
sensitivity_scores: Vec<f64>,
|
||||
min_sensitivity_threshold: f64,
|
||||
},
|
||||
|
||||
/// Body Velocity Profile extracted
|
||||
BvpExtracted {
|
||||
n_velocity_bins: usize,
|
||||
n_time_frames: usize,
|
||||
max_velocity: f64,
|
||||
carrier_frequency: f64,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Motion Analysis Events
|
||||
|
||||
```rust
|
||||
pub enum MotionAnalysisEvent {
|
||||
/// Human motion detected above threshold
|
||||
MotionDetected {
|
||||
score: MotionScore,
|
||||
confidence: f64,
|
||||
threshold: f64,
|
||||
timestamp: DateTime<Utc>,
|
||||
},
|
||||
|
||||
/// Breathing detected via Fresnel zone model
|
||||
BreathingDetected {
|
||||
rate_bpm: f64,
|
||||
amplitude_variation: f64,
|
||||
fresnel_confidence: f64,
|
||||
autocorrelation_peak: f64,
|
||||
},
|
||||
|
||||
/// Presence state changed (entered or left)
|
||||
PresenceChanged {
|
||||
previous: bool,
|
||||
current: bool,
|
||||
smoothed_confidence: f64,
|
||||
timestamp: DateTime<Utc>,
|
||||
},
|
||||
|
||||
/// Detector calibrated with baseline variance
|
||||
BaselineCalibrated {
|
||||
baseline_variance: f64,
|
||||
timestamp: DateTime<Utc>,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Invariants
|
||||
|
||||
### CSI Preprocessing Invariants
|
||||
|
||||
1. **Conjugate multiplication requires >= 2 antenna elements.** `compute_ratio_matrix` returns `CsiRatioError::InsufficientAntennas` if `n_ant < 2`. Without two antennas, there is no pair to cancel common-mode offsets.
|
||||
|
||||
2. **Hampel filter window must be >= 1 (half_window > 0).** A zero-width window cannot compute a local median. Enforced by `HampelError::InvalidWindow`.
|
||||
|
||||
3. **Phase data must be within configured range before sanitization.** Default range is `[-pi, pi]`. Enforced by `PhaseSanitizer::validate_phase_data`.
|
||||
|
||||
4. **Antenna stream lengths must match for conjugate multiplication.** `conjugate_multiply` returns `CsiRatioError::LengthMismatch` if `h_ref.len() != h_target.len()`.
|
||||
|
||||
### Feature Extraction Invariants
|
||||
|
||||
5. **Spectrogram window size must be > 0 and signal must be >= window_size samples.** Enforced by `SpectrogramError::SignalTooShort` and `SpectrogramError::InvalidWindowSize`.
|
||||
|
||||
6. **Subcarrier selection must receive matching subcarrier counts.** Motion and static data must have the same number of columns. Enforced by `SelectionError::SubcarrierCountMismatch`.
|
||||
|
||||
7. **BVP requires >= window_size temporal samples.** Insufficient history prevents STFT computation. Enforced by `BvpError::InsufficientSamples`.
|
||||
|
||||
8. **BVP carrier frequency must be > 0 for wavelength calculation.** Zero frequency would produce a division-by-zero in the Doppler-to-velocity mapping.
|
||||
|
||||
### Motion Analysis Invariants
|
||||
|
||||
9. **Fresnel geometry requires positive distances (d_tx_body > 0, d_body_rx > 0).** Zero or negative distances are physically impossible. Enforced by `FresnelError::InvalidDistance`.
|
||||
|
||||
10. **Fresnel frequency must be positive.** Required for wavelength computation. Enforced by `FresnelError::InvalidFrequency`.
|
||||
|
||||
11. **Breathing estimation requires >= 10 amplitude samples.** Fewer samples cannot support autocorrelation analysis. Enforced by `FresnelError::InsufficientData`.
|
||||
|
||||
12. **Motion detector history does not exceed configured max size.** Oldest entries are evicted via `VecDeque::pop_front` when capacity is reached.
|
||||
|
||||
---
|
||||
|
||||
## Domain Services
|
||||
|
||||
### CsiPreprocessor
|
||||
|
||||
Orchestrates the cleaning pipeline for a single CSI frame.
|
||||
|
||||
```rust
|
||||
pub struct CsiPreprocessor {
|
||||
noise_threshold: f64,
|
||||
}
|
||||
|
||||
impl CsiPreprocessor {
|
||||
/// Remove subcarriers below noise floor (amplitude in dB < threshold)
|
||||
pub fn remove_noise(&self, csi_data: &CsiData) -> Result<CsiData, CsiProcessorError>;
|
||||
|
||||
/// Apply Hamming window to reduce spectral leakage
|
||||
pub fn apply_windowing(&self, csi_data: &CsiData) -> Result<CsiData, CsiProcessorError>;
|
||||
|
||||
/// Normalize amplitude to unit variance
|
||||
pub fn normalize_amplitude(&self, csi_data: &CsiData) -> Result<CsiData, CsiProcessorError>;
|
||||
}
|
||||
```
|
||||
|
||||
### PhaseSanitizer
|
||||
|
||||
Full phase cleaning pipeline: unwrap -> outlier removal -> smoothing -> noise filtering.
|
||||
|
||||
```rust
|
||||
pub struct PhaseSanitizer {
|
||||
config: PhaseSanitizerConfig,
|
||||
statistics: SanitizationStatistics,
|
||||
}
|
||||
|
||||
impl PhaseSanitizer {
|
||||
/// Complete sanitization pipeline (all four stages)
|
||||
pub fn sanitize_phase(
|
||||
&mut self,
|
||||
phase_data: &Array2<f64>,
|
||||
) -> Result<Array2<f64>, PhaseSanitizationError>;
|
||||
}
|
||||
```
|
||||
|
||||
### FresnelBreathingEstimator
|
||||
|
||||
Physics-based breathing detection using Fresnel zone geometry.
|
||||
|
||||
```rust
|
||||
pub struct FresnelBreathingEstimator {
|
||||
geometry: FresnelGeometry,
|
||||
min_displacement: f64, // 3mm default
|
||||
max_displacement: f64, // 15mm default
|
||||
}
|
||||
|
||||
impl FresnelBreathingEstimator {
|
||||
/// Check if amplitude variation matches Fresnel breathing model
|
||||
pub fn breathing_confidence(&self, observed_amplitude_variation: f64) -> f64;
|
||||
|
||||
/// Estimate breathing rate via autocorrelation + Fresnel validation
|
||||
pub fn estimate_breathing_rate(
|
||||
&self,
|
||||
amplitude_signal: &[f64],
|
||||
sample_rate: f64,
|
||||
) -> Result<BreathingEstimate, FresnelError>;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Context Map
|
||||
|
||||
```
|
||||
+--------------------------------------------------------------+
|
||||
| Signal Processing System |
|
||||
+--------------------------------------------------------------+
|
||||
| |
|
||||
| +----------------+ Published +------------------+ |
|
||||
| | CSI | Language | Feature | |
|
||||
| | Preprocessing |------------>| Extraction | |
|
||||
| | Context | CsiData | Context | |
|
||||
| +-------+--------+ +--------+---------+ |
|
||||
| | | |
|
||||
| | Publishes | Publishes |
|
||||
| | CleanedCsiData | CsiFeatures |
|
||||
| v v |
|
||||
| +-------+-------------------------------+---------+ |
|
||||
| | Event Bus (Domain Events) | |
|
||||
| +---------------------------+---------------------+ |
|
||||
| | |
|
||||
| | Subscribes |
|
||||
| v |
|
||||
| +---------+---------+ |
|
||||
| | Motion | |
|
||||
| | Analysis | |
|
||||
| | Context | |
|
||||
| +-------------------+ |
|
||||
| |
|
||||
+---------------------------------------------------------------+
|
||||
| DOWNSTREAM (Customer/Supplier) |
|
||||
| +-----------------+ +------------------+ +--------------+ |
|
||||
| | wifi-densepose | | wifi-densepose | |wifi-densepose| |
|
||||
| | -nn | | -mat | | -train | |
|
||||
| | (consumes | | (consumes | |(consumes | |
|
||||
| | CsiFeatures, | | BreathingEst, | | CsiFeatures) | |
|
||||
| | Spectrogram) | | MotionScore) | | | |
|
||||
| +-----------------+ +------------------+ +--------------+ |
|
||||
+---------------------------------------------------------------+
|
||||
| UPSTREAM (Conformist) |
|
||||
| +-----------------+ +------------------+ |
|
||||
| | wifi-densepose | | wifi-densepose | |
|
||||
| | -core | | -hardware | |
|
||||
| | (CsiFrame | | (ESP32 raw CSI | |
|
||||
| | primitives) | | data ingestion) | |
|
||||
| +-----------------+ +------------------+ |
|
||||
+---------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Relationship Types**:
|
||||
- Preprocessing -> Feature Extraction: **Published Language** (CsiData is the shared contract)
|
||||
- Preprocessing -> Motion Analysis: **Customer/Supplier** (Preprocessing supplies cleaned data)
|
||||
- Feature Extraction -> Motion Analysis: **Customer/Supplier** (Features supplies CsiFeatures)
|
||||
- Signal -> wifi-densepose-nn: **Customer/Supplier** (Signal publishes Spectrogram, BVP)
|
||||
- Signal -> wifi-densepose-mat: **Customer/Supplier** (Signal publishes BreathingEstimate, MotionScore)
|
||||
- Signal <- wifi-densepose-core: **Conformist** (Signal adapts to core CsiFrame types)
|
||||
- Signal <- wifi-densepose-hardware: **Conformist** (Signal adapts to raw ESP32 CSI format)
|
||||
|
||||
---
|
||||
|
||||
## Anti-Corruption Layers
|
||||
|
||||
### Hardware ACL (Upstream)
|
||||
|
||||
Translates raw ESP32 CSI packets into the signal crate's `CsiData` value object, normalizing hardware-specific quirks (LLTF/HT-LTF format differences, antenna mapping, null subcarrier handling).
|
||||
|
||||
```rust
|
||||
/// Normalizes vendor-specific CSI frames to canonical CsiData
|
||||
pub struct HardwareNormalizer {
|
||||
hardware_type: HardwareType,
|
||||
}
|
||||
|
||||
impl HardwareNormalizer {
|
||||
/// Convert raw hardware bytes to canonical CsiData
|
||||
pub fn normalize(
|
||||
&self,
|
||||
raw_csi: &[u8],
|
||||
hardware_type: HardwareType,
|
||||
) -> Result<CanonicalCsiFrame, HardwareNormError>;
|
||||
}
|
||||
|
||||
pub enum HardwareType {
|
||||
Esp32S3,
|
||||
Intel5300,
|
||||
AtherosAr9580,
|
||||
Simulation,
|
||||
}
|
||||
```
|
||||
|
||||
### Neural Network ACL (Downstream)
|
||||
|
||||
Adapts signal processing outputs (Spectrogram, BVP, CsiFeatures) into tensor formats expected by the `wifi-densepose-nn` crate. This boundary prevents neural network model details from leaking into the signal processing domain.
|
||||
|
||||
```rust
|
||||
/// Adapts signal crate types to neural network tensor format
|
||||
pub struct SignalToTensorAdapter;
|
||||
|
||||
impl SignalToTensorAdapter {
|
||||
/// Convert Spectrogram to CNN-ready 2D tensor
|
||||
pub fn spectrogram_to_tensor(spec: &Spectrogram) -> Array2<f32> {
|
||||
spec.data.mapv(|v| v as f32)
|
||||
}
|
||||
|
||||
/// Convert BVP to domain-independent velocity tensor
|
||||
pub fn bvp_to_tensor(bvp: &BodyVelocityProfile) -> Array2<f32> {
|
||||
bvp.data.mapv(|v| v as f32)
|
||||
}
|
||||
|
||||
/// Convert selected subcarrier data to reduced-dimension input
|
||||
pub fn selected_csi_to_tensor(
|
||||
selection: &SubcarrierSelection,
|
||||
data: &Array2<f64>,
|
||||
) -> Result<Array2<f32>, SelectionError> {
|
||||
let extracted = extract_selected(data, selection)?;
|
||||
Ok(extracted.mapv(|v| v as f32))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### MAT ACL (Downstream)
|
||||
|
||||
Adapts motion analysis outputs for the Mass Casualty Assessment Tool, translating domain-generic motion scores and breathing estimates into disaster-context vital signs.
|
||||
|
||||
```rust
|
||||
/// Adapts signal processing outputs for disaster assessment
|
||||
pub struct SignalToMatAdapter;
|
||||
|
||||
impl SignalToMatAdapter {
|
||||
/// Convert BreathingEstimate to MAT-domain BreathingPattern
|
||||
pub fn to_breathing_pattern(est: &BreathingEstimate) -> BreathingPattern {
|
||||
BreathingPattern {
|
||||
rate_bpm: est.rate_bpm as f32,
|
||||
amplitude: est.amplitude_variation as f32,
|
||||
regularity: est.autocorrelation_peak as f32,
|
||||
pattern_type: classify_breathing_type(est.rate_bpm),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert MotionScore to MAT-domain presence indicator
|
||||
pub fn to_presence_indicator(score: &MotionScore) -> PresenceIndicator {
|
||||
PresenceIndicator {
|
||||
detected: score.total > 0.3,
|
||||
confidence: score.total,
|
||||
motion_level: classify_motion_level(score),
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,147 @@
|
||||
# Edge Intelligence Modules — WiFi-DensePose
|
||||
|
||||
> 60 WASM modules that run directly on an ESP32 sensor. No internet needed, no cloud fees, instant response. Each module is a tiny file (5-30 KB) that reads WiFi signal data and makes decisions locally in under 10 ms.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Build all modules for ESP32
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm-edge
|
||||
cargo build --target wasm32-unknown-unknown --release
|
||||
|
||||
# Run all 632 tests
|
||||
cargo test --features std
|
||||
|
||||
# Upload a module to your ESP32
|
||||
python scripts/wasm_upload.py --port COM7 --module target/wasm32-unknown-unknown/release/module_name.wasm
|
||||
```
|
||||
|
||||
## Module Categories
|
||||
|
||||
| | Category | Modules | Tests | Documentation |
|
||||
|---|----------|---------|-------|---------------|
|
||||
| | **Core** | 7 | 81 | [core.md](core.md) |
|
||||
| | **Medical & Health** | 5 | 38 | [medical.md](medical.md) |
|
||||
| | **Security & Safety** | 6 | 42 | [security.md](security.md) |
|
||||
| | **Smart Building** | 5 | 38 | [building.md](building.md) |
|
||||
| | **Retail & Hospitality** | 5 | 38 | [retail.md](retail.md) |
|
||||
| | **Industrial** | 5 | 38 | [industrial.md](industrial.md) |
|
||||
| | **Exotic & Research** | 10 | ~60 | [exotic.md](exotic.md) |
|
||||
| | **Signal Intelligence** | 6 | 54 | [signal-intelligence.md](signal-intelligence.md) |
|
||||
| | **Adaptive Learning** | 4 | 42 | [adaptive-learning.md](adaptive-learning.md) |
|
||||
| | **Spatial & Temporal** | 6 | 56 | [spatial-temporal.md](spatial-temporal.md) |
|
||||
| | **AI Security** | 2 | 20 | [ai-security.md](ai-security.md) |
|
||||
| | **Quantum & Autonomous** | 4 | 30 | [autonomous.md](autonomous.md) |
|
||||
| | **Total** | **65** | **632** | |
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **WiFi signals bounce off people and objects** in a room, creating a unique pattern
|
||||
2. **The ESP32 chip reads these patterns** as Channel State Information (CSI) — 52 numbers that describe how each WiFi channel changed
|
||||
3. **WASM modules analyze the patterns** to detect specific things: someone fell, a room is occupied, breathing rate changed
|
||||
4. **Events are emitted locally** — no cloud round-trip, response time under 10 ms
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
WiFi Router ──── radio waves ────→ ESP32-S3 Sensor
|
||||
│
|
||||
▼
|
||||
┌──────────────┐
|
||||
│ Tier 0-2 │ C firmware: phase unwrap,
|
||||
│ DSP Engine │ stats, top-K selection
|
||||
└──────┬───────┘
|
||||
│ CSI frame (52 subcarriers)
|
||||
▼
|
||||
┌──────────────┐
|
||||
│ WASM3 │ Tiny interpreter
|
||||
│ Runtime │ (60 KB overhead)
|
||||
└──────┬───────┘
|
||||
│
|
||||
┌───────────┼───────────┐
|
||||
▼ ▼ ▼
|
||||
┌──────────┐ ┌──────────┐ ┌──────────┐
|
||||
│ Module A │ │ Module B │ │ Module C │
|
||||
│ (5-30KB) │ │ (5-30KB) │ │ (5-30KB) │
|
||||
└────┬─────┘ └────┬─────┘ └────┬─────┘
|
||||
│ │ │
|
||||
└───────────┼───────────┘
|
||||
▼
|
||||
Events + Alerts
|
||||
(UDP to aggregator or local)
|
||||
```
|
||||
|
||||
## Host API
|
||||
|
||||
Every module talks to the ESP32 through 12 functions:
|
||||
|
||||
| Function | Returns | Description |
|
||||
|----------|---------|-------------|
|
||||
| `csi_get_phase(i)` | `f32` | WiFi signal phase angle for subcarrier `i` |
|
||||
| `csi_get_amplitude(i)` | `f32` | Signal strength for subcarrier `i` |
|
||||
| `csi_get_variance(i)` | `f32` | How much subcarrier `i` fluctuates |
|
||||
| `csi_get_bpm_breathing()` | `f32` | Breathing rate (BPM) |
|
||||
| `csi_get_bpm_heartrate()` | `f32` | Heart rate (BPM) |
|
||||
| `csi_get_presence()` | `i32` | Is anyone there? (0/1) |
|
||||
| `csi_get_motion_energy()` | `f32` | Overall movement level |
|
||||
| `csi_get_n_persons()` | `i32` | Estimated number of people |
|
||||
| `csi_get_timestamp()` | `i32` | Current timestamp (ms) |
|
||||
| `csi_emit_event(id, val)` | — | Send a detection result to the host |
|
||||
| `csi_log(ptr, len)` | — | Log a message to serial console |
|
||||
| `csi_get_phase_history(buf, max)` | `i32` | Past phase values for trend analysis |
|
||||
|
||||
## Event ID Registry
|
||||
|
||||
| Range | Category | Example Events |
|
||||
|-------|----------|---------------|
|
||||
| 0-99 | Core | Gesture detected, coherence score, anomaly |
|
||||
| 100-199 | Medical | Apnea, bradycardia, tachycardia, seizure |
|
||||
| 200-299 | Security | Intrusion, perimeter breach, loitering, panic |
|
||||
| 300-399 | Smart Building | Zone occupied, HVAC, lighting, elevator, meeting |
|
||||
| 400-499 | Retail | Queue length, dwell zone, customer flow, turnover |
|
||||
| 500-599 | Industrial | Proximity warning, confined space, vibration |
|
||||
| 600-699 | Exotic | Sleep stage, emotion, gesture language, rain |
|
||||
| 700-729 | Signal Intelligence | Attention, coherence gate, compression, recovery |
|
||||
| 730-759 | Adaptive Learning | Gesture learned, attractor, adaptation, EWC |
|
||||
| 760-789 | Spatial Reasoning | Influence, HNSW match, spike tracking |
|
||||
| 790-819 | Temporal Analysis | Pattern, LTL violation, GOAP goal |
|
||||
| 820-849 | AI Security | Replay attack, injection, jamming, behavior |
|
||||
| 850-879 | Quantum-Inspired | Entanglement, decoherence, hypothesis |
|
||||
| 880-899 | Autonomous | Inference, rule fired, mesh reconfigure |
|
||||
|
||||
## Module Development
|
||||
|
||||
### Adding a New Module
|
||||
|
||||
1. Create `src/your_module.rs` following the pattern:
|
||||
```rust
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
#[cfg(not(feature = "std"))]
|
||||
use libm::fabsf;
|
||||
|
||||
pub struct YourModule { /* fixed-size fields only */ }
|
||||
|
||||
impl YourModule {
|
||||
pub const fn new() -> Self { /* ... */ }
|
||||
pub fn process_frame(&mut self, /* inputs */) -> &[(i32, f32)] { /* ... */ }
|
||||
}
|
||||
```
|
||||
|
||||
2. Add `pub mod your_module;` to `lib.rs`
|
||||
3. Add event constants to `event_types` in `lib.rs`
|
||||
4. Add tests with `#[cfg(test)] mod tests { ... }`
|
||||
5. Run `cargo test --features std`
|
||||
|
||||
### Constraints
|
||||
|
||||
- **No heap allocation**: Use fixed-size arrays, not `Vec` or `String`
|
||||
- **No `std`**: Use `libm` for math functions
|
||||
- **Budget tiers**: L (<2ms), S (<5ms), H (<10ms) per frame
|
||||
- **Binary size**: Each module should be 5-30 KB as WASM
|
||||
|
||||
## References
|
||||
|
||||
- [ADR-039](../adr/ADR-039-esp32-edge-intelligence.md) — Edge processing tiers
|
||||
- [ADR-040](../adr/ADR-040-wasm-programmable-sensing.md) — WASM runtime design
|
||||
- [ADR-041](../adr/ADR-041-wasm-module-collection.md) — Full module specification
|
||||
- [Source code](../../rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm-edge/src/)
|
||||
@@ -0,0 +1,425 @@
|
||||
# Adaptive Learning Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> On-device machine learning that runs without cloud connectivity. The ESP32 chip teaches itself what "normal" looks like for each environment and adapts over time. No training data needed -- it learns from what it sees.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|-------------|-----------|--------|
|
||||
| DTW Gesture Learn | `lrn_dtw_gesture_learn.rs` | Teaches custom gestures via 3 rehearsals | 730-733 | H (<10ms) |
|
||||
| Anomaly Attractor | `lrn_anomaly_attractor.rs` | Models room dynamics as a chaotic attractor | 735-738 | S (<5ms) |
|
||||
| Meta Adapt | `lrn_meta_adapt.rs` | Self-tunes 8 detection thresholds via hill climbing | 740-743 | S (<5ms) |
|
||||
| EWC Lifelong | `lrn_ewc_lifelong.rs` | Learns new environments without forgetting old ones | 745-748 | L (<2ms) |
|
||||
|
||||
## How the Learning Modules Work Together
|
||||
|
||||
```
|
||||
Raw CSI data (from signal intelligence pipeline)
|
||||
|
|
||||
v
|
||||
+-------------------------+ +--------------------------+
|
||||
| Anomaly Attractor | | DTW Gesture Learn |
|
||||
| Learn what "normal" | | Users teach custom |
|
||||
| looks like, detect | | gestures by performing |
|
||||
| deviations from it | | them 3 times |
|
||||
+-------------------------+ +--------------------------+
|
||||
| |
|
||||
v v
|
||||
+-------------------------+ +--------------------------+
|
||||
| EWC Lifelong | | Meta Adapt |
|
||||
| Learn new rooms/layouts | | Auto-tune thresholds |
|
||||
| without forgetting | | based on TP/FP feedback |
|
||||
| old ones | | |
|
||||
+-------------------------+ +--------------------------+
|
||||
| |
|
||||
v v
|
||||
Persistent on-device knowledge Optimized detection parameters
|
||||
(survives power cycles via NVS) (fewer false alarms over time)
|
||||
```
|
||||
|
||||
- **Anomaly Attractor** learns the room's "normal" signal dynamics and alerts when something unexpected happens.
|
||||
- **DTW Gesture Learn** lets users define custom gestures without any programming.
|
||||
- **EWC Lifelong** ensures the device can move to a new room and learn it without losing knowledge of previous rooms.
|
||||
- **Meta Adapt** continuously improves detection accuracy by tuning thresholds based on real-world feedback.
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### DTW Gesture Learning (`lrn_dtw_gesture_learn.rs`)
|
||||
|
||||
**What it does**: You teach the device custom gestures by performing them 3 times. It remembers up to 16 different gestures. When it recognizes a gesture you taught it, it fires an event with the gesture ID.
|
||||
|
||||
**Algorithm**: Dynamic Time Warping (DTW) with 3-rehearsal enrollment protocol.
|
||||
|
||||
DTW measures the similarity between two temporal sequences that may vary in speed. Unlike simple correlation, DTW can match a gesture performed slowly against one performed quickly. The Sakoe-Chiba band (width=8) constrains the warping path to prevent pathological matches.
|
||||
|
||||
#### Learning Protocol
|
||||
|
||||
```
|
||||
State Machine:
|
||||
|
||||
Idle ──(60 frames stillness)──> WaitingStill
|
||||
^ |
|
||||
| (motion detected)
|
||||
| v
|
||||
| Recording ──(stillness)──> Captured
|
||||
| |
|
||||
| (save rehearsal)
|
||||
| |
|
||||
| +----- < 3 rehearsals? ──> WaitingStill
|
||||
| |
|
||||
| >= 3 rehearsals
|
||||
| |
|
||||
| (check DTW similarity)
|
||||
| |
|
||||
+-- (all 3 similar?) ──> commit template ──+
|
||||
+-- (too different?) ──> discard & reset ──+
|
||||
```
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct GestureLearner { /* ... */ }
|
||||
|
||||
impl GestureLearner {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, phases: &[f32], motion_energy: f32) -> &[(i32, f32)];
|
||||
pub fn template_count() -> usize; // Number of stored gesture templates (0-16)
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 730 | `GESTURE_LEARNED` | Gesture ID (100+) | A new gesture template was successfully committed |
|
||||
| 731 | `GESTURE_MATCHED` | Gesture ID | A stored gesture was recognized in the current signal |
|
||||
| 732 | `MATCH_DISTANCE` | DTW distance | How closely the input matched the template (lower = better) |
|
||||
| 733 | `TEMPLATE_COUNT` | Count (0-16) | Total number of stored templates |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `TEMPLATE_LEN` | 64 | Maximum samples per gesture template |
|
||||
| `MAX_TEMPLATES` | 16 | Maximum stored gestures |
|
||||
| `REHEARSALS_REQUIRED` | 3 | Times you must perform a gesture to teach it |
|
||||
| `STILLNESS_THRESHOLD` | 0.05 | Motion energy below this = stillness |
|
||||
| `STILLNESS_FRAMES` | 60 | Frames of stillness to enter learning mode (~3s at 20Hz) |
|
||||
| `LEARN_DTW_THRESHOLD` | 3.0 | Max DTW distance between rehearsals to accept as same gesture |
|
||||
| `RECOGNIZE_DTW_THRESHOLD` | 2.5 | Max DTW distance for recognition match |
|
||||
| `MATCH_COOLDOWN` | 40 | Frames between consecutive matches (~2s at 20Hz) |
|
||||
| `BAND_WIDTH` | 8 | Sakoe-Chiba band width for DTW |
|
||||
|
||||
#### Tutorial: Teaching Your ESP32 a Custom Gesture
|
||||
|
||||
**Step 1: Enter training mode.**
|
||||
Stand still for 3 seconds (60 frames at 20 Hz). The device detects sustained stillness and enters `WaitingStill` mode. There is no LED indicator in the base firmware, but you can add one by listening for the state transition.
|
||||
|
||||
**Step 2: Perform the gesture.**
|
||||
Move your hand through the WiFi field. The device records the phase-delta trajectory. The recording captures up to 64 samples (3.2 seconds at 20 Hz). Keep the gesture under 3 seconds.
|
||||
|
||||
**Step 3: Return to stillness.**
|
||||
Stop moving. The device captures the recording as "rehearsal 1 of 3."
|
||||
|
||||
**Step 4: Repeat 2 more times.**
|
||||
The device stays in learning mode. Perform the same gesture two more times, returning to stillness after each.
|
||||
|
||||
**Step 5: Automatic validation.**
|
||||
After the 3rd rehearsal, the device computes pairwise DTW distances between all 3 recordings. If all 3 are mutually similar (DTW distance < 3.0), it averages them into a template and assigns gesture ID 100 (the first custom gesture). Subsequent gestures get IDs 101, 102, etc.
|
||||
|
||||
**Step 6: Recognition.**
|
||||
Once a template is stored, the device continuously matches the incoming phase-delta stream against all stored templates. When a match is found (DTW distance < 2.5), it emits `GESTURE_MATCHED` with the gesture ID and enters a 2-second cooldown to prevent double-firing.
|
||||
|
||||
**Tips for reliable gesture recognition:**
|
||||
- Perform gestures in the same general area of the room
|
||||
- Make gestures distinct (a wave is easier to distinguish from a circle than from a slower wave)
|
||||
- Avoid ambient motion during training (other people walking, fans)
|
||||
- Shorter gestures (0.5-1.5 seconds) tend to be more reliable than long ones
|
||||
|
||||
---
|
||||
|
||||
### Anomaly Attractor (`lrn_anomaly_attractor.rs`)
|
||||
|
||||
**What it does**: Models the room's WiFi signal as a dynamical system and classifies its behavior. An empty room produces a "point attractor" (stable signal). A room with HVAC produces a "limit cycle" (periodic). A room with people produces a "strange attractor" (complex but bounded). When the signal leaves the learned attractor basin, something unusual is happening.
|
||||
|
||||
**Algorithm**: 4D dynamical system analysis with Lyapunov exponent estimation.
|
||||
|
||||
The state vector is: `(mean_phase, mean_amplitude, variance, motion_energy)`
|
||||
|
||||
The Lyapunov exponent quantifies trajectory divergence:
|
||||
```
|
||||
lambda = (1/N) * sum(log(|delta_n+1| / |delta_n|))
|
||||
```
|
||||
- lambda < -0.01: **Point attractor** (stable, empty room)
|
||||
- -0.01 <= lambda < 0.01: **Limit cycle** (periodic, machinery/HVAC)
|
||||
- lambda >= 0.01: **Strange attractor** (chaotic, occupied room)
|
||||
|
||||
After 200 frames of learning (~10 seconds), the attractor type is classified and the basin radius is established. Subsequent departures beyond 3x the basin radius trigger anomaly alerts.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct AttractorDetector { /* ... */ }
|
||||
|
||||
impl AttractorDetector {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, phases: &[f32], amplitudes: &[f32], motion_energy: f32)
|
||||
-> &[(i32, f32)];
|
||||
pub fn lyapunov_exponent() -> f32;
|
||||
pub fn attractor_type() -> AttractorType; // Unknown/PointAttractor/LimitCycle/StrangeAttractor
|
||||
pub fn is_initialized() -> bool; // True after 200 learning frames
|
||||
}
|
||||
|
||||
pub enum AttractorType { Unknown, PointAttractor, LimitCycle, StrangeAttractor }
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 735 | `ATTRACTOR_TYPE` | 1/2/3 | Point(1), LimitCycle(2), Strange(3) -- emitted when classification changes |
|
||||
| 736 | `LYAPUNOV_EXPONENT` | Lambda | Current Lyapunov exponent estimate |
|
||||
| 737 | `BASIN_DEPARTURE` | Distance ratio | Trajectory left the attractor basin (value = distance / radius) |
|
||||
| 738 | `LEARNING_COMPLETE` | 1.0 | Initial 200-frame learning phase finished |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `TRAJ_LEN` | 128 | Trajectory buffer length (circular) |
|
||||
| `STATE_DIM` | 4 | State vector dimensionality |
|
||||
| `MIN_FRAMES_FOR_CLASSIFICATION` | 200 | Learning phase length (~10s at 20Hz) |
|
||||
| `LYAPUNOV_STABLE_UPPER` | -0.01 | Lambda below this = point attractor |
|
||||
| `LYAPUNOV_PERIODIC_UPPER` | 0.01 | Lambda below this = limit cycle |
|
||||
| `BASIN_DEPARTURE_MULT` | 3.0 | Departure threshold (3x learned radius) |
|
||||
| `CENTER_ALPHA` | 0.01 | EMA alpha for attractor center tracking |
|
||||
| `DEPARTURE_COOLDOWN` | 100 | Frames between departure alerts (~5s at 20Hz) |
|
||||
|
||||
#### Tutorial: Understanding Attractor Types
|
||||
|
||||
**Point Attractor (lambda < -0.01)**
|
||||
The signal converges to a fixed point. This means the environment is completely static -- no people, no machinery, no airflow. The WiFi signal is deterministic and unchanging. Any disturbance will trigger a basin departure.
|
||||
|
||||
**Limit Cycle (lambda near 0)**
|
||||
The signal follows a periodic orbit. This typically indicates mechanical systems: HVAC cycling, fans, elevator machinery. The period usually matches the equipment's duty cycle. Human activity on top of a limit cycle will push the Lyapunov exponent positive.
|
||||
|
||||
**Strange Attractor (lambda > 0.01)**
|
||||
The signal is bounded but aperiodic -- classical chaos. This is the signature of human activity: walking, gesturing, breathing all create complex but bounded signal dynamics. The more people, the higher the Lyapunov exponent tends to be.
|
||||
|
||||
**Basin Departure**
|
||||
A basin departure means the current signal state is more than 3x the learned radius away from the attractor center. This can indicate:
|
||||
- Someone new entered the room
|
||||
- A door or window opened
|
||||
- Equipment turned on/off
|
||||
- Environmental change (rain, temperature)
|
||||
|
||||
---
|
||||
|
||||
### Meta Adapt (`lrn_meta_adapt.rs`)
|
||||
|
||||
**What it does**: Automatically tunes 8 detection thresholds to reduce false alarms and improve detection accuracy. Uses real-world feedback (true positives and false positives) to drive a simple hill-climbing optimizer.
|
||||
|
||||
**Algorithm**: Iterative parameter perturbation with safety rollback.
|
||||
|
||||
The optimizer maintains 8 parameters, each with bounds and step sizes:
|
||||
|
||||
| Index | Parameter | Default | Range | Step |
|
||||
|-------|-----------|---------|-------|------|
|
||||
| 0 | Presence threshold | 0.05 | 0.01-0.50 | 0.01 |
|
||||
| 1 | Motion threshold | 0.10 | 0.02-1.00 | 0.02 |
|
||||
| 2 | Coherence threshold | 0.70 | 0.30-0.99 | 0.02 |
|
||||
| 3 | Gesture DTW threshold | 2.50 | 0.50-5.00 | 0.20 |
|
||||
| 4 | Anomaly energy ratio | 50.0 | 10.0-200.0 | 5.0 |
|
||||
| 5 | Zone occupancy threshold | 0.02 | 0.005-0.10 | 0.005 |
|
||||
| 6 | Vital apnea seconds | 20.0 | 10.0-60.0 | 2.0 |
|
||||
| 7 | Intrusion sensitivity | 0.30 | 0.05-0.90 | 0.03 |
|
||||
|
||||
The optimization loop (runs on timer, not per-frame):
|
||||
1. Measure baseline performance score: `score = TP_rate - 2 * FP_rate`
|
||||
2. Perturb one parameter by its step size (alternating +/- direction)
|
||||
3. Wait for `EVAL_WINDOW` (10) timer ticks
|
||||
4. Measure new performance score
|
||||
5. If improved, keep the change. If not, revert.
|
||||
6. After 3 consecutive failures, safety rollback to the last known-good snapshot.
|
||||
7. Sweep through all 8 parameters, then increment the meta-level counter.
|
||||
|
||||
The 2x penalty on false positives reflects the real-world cost: a false alarm (waking someone up at 3 AM because the system thought it detected motion) is worse than occasionally missing a true event.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct MetaAdapter { /* ... */ }
|
||||
|
||||
impl MetaAdapter {
|
||||
pub const fn new() -> Self;
|
||||
pub fn report_true_positive(&mut self); // Confirmed correct detection
|
||||
pub fn report_false_positive(&mut self); // Detection that should not have fired
|
||||
pub fn report_event(&mut self); // Generic event for normalization
|
||||
pub fn get_param(idx: usize) -> f32; // Current value of parameter idx
|
||||
pub fn on_timer() -> &[(i32, f32)]; // Drive optimization loop (call at 1 Hz)
|
||||
pub fn iteration_count() -> u32;
|
||||
pub fn success_count() -> u32;
|
||||
pub fn meta_level() -> u16; // Number of complete sweeps
|
||||
pub fn consecutive_failures() -> u8;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 740 | `PARAM_ADJUSTED` | param_idx + value/1000 | A parameter was successfully tuned |
|
||||
| 741 | `ADAPTATION_SCORE` | Score [-2, 1] | Performance score after successful adaptation |
|
||||
| 742 | `ROLLBACK_TRIGGERED` | Meta level | Safety rollback: 3 consecutive failures, reverting all params |
|
||||
| 743 | `META_LEVEL` | Level | Number of complete optimization sweeps completed |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `NUM_PARAMS` | 8 | Number of tunable parameters |
|
||||
| `MAX_CONSECUTIVE_FAILURES` | 3 | Failures before safety rollback |
|
||||
| `EVAL_WINDOW` | 10 | Timer ticks per evaluation phase |
|
||||
| `DEFAULT_STEP_FRAC` | 0.05 | Step size as fraction of range |
|
||||
|
||||
#### Tutorial: Providing Feedback to Meta Adapt
|
||||
|
||||
The meta adapter needs feedback to know whether its changes helped. In a typical deployment:
|
||||
|
||||
1. **True positives**: When an event (presence detection, gesture match) is confirmed correct by another sensor or user acknowledgment, call `report_true_positive()`.
|
||||
2. **False positives**: When an event fires but nothing actually happened (e.g., presence detected in an empty room), call `report_false_positive()`.
|
||||
3. **Generic events**: Call `report_event()` for all events, regardless of correctness, to normalize the score.
|
||||
|
||||
In autonomous operation without human feedback, you can use cross-validation between modules: if both the coherence gate and the anomaly attractor agree that something happened, treat it as a true positive. If only one fires, it might be a false positive.
|
||||
|
||||
---
|
||||
|
||||
### EWC Lifelong (`lrn_ewc_lifelong.rs`)
|
||||
|
||||
**What it does**: Learns to classify which zone a person is in (up to 4 zones) using WiFi signal features. Critically, when moved to a new environment, it learns the new layout without forgetting previously learned ones. This is the "lifelong learning" property enabled by Elastic Weight Consolidation.
|
||||
|
||||
**Algorithm**: EWC (Kirkpatrick et al., 2017) on an 8-input, 4-output linear classifier.
|
||||
|
||||
The classifier has 32 learnable parameters (8 inputs x 4 outputs). Training uses gradient descent with an EWC penalty term:
|
||||
|
||||
```
|
||||
L_total = L_current + (lambda/2) * sum_i(F_i * (theta_i - theta_i*)^2)
|
||||
```
|
||||
|
||||
- `L_current` = MSE between predicted zone and one-hot target
|
||||
- `F_i` = Fisher Information diagonal (how important each parameter is for previous tasks)
|
||||
- `theta_i*` = parameter values at the end of the previous task
|
||||
- `lambda` = 1000 (strong regularization to prevent forgetting)
|
||||
|
||||
Gradients are estimated via finite differences (perturb each parameter by epsilon=0.01, measure loss change). Only 4 parameters are updated per frame (round-robin) to stay within the 2ms budget.
|
||||
|
||||
#### Task Boundary Detection
|
||||
|
||||
A "task" corresponds to a stable environment (room layout). Task boundaries are detected automatically:
|
||||
1. Track consecutive frames where loss < 0.1
|
||||
2. After 100 consecutive stable frames, commit the task:
|
||||
- Snapshot parameters as `theta_star`
|
||||
- Update Fisher diagonal from accumulated gradient squares
|
||||
- Reset stability counter
|
||||
|
||||
Up to 32 tasks can be learned before the Fisher memory saturates.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct EwcLifelong { /* ... */ }
|
||||
|
||||
impl EwcLifelong {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, features: &[f32], target_zone: i32) -> &[(i32, f32)];
|
||||
pub fn predict(features: &[f32]) -> u8; // Inference only (zone 0-3)
|
||||
pub fn parameters() -> &[f32; 32]; // Current model weights
|
||||
pub fn fisher_diagonal() -> &[f32; 32]; // Parameter importance
|
||||
pub fn task_count() -> u8; // Completed tasks
|
||||
pub fn last_loss() -> f32; // Last total loss
|
||||
pub fn last_penalty() -> f32; // Last EWC penalty
|
||||
pub fn frame_count() -> u32;
|
||||
pub fn has_prior_task() -> bool;
|
||||
pub fn reset(&mut self);
|
||||
}
|
||||
```
|
||||
|
||||
Note: `target_zone = -1` means inference only (no gradient update).
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 745 | `KNOWLEDGE_RETAINED` | Penalty | EWC penalty magnitude (lower = less forgetting, emitted every 20 frames) |
|
||||
| 746 | `NEW_TASK_LEARNED` | Task count | A new task was committed (environment successfully learned) |
|
||||
| 747 | `FISHER_UPDATE` | Mean Fisher | Average Fisher information across all parameters |
|
||||
| 748 | `FORGETTING_RISK` | Ratio | Ratio of EWC penalty to current loss (high = risk of forgetting) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_PARAMS` | 32 | Total learnable parameters (8x4) |
|
||||
| `N_INPUT` | 8 | Input features (subcarrier group means) |
|
||||
| `N_OUTPUT` | 4 | Output zones |
|
||||
| `LAMBDA` | 1000.0 | EWC regularization strength |
|
||||
| `EPSILON` | 0.01 | Finite-difference perturbation size |
|
||||
| `PARAMS_PER_FRAME` | 4 | Round-robin gradient updates per frame |
|
||||
| `LEARNING_RATE` | 0.001 | Gradient descent step size |
|
||||
| `STABLE_FRAMES_THRESHOLD` | 100 | Consecutive stable frames to trigger task boundary |
|
||||
| `STABLE_LOSS_THRESHOLD` | 0.1 | Loss below this = "stable" frame |
|
||||
| `FISHER_ALPHA` | 0.01 | EMA alpha for Fisher diagonal updates |
|
||||
| `MAX_TASKS` | 32 | Maximum tasks before Fisher saturates |
|
||||
|
||||
#### Tutorial: How Lifelong Learning Works on a Microcontroller
|
||||
|
||||
**The Problem**: Traditional neural networks suffer from "catastrophic forgetting." If you train a network on Room A and then train it on Room B, it forgets everything about Room A. This is a fundamental limitation, not a bug.
|
||||
|
||||
**The EWC Solution**: Before learning Room B, the system measures which parameters were important for Room A (via the Fisher Information diagonal). Then, while learning Room B, it adds a penalty that prevents important-for-Room-A parameters from changing too much. The result: the network learns Room B while retaining Room A knowledge.
|
||||
|
||||
**On the ESP32**: The classifier is intentionally tiny (32 parameters) to keep computation within 2ms per frame. Despite its simplicity, a linear classifier over 8 subcarrier group features can reliably distinguish 4 spatial zones. The Fisher diagonal only requires 32 floats (128 bytes) per task. With 32 tasks maximum, total Fisher memory is ~4 KB.
|
||||
|
||||
**Monitoring forgetting risk**: The `FORGETTING_RISK` event (ID 748) reports the ratio of EWC penalty to current loss. If this ratio exceeds 1.0, the EWC constraint is dominating the learning signal, meaning the system is struggling to learn the new task without forgetting old ones. This can happen when:
|
||||
- The new environment is very different from all previous ones
|
||||
- The 32-parameter model capacity is exhausted
|
||||
- The Fisher diagonal has saturated from too many tasks
|
||||
|
||||
---
|
||||
|
||||
## How Learning Works on a Microcontroller
|
||||
|
||||
ESP32-S3 constraints that shape the design of all adaptive learning modules:
|
||||
|
||||
### No GPU
|
||||
All computation is done on the CPU (Xtensa LX7 dual-core at 240 MHz) via the WASM3 interpreter. This means:
|
||||
- No matrix multiplication hardware
|
||||
- No parallel SIMD operations
|
||||
- Every floating-point operation counts
|
||||
|
||||
### Fixed Memory
|
||||
WASM3 allocates a fixed linear memory region. There is no heap, no `malloc`, no dynamic allocation:
|
||||
- All arrays are fixed-size and stack-allocated
|
||||
- Maximum data structure sizes are compile-time constants
|
||||
- Buffer overflows are impossible (Rust's bounds checking + fixed arrays)
|
||||
|
||||
### EWC for Preventing Forgetting
|
||||
Without EWC, moving the device to a new room would erase everything learned about the previous room. EWC adds ~32 floats of overhead per task (the Fisher diagonal snapshot), which is negligible on the ESP32.
|
||||
|
||||
### Round-Robin Gradient Estimation
|
||||
Computing gradients for all 32 parameters every frame would take too long. Instead, the EWC module uses round-robin scheduling: 4 parameters per frame, cycling through all 32 in 8 frames. At 20 Hz, a full gradient pass takes 0.4 seconds -- fast enough for the slow dynamics of room occupancy.
|
||||
|
||||
### Task Boundary Detection
|
||||
The system automatically detects when it has "converged" on a new environment (100 consecutive stable frames = 5 seconds of consistent low loss). No manual intervention needed. The user just places the device in a new room, and the learning happens automatically.
|
||||
|
||||
### Energy Budget
|
||||
|
||||
| Module | Budget | Per-Frame Operations | Memory |
|
||||
|--------|--------|---------------------|--------|
|
||||
| DTW Gesture Learn | H (<10ms) | DTW: 64x64=4096 mults per template, up to 16 templates | ~18 KB (templates + rehearsals) |
|
||||
| Anomaly Attractor | S (<5ms) | 4D distance + log for Lyapunov + EMA | ~2.5 KB (128 trajectory points) |
|
||||
| Meta Adapt | S (<5ms) | Score computation + perturbation (timer only, not per-frame) | ~256 bytes |
|
||||
| EWC Lifelong | L (<2ms) | 4 finite-difference evals + gradient step | ~512 bytes (params + Fisher + theta_star) |
|
||||
|
||||
Total static memory for all 4 learning modules: approximately 21 KB.
|
||||
@@ -0,0 +1,246 @@
|
||||
# AI Security Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Tamper detection and behavioral anomaly profiling that protect the sensing system from manipulation. These modules detect replay attacks, signal injection, jamming, and unusual behavior patterns -- all running on-device with no cloud dependency.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| Signal Shield | `ais_prompt_shield.rs` | Detects replay, injection, and jamming attacks on CSI data | 820-823 | S (<5 ms) |
|
||||
| Behavioral Profiler | `ais_behavioral_profiler.rs` | Learns normal behavior and detects anomalous deviations | 825-828 | S (<5 ms) |
|
||||
|
||||
---
|
||||
|
||||
## Signal Shield (`ais_prompt_shield.rs`)
|
||||
|
||||
**What it does**: Detects three types of attack on the WiFi sensing system:
|
||||
|
||||
1. **Replay attacks**: An adversary records legitimate CSI frames and plays them back to fool the sensor into seeing a "normal" scene while actually present in the room.
|
||||
2. **Signal injection**: An adversary transmits a strong WiFi signal to overpower the legitimate CSI, creating amplitude spikes across many subcarriers.
|
||||
3. **Jamming**: An adversary floods the WiFi channel with noise, degrading the signal-to-noise ratio below usable levels.
|
||||
|
||||
**How it works**:
|
||||
|
||||
- **Replay detection**: Each frame's features (mean phase, mean amplitude, amplitude variance) are quantized and hashed using FNV-1a. The hash is stored in a 64-entry ring buffer. If a new frame's hash matches any recent hash, it flags a replay.
|
||||
- **Injection detection**: If more than 25% of subcarriers show a >10x amplitude jump from the previous frame, it flags injection.
|
||||
- **Jamming detection**: The module calibrates a baseline SNR (signal / sqrt(variance)) over the first 100 frames. If the current SNR drops below 10% of baseline for 5+ consecutive frames, it flags jamming.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ais_prompt_shield::PromptShield;
|
||||
|
||||
let mut shield = PromptShield::new(); // const fn, zero-alloc
|
||||
let events = shield.process_frame(&phases, &litudes); // per-frame analysis
|
||||
let calibrated = shield.is_calibrated(); // true after 100 frames
|
||||
let frames = shield.frame_count(); // total frames processed
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 820 | `EVENT_REPLAY_ATTACK` | 1.0 (detected) | On detection (cooldown: 40 frames) |
|
||||
| 821 | `EVENT_INJECTION_DETECTED` | Fraction of subcarriers with spikes [0.25, 1.0] | On detection (cooldown: 40 frames) |
|
||||
| 822 | `EVENT_JAMMING_DETECTED` | SNR drop in dB (10 * log10(baseline/current)) | On detection (cooldown: 40 frames) |
|
||||
| 823 | `EVENT_SIGNAL_INTEGRITY` | Composite integrity score [0.0, 1.0] | Every 20 frames |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_SC` | 32 | Maximum subcarriers processed |
|
||||
| `HASH_RING` | 64 | Size of replay detection hash ring buffer |
|
||||
| `INJECTION_FACTOR` | 10.0 | Amplitude jump threshold (10x previous) |
|
||||
| `INJECTION_FRAC` | 0.25 | Minimum fraction of subcarriers with spikes |
|
||||
| `JAMMING_SNR_FRAC` | 0.10 | SNR must drop below 10% of baseline |
|
||||
| `JAMMING_CONSEC` | 5 | Consecutive low-SNR frames required |
|
||||
| `BASELINE_FRAMES` | 100 | Calibration period length |
|
||||
| `COOLDOWN` | 40 | Frames between repeated alerts (2 seconds at 20 Hz) |
|
||||
|
||||
#### Signal Integrity Score
|
||||
|
||||
The composite score (event 823) is emitted every 20 frames and ranges from 0.0 (compromised) to 1.0 (clean):
|
||||
|
||||
| Factor | Score Reduction | Condition |
|
||||
|--------|-----------------|-----------|
|
||||
| Replay detected | -0.4 | Frame hash matches ring buffer |
|
||||
| Injection detected | up to -0.3 | Proportional to injection fraction |
|
||||
| SNR degradation | up to -0.3 | Proportional to SNR drop below baseline |
|
||||
|
||||
#### FNV-1a Hash Details
|
||||
|
||||
The hash function quantizes three frame statistics to integer precision before hashing:
|
||||
|
||||
```
|
||||
hash = FNV_OFFSET (2166136261)
|
||||
for each of [mean_phase*100, mean_amp*100, amp_variance*100]:
|
||||
for each byte in value.to_le_bytes():
|
||||
hash ^= byte
|
||||
hash = hash.wrapping_mul(FNV_PRIME) // FNV_PRIME = 16777619
|
||||
```
|
||||
|
||||
This means two frames must have nearly identical statistical profiles (within 1% quantization) to trigger a replay alert.
|
||||
|
||||
#### Example: Detecting a Replay Attack
|
||||
|
||||
```
|
||||
Calibration (frames 1-100):
|
||||
Normal CSI with varying phases -> baseline SNR established
|
||||
No alerts emitted during calibration
|
||||
|
||||
Frame 150: Normal operation
|
||||
phases = [0.31, 0.28, ...], amps = [1.02, 0.98, ...]
|
||||
hash = 0xA7F3B21C -> stored in ring buffer
|
||||
No alerts
|
||||
|
||||
Frame 200: Attacker replays frame 150 exactly
|
||||
phases = [0.31, 0.28, ...], amps = [1.02, 0.98, ...]
|
||||
hash = 0xA7F3B21C -> MATCH found in ring buffer!
|
||||
-> EVENT_REPLAY_ATTACK = 1.0
|
||||
-> EVENT_SIGNAL_INTEGRITY = 0.6 (reduced by 0.4)
|
||||
```
|
||||
|
||||
#### Example: Detecting Signal Injection
|
||||
|
||||
```
|
||||
Frame 300: Normal amplitudes
|
||||
amps = [1.0, 1.1, 0.9, 1.0, ...]
|
||||
|
||||
Frame 301: Adversary injects strong signal
|
||||
amps = [15.0, 12.0, 14.0, 13.0, ...] (>10x jump on all subcarriers)
|
||||
injection_fraction = 1.0 (100% of subcarriers spiked)
|
||||
-> EVENT_INJECTION_DETECTED = 1.0
|
||||
-> EVENT_SIGNAL_INTEGRITY = 0.4
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Behavioral Profiler (`ais_behavioral_profiler.rs`)
|
||||
|
||||
**What it does**: Learns what "normal" behavior looks like over time, then detects anomalous deviations. It builds a 6-dimensional behavioral profile using online statistics (Welford's algorithm) and flags when new observations deviate significantly from the learned baseline.
|
||||
|
||||
**How it works**: Every 200 frames, the module computes a 6D feature vector from the observation window. During the learning phase (first 1000 frames), it trains Welford accumulators for each dimension. After maturity, it computes per-dimension Z-scores and a combined RMS Z-score. If the combined score exceeds 3.0, an anomaly is reported.
|
||||
|
||||
#### The 6 Behavioral Dimensions
|
||||
|
||||
| # | Dimension | Description | Typical Range |
|
||||
|---|-----------|-------------|---------------|
|
||||
| 0 | Presence Rate | Fraction of frames with presence | [0, 1] |
|
||||
| 1 | Average Motion | Mean motion energy in window | [0, ~5] |
|
||||
| 2 | Average Persons | Mean person count | [0, ~4] |
|
||||
| 3 | Activity Variance | Variance of motion energy | [0, ~10] |
|
||||
| 4 | Transition Rate | Presence state changes per frame | [0, 0.5] |
|
||||
| 5 | Dwell Time | Average consecutive presence run length | [0, 200] |
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ais_behavioral_profiler::BehavioralProfiler;
|
||||
|
||||
let mut bp = BehavioralProfiler::new(); // const fn
|
||||
let events = bp.process_frame(present, motion, n_persons); // per-frame
|
||||
let mature = bp.is_mature(); // true after learning
|
||||
let anomalies = bp.total_anomalies(); // cumulative count
|
||||
let mean = bp.dim_mean(0); // mean of dimension 0
|
||||
let var = bp.dim_variance(1); // variance of dim 1
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 825 | `EVENT_BEHAVIOR_ANOMALY` | Combined Z-score (RMS, > 3.0) | On detection (cooldown: 100 frames) |
|
||||
| 826 | `EVENT_PROFILE_DEVIATION` | Index of most deviant dimension (0-5) | Paired with anomaly |
|
||||
| 827 | `EVENT_NOVEL_PATTERN` | Count of dimensions with Z > 2.0 | When 3+ dimensions deviate |
|
||||
| 828 | `EVENT_PROFILE_MATURITY` | Days since sensor start | On maturity + periodically |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_DIM` | 6 | Behavioral dimensions |
|
||||
| `LEARNING_FRAMES` | 1000 | Frames before profiler matures |
|
||||
| `ANOMALY_Z` | 3.0 | Combined Z-score threshold for anomaly |
|
||||
| `NOVEL_Z` | 2.0 | Per-dimension Z-score threshold for novelty |
|
||||
| `NOVEL_MIN` | 3 | Minimum deviating dimensions for NOVEL_PATTERN |
|
||||
| `OBS_WIN` | 200 | Observation window size (frames) |
|
||||
| `COOLDOWN` | 100 | Frames between repeated anomaly alerts |
|
||||
| `MATURITY_INTERVAL` | 72000 | Frames between maturity reports (1 hour at 20 Hz) |
|
||||
|
||||
#### Welford's Online Algorithm
|
||||
|
||||
Each dimension maintains running statistics without storing all past values:
|
||||
|
||||
```
|
||||
On each new observation x:
|
||||
count += 1
|
||||
delta = x - mean
|
||||
mean += delta / count
|
||||
m2 += delta * (x - mean)
|
||||
|
||||
Variance = m2 / count
|
||||
Z-score = |x - mean| / sqrt(variance)
|
||||
```
|
||||
|
||||
This is numerically stable and requires only 12 bytes per dimension (count + mean + m2).
|
||||
|
||||
#### Example: Detecting an Intruder's Behavioral Signature
|
||||
|
||||
```
|
||||
Learning phase (day 1-2):
|
||||
Normal pattern: 1 person, present 8am-10pm, moderate motion
|
||||
Profile matures -> EVENT_PROFILE_MATURITY = 0.58 (days)
|
||||
|
||||
Day 3, 3am:
|
||||
Observation window: presence=1, high motion, 1 person
|
||||
Z-scores: presence_rate=2.8, motion=4.1, persons=0.3,
|
||||
variance=3.5, transition=2.2, dwell=1.9
|
||||
Combined Z = sqrt(mean(z^2)) = 3.4 > 3.0
|
||||
-> EVENT_BEHAVIOR_ANOMALY = 3.4
|
||||
-> EVENT_PROFILE_DEVIATION = 1 (motion dimension most deviant)
|
||||
-> EVENT_NOVEL_PATTERN = 3 (3 dimensions above Z=2.0)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Threat Model
|
||||
|
||||
### Attacks These Modules Detect
|
||||
|
||||
| Attack | Detection Module | Method | False Positive Rate |
|
||||
|--------|-----------------|--------|---------------------|
|
||||
| CSI frame replay | Signal Shield | FNV-1a hash ring matching | Low (1% quantization) |
|
||||
| Signal injection (e.g., rogue AP) | Signal Shield | >25% subcarriers with >10x amplitude spike | Very low |
|
||||
| Broadband jamming | Signal Shield | SNR drop below 10% of baseline for 5+ frames | Very low |
|
||||
| Narrowband jamming | Partially -- Signal Shield | May not trigger if < 25% subcarriers affected | Medium |
|
||||
| Behavioral anomaly (intruder at unusual time) | Behavioral Profiler | Combined Z-score > 3.0 across 6 dimensions | Low after maturation |
|
||||
| Gradual environmental change | Behavioral Profiler | Welford stats adapt, may flag if change is abrupt | Very low |
|
||||
|
||||
### Attacks These Modules Cannot Detect
|
||||
|
||||
| Attack | Why Not | Recommended Mitigation |
|
||||
|--------|---------|----------------------|
|
||||
| Sophisticated replay with slight phase variation | FNV-1a uses 1% quantization; small perturbations change the hash | Add temporal correlation checks (consecutive frame deltas) |
|
||||
| Man-in-the-middle on the WiFi channel | Modules analyze CSI content, not channel authentication | Use WPA3 encryption + MAC filtering |
|
||||
| Physical obstruction (blocking line-of-sight) | Looks like a person leaving, not an attack | Cross-reference with PIR sensors |
|
||||
| Slow amplitude drift (gradual injection) | Below the 10x threshold per frame | Add longer-term amplitude trend monitoring |
|
||||
| Firmware tampering | Modules run in WASM sandbox, cannot detect host compromise | Secure boot + signed firmware (ADR-032) |
|
||||
|
||||
### Deployment Recommendations
|
||||
|
||||
1. **Always run both modules together**: Signal Shield catches active attacks, Behavioral Profiler catches passive anomalies.
|
||||
2. **Allow full calibration**: Signal Shield needs 100 frames (5 seconds) for SNR baseline. Behavioral Profiler needs 1000 frames (~50 seconds) for reliable Z-scores.
|
||||
3. **Combine with Temporal Logic Guard** (`tmp_temporal_logic_guard.rs`): Its safety invariants catch impossible state combinations (e.g., "fall alert when room is empty") that indicate sensor manipulation.
|
||||
4. **Connect to the Self-Healing Mesh** (`aut_self_healing_mesh.rs`): If a node in the mesh is being jammed, the mesh can automatically reconfigure around the compromised node.
|
||||
|
||||
---
|
||||
|
||||
## Memory Layout
|
||||
|
||||
| Module | State Size (approx) | Static Event Buffer |
|
||||
|--------|---------------------|---------------------|
|
||||
| Signal Shield | ~420 bytes (64 hashes + 32 prev_amps + calibration) | 4 entries |
|
||||
| Behavioral Profiler | ~2.4 KB (200-entry observation window + 6 Welford stats) | 4 entries |
|
||||
|
||||
Both modules use fixed-size arrays and static event buffers. No heap allocation. Fully no_std compliant.
|
||||
@@ -0,0 +1,438 @@
|
||||
# Quantum-Inspired & Autonomous Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Advanced algorithms inspired by quantum computing, neuroscience, and AI planning. These modules let the ESP32 make autonomous decisions, heal its own mesh network, interpret high-level scene semantics, and explore room states using quantum-inspired search.
|
||||
|
||||
## Quantum-Inspired
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| Quantum Coherence | `qnt_quantum_coherence.rs` | Maps CSI phases onto a Bloch sphere to detect sudden environmental changes | 850-852 | H (<10 ms) |
|
||||
| Interference Search | `qnt_interference_search.rs` | Grover-inspired multi-hypothesis room state classifier | 855-857 | H (<10 ms) |
|
||||
|
||||
---
|
||||
|
||||
### Quantum Coherence (`qnt_quantum_coherence.rs`)
|
||||
|
||||
**What it does**: Maps each subcarrier's phase onto a point on the quantum Bloch sphere and computes an aggregate coherence metric from the mean Bloch vector magnitude. When all subcarrier phases are aligned, the system is "coherent" (like a quantum pure state). When phases scatter randomly, it is "decoherent" (like a maximally mixed state). Sudden decoherence -- a rapid entropy spike -- indicates an environmental disturbance such as a door opening, a person entering, or furniture being moved.
|
||||
|
||||
**Algorithm**: Each subcarrier phase is mapped to a 3D Bloch vector:
|
||||
- theta = |phase| (polar angle)
|
||||
- phi = sign(phase) * pi/2 (azimuthal angle)
|
||||
|
||||
Since phi is always +/- pi/2, cos(phi) = 0 and sin(phi) = +/- 1. This eliminates 2 trig calls per subcarrier (saving 64+ cosf/sinf calls per frame for 32 subcarriers). The x-component of the mean Bloch vector is always zero.
|
||||
|
||||
Von Neumann entropy: S = -p*log(p) - (1-p)*log(1-p) where p = (1 + |bloch|) / 2. S=0 when perfectly coherent (|bloch|=1), S=ln(2) when maximally mixed (|bloch|=0). EMA smoothing with alpha=0.15.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::qnt_quantum_coherence::QuantumCoherenceMonitor;
|
||||
|
||||
let mut mon = QuantumCoherenceMonitor::new(); // const fn
|
||||
let events = mon.process_frame(&phases); // per-frame
|
||||
let coh = mon.coherence(); // [0, 1], 1=pure state
|
||||
let ent = mon.entropy(); // [0, ln(2)]
|
||||
let norm_ent = mon.normalized_entropy(); // [0, 1]
|
||||
let bloch = mon.bloch_vector(); // [f32; 3]
|
||||
let frames = mon.frame_count(); // total frames
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 850 | `EVENT_ENTANGLEMENT_ENTROPY` | EMA-smoothed Von Neumann entropy [0, ln(2)] | Every 10 frames |
|
||||
| 851 | `EVENT_DECOHERENCE_EVENT` | Entropy jump magnitude (> 0.3) | On detection |
|
||||
| 852 | `EVENT_BLOCH_DRIFT` | Euclidean distance between consecutive Bloch vectors | Every 5 frames |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_SC` | 32 | Maximum subcarriers |
|
||||
| `ALPHA` | 0.15 | EMA smoothing factor |
|
||||
| `DECOHERENCE_THRESHOLD` | 0.3 | Entropy jump threshold |
|
||||
| `ENTROPY_EMIT_INTERVAL` | 10 | Frames between entropy reports |
|
||||
| `DRIFT_EMIT_INTERVAL` | 5 | Frames between drift reports |
|
||||
| `LN2` | 0.693147 | Maximum binary entropy |
|
||||
|
||||
#### Example: Door Opening Detection via Decoherence
|
||||
|
||||
```
|
||||
Frames 1-50: Empty room, phases stable at ~0.1 rad
|
||||
Bloch vector: (0, 0.10, 0.99) -> coherence = 0.995
|
||||
Entropy ~ 0.005 (near zero, pure state)
|
||||
|
||||
Frame 51: Door opens, multipath changes suddenly
|
||||
Phases scatter: [-2.1, 0.8, 1.5, -0.3, ...]
|
||||
Bloch vector: (0, 0.12, 0.34) -> coherence = 0.36
|
||||
Entropy jumps to 0.61
|
||||
-> EVENT_DECOHERENCE_EVENT = 0.605 (jump magnitude)
|
||||
-> EVENT_BLOCH_DRIFT = 0.65 (large Bloch vector displacement)
|
||||
|
||||
Frames 52-100: New stable multipath
|
||||
Phases settle at new values
|
||||
Entropy gradually decays via EMA
|
||||
No more decoherence events
|
||||
```
|
||||
|
||||
#### Bloch Sphere Intuition
|
||||
|
||||
Think of each subcarrier as a compass needle. When the room is stable, all needles point roughly the same direction (high coherence, low entropy). When something changes the WiFi multipath -- a person enters, a door opens, furniture moves -- the needles scatter in different directions (low coherence, high entropy). The Bloch sphere formalism quantifies this in a way that is mathematically precise and computationally cheap.
|
||||
|
||||
---
|
||||
|
||||
### Interference Search (`qnt_interference_search.rs`)
|
||||
|
||||
**What it does**: Maintains 16 amplitude-weighted hypotheses for the current room state (empty, person in zone A/B/C/D, two persons, exercising, sleeping, etc.) and uses a Grover-inspired oracle+diffusion process to converge on the most likely state.
|
||||
|
||||
**Algorithm**: Inspired by Grover's quantum search algorithm, adapted for classical computation:
|
||||
|
||||
1. **Oracle**: CSI evidence (presence, motion, person count) multiplies hypothesis amplitudes by boost (1.3) or dampen (0.7) factors depending on consistency.
|
||||
2. **Grover diffusion**: Reflects all amplitudes about their mean (a_i = 2*mean - a_i), concentrating probability mass on oracle-boosted hypotheses. Negative amplitudes are clamped to zero (classical approximation).
|
||||
3. **Normalization**: Amplitudes are renormalized so sum-of-squares = 1.0 (probability conservation).
|
||||
|
||||
After enough iterations, the winner emerges with probability > 0.5 (convergence threshold).
|
||||
|
||||
#### The 16 Hypotheses
|
||||
|
||||
| Index | Hypothesis | Oracle Evidence |
|
||||
|-------|-----------|----------------|
|
||||
| 0 | Empty | presence=0 |
|
||||
| 1-4 | Person in Zone A/B/C/D | presence=1, 1 person |
|
||||
| 5 | Two Persons | n_persons=2 |
|
||||
| 6 | Three Persons | n_persons>=3 |
|
||||
| 7 | Moving Left | high motion, moving state |
|
||||
| 8 | Moving Right | high motion, moving state |
|
||||
| 9 | Sitting | low motion, present |
|
||||
| 10 | Standing | low motion, present |
|
||||
| 11 | Falling | high motion (transient) |
|
||||
| 12 | Exercising | high motion, present |
|
||||
| 13 | Sleeping | low motion, present |
|
||||
| 14 | Cooking | moderate motion + moving |
|
||||
| 15 | Working | low motion, present |
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::qnt_interference_search::{InterferenceSearch, Hypothesis};
|
||||
|
||||
let mut search = InterferenceSearch::new(); // const fn, uniform amplitudes
|
||||
let events = search.process_frame(presence, motion_energy, n_persons);
|
||||
let winner = search.winner(); // Hypothesis enum
|
||||
let prob = search.winner_probability(); // [0, 1]
|
||||
let converged = search.is_converged(); // prob > 0.5
|
||||
let amp = search.amplitude(Hypothesis::Sleeping); // raw amplitude
|
||||
let p = search.probability(Hypothesis::Exercising); // amplitude^2
|
||||
let iters = search.iterations(); // total iterations
|
||||
search.reset(); // back to uniform
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 855 | `EVENT_HYPOTHESIS_WINNER` | Winning hypothesis index (0-15) | Every 10 frames or on change |
|
||||
| 856 | `EVENT_HYPOTHESIS_AMPLITUDE` | Winning hypothesis probability | Every 20 frames |
|
||||
| 857 | `EVENT_SEARCH_ITERATIONS` | Total Grover iterations | Every 50 frames |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_HYPO` | 16 | Number of room-state hypotheses |
|
||||
| `CONVERGENCE_PROB` | 0.5 | Threshold for declaring convergence |
|
||||
| `ORACLE_BOOST` | 1.3 | Amplitude multiplier for supported hypotheses |
|
||||
| `ORACLE_DAMPEN` | 0.7 | Amplitude multiplier for contradicted hypotheses |
|
||||
| `MOTION_HIGH_THRESH` | 0.5 | Motion energy threshold for "high motion" |
|
||||
| `MOTION_LOW_THRESH` | 0.15 | Motion energy threshold for "low motion" |
|
||||
|
||||
#### Example: Room State Classification
|
||||
|
||||
```
|
||||
Initial state: All 16 hypotheses at probability 1/16 = 0.0625
|
||||
|
||||
Frames 1-30: presence=0, motion=0, n_persons=0
|
||||
Oracle boosts Empty (index 0), dampens all others
|
||||
Diffusion concentrates probability mass on Empty
|
||||
After 30 iterations: P(Empty) = 0.72, P(others) < 0.03
|
||||
-> EVENT_HYPOTHESIS_WINNER = 0 (Empty)
|
||||
|
||||
Frames 31-60: presence=1, motion=0.8, n_persons=1
|
||||
Oracle boosts Exercising, MovingLeft, MovingRight
|
||||
Oracle dampens Empty, Sitting, Sleeping
|
||||
After 30 more iterations: P(Exercising) = 0.45
|
||||
-> EVENT_HYPOTHESIS_WINNER = 12 (Exercising)
|
||||
Winner changed -> event emitted immediately
|
||||
|
||||
Frames 61-90: presence=1, motion=0.05, n_persons=1
|
||||
Oracle boosts Sitting, Sleeping, Working, Standing
|
||||
Oracle dampens Exercising, MovingLeft, MovingRight
|
||||
-> Convergence shifts to static hypotheses
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Autonomous Systems
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| Psycho-Symbolic | `aut_psycho_symbolic.rs` | Context-aware inference using forward-chaining symbolic rules | 880-883 | H (<10 ms) |
|
||||
| Self-Healing Mesh | `aut_self_healing_mesh.rs` | Monitors mesh node health and auto-reconfigures via min-cut analysis | 885-888 | S (<5 ms) |
|
||||
|
||||
---
|
||||
|
||||
### Psycho-Symbolic Inference (`aut_psycho_symbolic.rs`)
|
||||
|
||||
**What it does**: Interprets raw CSI-derived features into high-level semantic conclusions using a knowledge base of 16 forward-chaining rules. Given presence, motion energy, breathing rate, heart rate, person count, coherence, and time of day, it determines conclusions like "person resting", "possible intruder", "medical distress", or "social activity".
|
||||
|
||||
**Algorithm**: Forward-chaining rule evaluation. Each rule has 4 condition slots (feature_id, comparison_op, threshold). A rule fires when all non-disabled conditions match. Confidence propagation: the final confidence is the rule's base confidence multiplied by per-condition match-quality scores (how far above/below threshold the feature is, clamped to [0.5, 1.0]). Contradiction detection resolves mutually exclusive conclusions by keeping the higher-confidence one.
|
||||
|
||||
#### The 16 Rules
|
||||
|
||||
| Rule | Conclusion | Conditions | Base Confidence |
|
||||
|------|-----------|------------|----------------|
|
||||
| R0 | Possible Intruder | Presence + high motion (>=200) + night | 0.80 |
|
||||
| R1 | Person Resting | Presence + low motion (<30) + breathing 10-22 BPM | 0.90 |
|
||||
| R2 | Pet or Environment | No presence + motion (>=15) | 0.60 |
|
||||
| R3 | Social Activity | Multi-person (>=2) + high motion (>=100) | 0.70 |
|
||||
| R4 | Exercise | 1 person + high motion (>=150) + elevated HR (>=100) | 0.80 |
|
||||
| R5 | Possible Fall | Presence + sudden stillness (motion<10, prev_motion>=150) | 0.70 |
|
||||
| R6 | Interference | Low coherence (<0.4) + presence | 0.50 |
|
||||
| R7 | Sleeping | Presence + very low motion (<5) + night + breathing (>=8) | 0.90 |
|
||||
| R8 | Cooking Activity | Presence + moderate motion (40-120) + evening | 0.60 |
|
||||
| R9 | Leaving Home | No presence + previous motion (>=50) + morning | 0.65 |
|
||||
| R10 | Arriving Home | Presence + motion (>=60) + low prev_motion (<15) + evening | 0.70 |
|
||||
| R11 | Child Playing | Multi-person (>=2) + very high motion (>=250) + daytime | 0.60 |
|
||||
| R12 | Working at Desk | 1 person + low motion (<20) + good coherence (>=0.6) + morning | 0.75 |
|
||||
| R13 | Medical Distress | Presence + very high HR (>=130) + low motion (<15) | 0.85 |
|
||||
| R14 | Room Empty (Stable) | No presence + no motion (<5) + good coherence (>=0.6) | 0.95 |
|
||||
| R15 | Crowd Gathering | Many persons (>=4) + high motion (>=120) | 0.70 |
|
||||
|
||||
#### Contradiction Pairs
|
||||
|
||||
These conclusions are mutually exclusive. When both fire, only the one with higher confidence survives:
|
||||
|
||||
| Pair A | Pair B |
|
||||
|--------|--------|
|
||||
| Sleeping | Exercise |
|
||||
| Sleeping | Social Activity |
|
||||
| Room Empty (Stable) | Possible Intruder |
|
||||
| Person Resting | Exercise |
|
||||
|
||||
#### Input Features
|
||||
|
||||
| Index | Feature | Source | Range |
|
||||
|-------|---------|--------|-------|
|
||||
| 0 | Presence | Tier 2 DSP | 0 (absent) or 1 (present) |
|
||||
| 1 | Motion Energy | Tier 2 DSP | 0 to ~1000 |
|
||||
| 2 | Breathing BPM | Tier 2 vitals | 0-60 |
|
||||
| 3 | Heart Rate BPM | Tier 2 vitals | 0-200 |
|
||||
| 4 | Person Count | Tier 2 occupancy | 0-8 |
|
||||
| 5 | Coherence | QuantumCoherenceMonitor or upstream | 0-1 |
|
||||
| 6 | Time Bucket | Host clock | 0=morning, 1=afternoon, 2=evening, 3=night |
|
||||
| 7 | Previous Motion | Internal (auto-tracked) | 0 to ~1000 |
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::aut_psycho_symbolic::PsychoSymbolicEngine;
|
||||
|
||||
let mut engine = PsychoSymbolicEngine::new(); // const fn
|
||||
engine.set_coherence(0.8); // from upstream module
|
||||
let events = engine.process_frame(
|
||||
presence, motion, breathing, heartrate, n_persons, time_bucket
|
||||
);
|
||||
let rules = engine.fired_rules(); // u16 bitmap
|
||||
let count = engine.fired_count(); // number of rules that fired
|
||||
let prev = engine.prev_conclusion(); // last winning conclusion ID
|
||||
let contras = engine.contradiction_count(); // total contradictions
|
||||
engine.reset(); // clear state
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 880 | `EVENT_INFERENCE_RESULT` | Conclusion ID (1-16) | When any rule fires |
|
||||
| 881 | `EVENT_INFERENCE_CONFIDENCE` | Confidence [0, 1] of the winning conclusion | Paired with result |
|
||||
| 882 | `EVENT_RULE_FIRED` | Rule index (0-15) | For each rule that fired |
|
||||
| 883 | `EVENT_CONTRADICTION` | Encoded pair: conclusion_a * 100 + conclusion_b | On contradiction |
|
||||
|
||||
#### Example: Fall Detection Sequence
|
||||
|
||||
```
|
||||
Frame 1: Person walking briskly
|
||||
Features: presence=1, motion=200, breathing=20, HR=90, persons=1, time=1
|
||||
R4 (Exercise) fires: confidence = 0.80 * 0.75 = 0.60
|
||||
-> EVENT_INFERENCE_RESULT = 5 (Exercise)
|
||||
-> EVENT_INFERENCE_CONFIDENCE = 0.60
|
||||
|
||||
Frame 2: Sudden stillness (prev_motion=200, current motion=3)
|
||||
R5 (Possible Fall) fires: confidence = 0.70 * 0.85 = 0.595
|
||||
R1 (Person Resting) also fires: confidence = 0.90 * 0.50 = 0.45
|
||||
No contradiction between these two
|
||||
-> EVENT_RULE_FIRED = 5 (Fall rule)
|
||||
-> EVENT_RULE_FIRED = 1 (Resting rule)
|
||||
-> EVENT_INFERENCE_RESULT = 6 (Possible Fall, highest confidence)
|
||||
-> EVENT_INFERENCE_CONFIDENCE = 0.595
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Self-Healing Mesh (`aut_self_healing_mesh.rs`)
|
||||
|
||||
**What it does**: Monitors the health of an 8-node sensor mesh and automatically detects when the network topology becomes fragile. Uses the Stoer-Wagner minimum graph cut algorithm to find the weakest link in the mesh. When the min-cut value drops below a threshold, it identifies the degraded node and triggers a reconfiguration event.
|
||||
|
||||
**Algorithm**: Stoer-Wagner min-cut on a weighted graph of up to 8 nodes. Edge weights are the minimum quality score of the two endpoints (min(q_i, q_j)). Quality scores are EMA-smoothed (alpha=0.15) per-node CSI coherence values. O(n^3) complexity, which is only 512 operations for n=8. State machine transitions between healthy and healing modes.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::aut_self_healing_mesh::SelfHealingMesh;
|
||||
|
||||
let mut mesh = SelfHealingMesh::new(); // const fn
|
||||
mesh.update_node_quality(0, coherence); // update single node
|
||||
let events = mesh.process_frame(&node_qualities); // process all nodes
|
||||
let q = mesh.node_quality(2); // EMA quality for node 2
|
||||
let n = mesh.active_nodes(); // count
|
||||
let mc = mesh.prev_mincut(); // last min-cut value
|
||||
let healing = mesh.is_healing(); // fragile state?
|
||||
let weak = mesh.weakest_node(); // node ID or 0xFF
|
||||
mesh.reset(); // clear state
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 885 | `EVENT_NODE_DEGRADED` | Index of the degraded node (0-7) | When min-cut < 0.3 |
|
||||
| 886 | `EVENT_MESH_RECONFIGURE` | Min-cut value (measure of fragility) | Paired with degraded |
|
||||
| 887 | `EVENT_COVERAGE_SCORE` | Mean quality across all active nodes [0, 1] | Every frame |
|
||||
| 888 | `EVENT_HEALING_COMPLETE` | Min-cut value (now healthy) | When min-cut recovers >= 0.6 |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_NODES` | 8 | Maximum mesh nodes |
|
||||
| `QUALITY_ALPHA` | 0.15 | EMA smoothing for node quality |
|
||||
| `MINCUT_FRAGILE` | 0.3 | Below this, mesh is considered fragile |
|
||||
| `MINCUT_HEALTHY` | 0.6 | Above this, healing is considered complete |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
mincut < 0.3
|
||||
[Healthy] ----------------------> [Healing]
|
||||
^ |
|
||||
| mincut >= 0.6 |
|
||||
+---------------------------------+
|
||||
```
|
||||
|
||||
#### Stoer-Wagner Min-Cut Details
|
||||
|
||||
The algorithm finds the minimum weight of edges that, if removed, would disconnect the graph into two components. For an 8-node mesh:
|
||||
|
||||
1. Start with the full weighted adjacency matrix
|
||||
2. For each phase (n-1 phases total):
|
||||
- Grow a set A by repeatedly adding the node with the highest total edge weight to A
|
||||
- The last two nodes added (prev, last) define a "cut of the phase" = weight to last
|
||||
- Track the global minimum cut across all phases
|
||||
- Merge the last two nodes (combine their edge weights)
|
||||
3. Return (global_min_cut, node_on_lighter_side)
|
||||
|
||||
#### Example: Node Failure and Recovery
|
||||
|
||||
```
|
||||
Frame 1: All 4 nodes healthy
|
||||
qualities = [0.9, 0.85, 0.88, 0.92]
|
||||
Coverage = 0.89
|
||||
Min-cut = 0.85 (well above 0.6)
|
||||
-> EVENT_COVERAGE_SCORE = 0.89
|
||||
|
||||
Frame 50: Node 1 starts degrading
|
||||
qualities = [0.9, 0.20, 0.88, 0.92]
|
||||
EMA-smoothed quality[1] drops gradually
|
||||
Min-cut drops to 0.20 (edge weights use min(q_i, q_j))
|
||||
Min-cut < 0.3 -> FRAGILE!
|
||||
-> EVENT_NODE_DEGRADED = 1
|
||||
-> EVENT_MESH_RECONFIGURE = 0.20
|
||||
-> Mesh enters healing mode
|
||||
|
||||
Host firmware can now:
|
||||
- Increase node 1's transmit power
|
||||
- Route traffic around node 1
|
||||
- Wake up a backup node
|
||||
- Alert the operator
|
||||
|
||||
Frame 100: Node 1 recovers (antenna repositioned)
|
||||
qualities = [0.9, 0.85, 0.88, 0.92]
|
||||
Min-cut climbs back to 0.85
|
||||
Min-cut >= 0.6 -> HEALTHY!
|
||||
-> EVENT_HEALING_COMPLETE = 0.85
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## How Quantum-Inspired Algorithms Help WiFi Sensing
|
||||
|
||||
These modules use quantum computing metaphors -- not because the ESP32 is a quantum computer, but because the mathematical frameworks from quantum mechanics map naturally onto CSI signal analysis:
|
||||
|
||||
**Bloch Sphere / Coherence**: WiFi subcarrier phases behave like quantum phases. When multipath is stable, all phases align (pure state). When the environment changes, phases randomize (mixed state). The Von Neumann entropy quantifies this exactly, providing a single scalar "change detector" that is more robust than tracking individual subcarrier phases.
|
||||
|
||||
**Grover's Algorithm / Hypothesis Search**: The oracle+diffusion loop is a principled way to combine evidence from multiple noisy sensors. Instead of hard-coding "if motion > 0.5 then exercising", the Grover-inspired search lets multiple hypotheses compete. Evidence gradually amplifies the correct hypothesis while suppressing incorrect ones. This is more robust to noisy CSI data than a single threshold.
|
||||
|
||||
**Why not just use classical statistics?** You could. But the quantum-inspired formulations have three practical advantages on embedded hardware:
|
||||
|
||||
1. **Fixed memory**: The Bloch vector is always 3 floats. The hypothesis array is always 16 floats. No dynamic allocation needed.
|
||||
2. **Graceful degradation**: If CSI data is noisy, the Grover search does not crash or give a wrong answer immediately -- it just converges more slowly.
|
||||
3. **Composability**: The coherence score from the Bloch sphere module feeds directly into the Temporal Logic Guard (rule 3: "no vital signs when coherence < 0.3") and the Psycho-Symbolic engine (feature 5: coherence). This creates a pipeline where quantum-inspired metrics inform classical reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Memory Layout
|
||||
|
||||
| Module | State Size (approx) | Static Event Buffer |
|
||||
|--------|---------------------|---------------------|
|
||||
| Quantum Coherence | ~40 bytes (3D Bloch vector + 2 entropy floats + counter) | 3 entries |
|
||||
| Interference Search | ~80 bytes (16 amplitudes + counters) | 3 entries |
|
||||
| Psycho-Symbolic | ~24 bytes (bitmap + counters + prev_motion) | 8 entries |
|
||||
| Self-Healing Mesh | ~360 bytes (8x8 adjacency + 8 qualities + state) | 6 entries |
|
||||
|
||||
All modules use fixed-size arrays and static event buffers. No heap allocation. Fully no_std compliant for WASM3 deployment on ESP32-S3.
|
||||
|
||||
---
|
||||
|
||||
## Cross-Module Integration
|
||||
|
||||
These modules are designed to work together in a pipeline:
|
||||
|
||||
```
|
||||
CSI Frame (Tier 2 DSP)
|
||||
|
|
||||
v
|
||||
[Quantum Coherence] --coherence--> [Psycho-Symbolic Engine]
|
||||
| |
|
||||
v v
|
||||
[Interference Search] [Inference Result]
|
||||
| |
|
||||
v v
|
||||
[Room State Hypothesis] [GOAP Planner]
|
||||
|
|
||||
v
|
||||
[Module Activate/Deactivate]
|
||||
|
|
||||
v
|
||||
[Self-Healing Mesh]
|
||||
|
|
||||
v
|
||||
[Reconfiguration Events]
|
||||
```
|
||||
|
||||
The Quantum Coherence monitor feeds its coherence score to:
|
||||
- **Psycho-Symbolic Engine**: As feature 5 (coherence), enabling rules R3 (interference) and R6 (low coherence)
|
||||
- **Temporal Logic Guard**: Rule 3 checks "no vital signs when coherence < 0.3"
|
||||
- **Self-Healing Mesh**: Node quality can be derived from coherence
|
||||
|
||||
The GOAP Planner uses inference results to decide which modules to activate (e.g., activate vitals monitoring when a person is present, enter low-power mode when the room is empty).
|
||||
@@ -0,0 +1,397 @@
|
||||
# Smart Building Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Make any building smarter using WiFi signals you already have. Know which rooms are occupied, control HVAC and lighting automatically, count elevator passengers, track meeting room usage, and audit energy waste -- all without cameras or badges.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Frame Budget |
|
||||
|--------|------|--------------|-----------|--------------|
|
||||
| HVAC Presence | `bld_hvac_presence.rs` | Presence detection tuned for HVAC energy management | 310-312 | ~0.5 us/frame |
|
||||
| Lighting Zones | `bld_lighting_zones.rs` | Per-zone lighting control (On/Dim/Off) based on spatial occupancy | 320-322 | ~1 us/frame |
|
||||
| Elevator Count | `bld_elevator_count.rs` | Occupant counting in elevator cabins (1-12 persons) | 330-333 | ~1.5 us/frame |
|
||||
| Meeting Room | `bld_meeting_room.rs` | Meeting lifecycle tracking with utilization metrics | 340-343 | ~0.3 us/frame |
|
||||
| Energy Audit | `bld_energy_audit.rs` | 24x7 hourly occupancy histograms for scheduling optimization | 350-352 | ~0.2 us/frame |
|
||||
|
||||
All modules target the ESP32-S3 running WASM3 (ADR-040 Tier 3). They receive pre-processed CSI signals from Tier 2 DSP and emit structured events via `csi_emit_event()`.
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### HVAC Presence Control (`bld_hvac_presence.rs`)
|
||||
|
||||
**What it does**: Tells your HVAC system whether a room is occupied, with intentionally asymmetric timing -- fast arrival detection (10 seconds) so cooling/heating starts quickly, and slow departure timeout (5 minutes) to avoid premature shutoff when someone briefly steps out. Also classifies whether the occupant is sedentary (desk work, reading) or active (walking, exercising).
|
||||
|
||||
**How it works**: A four-state machine processes presence scores and motion energy each frame:
|
||||
|
||||
```
|
||||
Vacant --> ArrivalPending --> Occupied --> DeparturePending --> Vacant
|
||||
(10s debounce) (5 min timeout)
|
||||
```
|
||||
|
||||
Motion energy is smoothed with an exponential moving average (alpha=0.1) and classified against a threshold of 0.3 to distinguish sedentary from active behavior.
|
||||
|
||||
#### State Machine
|
||||
|
||||
| State | Entry Condition | Exit Condition |
|
||||
|-------|----------------|----------------|
|
||||
| `Vacant` | No presence detected | Presence score > 0.5 |
|
||||
| `ArrivalPending` | Presence detected, debounce counting | 200 consecutive frames with presence -> Occupied; any absence -> Vacant |
|
||||
| `Occupied` | Arrival debounce completed | First frame without presence -> DeparturePending |
|
||||
| `DeparturePending` | Presence lost | 6000 frames without presence -> Vacant; any presence -> Occupied |
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 310 | `HVAC_OCCUPIED` | 1.0 (occupied) or 0.0 (vacant) | Every 20 frames |
|
||||
| 311 | `ACTIVITY_LEVEL` | 0.0-0.99 (sedentary + EMA) or 1.0 (active) | Every 20 frames |
|
||||
| 312 | `DEPARTURE_COUNTDOWN` | 0.0-1.0 (fraction of timeout remaining) | Every 20 frames during DeparturePending |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::bld_hvac_presence::HvacPresenceDetector;
|
||||
|
||||
let mut det = HvacPresenceDetector::new();
|
||||
|
||||
// Per-frame processing
|
||||
let events = det.process_frame(presence_score, motion_energy);
|
||||
// events: &[(event_type: i32, value: f32)]
|
||||
|
||||
// Queries
|
||||
det.state() // -> HvacState (Vacant|ArrivalPending|Occupied|DeparturePending)
|
||||
det.is_occupied() // -> bool (true during Occupied or DeparturePending)
|
||||
det.activity() // -> ActivityLevel (Sedentary|Active)
|
||||
det.motion_ema() // -> f32 (smoothed motion energy)
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `ARRIVAL_DEBOUNCE` | 200 frames (10s) | Frames of continuous presence before confirming occupancy |
|
||||
| `DEPARTURE_TIMEOUT` | 6000 frames (5 min) | Frames of continuous absence before declaring vacant |
|
||||
| `ACTIVITY_THRESHOLD` | 0.3 | Motion EMA above this = Active |
|
||||
| `MOTION_ALPHA` | 0.1 | EMA smoothing factor for motion energy |
|
||||
| `PRESENCE_THRESHOLD` | 0.5 | Minimum presence score to consider someone present |
|
||||
| `EMIT_INTERVAL` | 20 frames (1s) | Event emission interval |
|
||||
|
||||
#### Example: BACnet Integration
|
||||
|
||||
```python
|
||||
# Python host reading events from ESP32 UDP packet
|
||||
if event_id == 310: # HVAC_OCCUPIED
|
||||
bacnet_write(device_id, "Occupancy", int(value)) # 1=occupied, 0=vacant
|
||||
elif event_id == 311: # ACTIVITY_LEVEL
|
||||
if value >= 1.0:
|
||||
bacnet_write(device_id, "CoolingSetpoint", 72) # Active: cooler
|
||||
else:
|
||||
bacnet_write(device_id, "CoolingSetpoint", 76) # Sedentary: warmer
|
||||
elif event_id == 312: # DEPARTURE_COUNTDOWN
|
||||
if value < 0.2: # Less than 1 minute remaining
|
||||
bacnet_write(device_id, "FanMode", "low") # Start reducing
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Lighting Zone Control (`bld_lighting_zones.rs`)
|
||||
|
||||
**What it does**: Manages up to 4 independent lighting zones, automatically transitioning each zone between On (occupied and active), Dim (occupied but sedentary for over 10 minutes), and Off (vacant for over 30 seconds). Uses per-zone variance analysis to determine which areas of the room have people.
|
||||
|
||||
**How it works**: Subcarriers are divided into groups (one per zone). Each group's amplitude variance is computed and compared against a calibrated baseline. Variance deviation above threshold indicates occupancy in that zone. A calibration phase (200 frames = 10 seconds) establishes the baseline with an empty room.
|
||||
|
||||
```
|
||||
Off --> On (occupancy + activity detected)
|
||||
On --> Dim (occupied but sedentary for 10 min)
|
||||
On --> Dim (vacancy detected, grace period)
|
||||
Dim --> Off (vacant for 30 seconds)
|
||||
Dim --> On (activity resumes)
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 320 | `LIGHT_ON` | zone_id (0-3) | On state transition |
|
||||
| 321 | `LIGHT_DIM` | zone_id (0-3) | Dim state transition |
|
||||
| 322 | `LIGHT_OFF` | zone_id (0-3) | Off state transition |
|
||||
|
||||
Periodic summaries encode `zone_id + confidence` in the value field (integer part = zone, fractional part = occupancy score).
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::bld_lighting_zones::LightingZoneController;
|
||||
|
||||
let mut ctrl = LightingZoneController::new();
|
||||
|
||||
// Per-frame: pass subcarrier amplitudes and overall motion energy
|
||||
let events = ctrl.process_frame(&litudes, motion_energy);
|
||||
|
||||
// Queries
|
||||
ctrl.zone_state(zone_id) // -> LightState (Off|Dim|On)
|
||||
ctrl.n_zones() // -> usize (number of active zones, 1-4)
|
||||
ctrl.is_calibrated() // -> bool
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `MAX_ZONES` | 4 | Maximum lighting zones |
|
||||
| `OCCUPANCY_THRESHOLD` | 0.03 | Variance deviation ratio for occupancy |
|
||||
| `ACTIVE_THRESHOLD` | 0.25 | Motion energy for active classification |
|
||||
| `DIM_TIMEOUT` | 12000 frames (10 min) | Sedentary frames before dimming |
|
||||
| `OFF_TIMEOUT` | 600 frames (30s) | Vacant frames before turning off |
|
||||
| `BASELINE_FRAMES` | 200 frames (10s) | Calibration duration |
|
||||
|
||||
#### Example: DALI/KNX Lighting
|
||||
|
||||
```python
|
||||
# Map zone events to DALI addresses
|
||||
DALI_ADDR = {0: 1, 1: 2, 2: 3, 3: 4}
|
||||
|
||||
if event_id == 320: # LIGHT_ON
|
||||
zone = int(value)
|
||||
dali_send(DALI_ADDR[zone], level=254) # Full brightness
|
||||
elif event_id == 321: # LIGHT_DIM
|
||||
zone = int(value)
|
||||
dali_send(DALI_ADDR[zone], level=80) # 30% brightness
|
||||
elif event_id == 322: # LIGHT_OFF
|
||||
zone = int(value)
|
||||
dali_send(DALI_ADDR[zone], level=0) # Off
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Elevator Occupancy Counting (`bld_elevator_count.rs`)
|
||||
|
||||
**What it does**: Counts the number of people in an elevator cabin (0-12), detects door open/close events, and emits overload warnings when the count exceeds a configurable threshold. Uses the confined-space multipath characteristics of an elevator to correlate amplitude variance with body count.
|
||||
|
||||
**How it works**: In a small reflective metal box like an elevator, each additional person adds significant multipath scattering. The module calibrates on the empty cabin, then maps the ratio of current variance to baseline variance onto a person count. Frame-to-frame amplitude deltas detect sudden geometry changes (door open/close). Count estimate fuses the module's own variance-based estimate (40% weight) with the host's person count hint (60% weight) when available.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 330 | `ELEVATOR_COUNT` | Person count (0-12) | Every 10 frames |
|
||||
| 331 | `DOOR_OPEN` | Current count at time of opening | On door open detection |
|
||||
| 332 | `DOOR_CLOSE` | Current count at time of closing | On door close detection |
|
||||
| 333 | `OVERLOAD_WARNING` | Current count | When count >= overload threshold |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::bld_elevator_count::ElevatorCounter;
|
||||
|
||||
let mut ec = ElevatorCounter::new();
|
||||
|
||||
// Per-frame: amplitudes, phases, motion energy, host person count hint
|
||||
let events = ec.process_frame(&litudes, &phases, motion_energy, host_n_persons);
|
||||
|
||||
// Queries
|
||||
ec.occupant_count() // -> u8 (0-12)
|
||||
ec.door_state() // -> DoorState (Open|Closed)
|
||||
ec.is_calibrated() // -> bool
|
||||
|
||||
// Configuration
|
||||
ec.set_overload_threshold(8); // Set custom overload limit
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `MAX_OCCUPANTS` | 12 | Maximum tracked occupants |
|
||||
| `DEFAULT_OVERLOAD` | 10 | Default overload warning threshold |
|
||||
| `DOOR_VARIANCE_RATIO` | 4.0 | Delta magnitude for door detection |
|
||||
| `DOOR_DEBOUNCE` | 3 frames | Debounce for door events |
|
||||
| `DOOR_COOLDOWN` | 40 frames (2s) | Cooldown after door event |
|
||||
| `BASELINE_FRAMES` | 200 frames (10s) | Calibration with empty cabin |
|
||||
|
||||
---
|
||||
|
||||
### Meeting Room Tracker (`bld_meeting_room.rs`)
|
||||
|
||||
**What it does**: Tracks the full lifecycle of meeting room usage -- from someone entering, to confirming a genuine multi-person meeting, to detecting when the meeting ends and the room is available again. Distinguishes actual meetings (2+ people for more than 3 seconds) from a single person briefly using the room. Tracks peak headcount and calculates room utilization rate.
|
||||
|
||||
**How it works**: A four-state machine processes presence and person count:
|
||||
|
||||
```
|
||||
Empty --> PreMeeting --> Active --> PostMeeting --> Empty
|
||||
(someone (2+ people (everyone left,
|
||||
entered) confirmed) 2 min cooldown)
|
||||
```
|
||||
|
||||
The PreMeeting state has a 3-minute timeout: if only one person remains, the room is not promoted to "Active" (it is not counted as a meeting).
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 340 | `MEETING_START` | Current person count | On transition to Active |
|
||||
| 341 | `MEETING_END` | Duration in minutes | On transition to PostMeeting |
|
||||
| 342 | `PEAK_HEADCOUNT` | Peak person count | On meeting end + periodic during Active |
|
||||
| 343 | `ROOM_AVAILABLE` | 1.0 | On transition from PostMeeting to Empty |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::bld_meeting_room::MeetingRoomTracker;
|
||||
|
||||
let mut mt = MeetingRoomTracker::new();
|
||||
|
||||
// Per-frame: presence (0/1), person count, motion energy
|
||||
let events = mt.process_frame(presence, n_persons, motion_energy);
|
||||
|
||||
// Queries
|
||||
mt.state() // -> MeetingState (Empty|PreMeeting|Active|PostMeeting)
|
||||
mt.peak_headcount() // -> u8
|
||||
mt.meeting_count() // -> u32 (total meetings since reset)
|
||||
mt.utilization_rate() // -> f32 (fraction of time in meetings, 0.0-1.0)
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `MEETING_MIN_PERSONS` | 2 | Minimum people for a "meeting" |
|
||||
| `PRE_MEETING_TIMEOUT` | 3600 frames (3 min) | Max time waiting for meeting to form |
|
||||
| `POST_MEETING_TIMEOUT` | 2400 frames (2 min) | Cooldown before marking room available |
|
||||
| `MEETING_MIN_FRAMES` | 6000 frames (5 min) | Reference minimum meeting duration |
|
||||
|
||||
#### Example: Calendar Integration
|
||||
|
||||
```python
|
||||
# Sync meeting room status with calendar system
|
||||
if event_id == 340: # MEETING_START
|
||||
calendar_api.mark_room_in_use(room_id, headcount=int(value))
|
||||
elif event_id == 341: # MEETING_END
|
||||
duration_min = value
|
||||
calendar_api.log_actual_usage(room_id, duration_min)
|
||||
elif event_id == 343: # ROOM_AVAILABLE
|
||||
calendar_api.mark_room_available(room_id)
|
||||
display_screen.show("Room Available")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Energy Audit (`bld_energy_audit.rs`)
|
||||
|
||||
**What it does**: Builds a 7-day, 24-hour occupancy histogram (168 hourly bins) to identify energy waste patterns. Finds which hours are consistently unoccupied (candidates for HVAC/lighting shutoff), detects after-hours occupancy anomalies (security/safety concern), and reports overall building utilization.
|
||||
|
||||
**How it works**: Each frame increments the appropriate hour bin's counters. The module maintains its own simulated clock (hour/day) that advances by counting frames (72,000 frames = 1 hour at 20 Hz). The host can set the real time via `set_time()`. After-hours is defined as 22:00-06:00 (wraps midnight correctly). Sustained presence (30+ seconds) during after-hours triggers an alert.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 350 | `SCHEDULE_SUMMARY` | Current hour's occupancy rate (0.0-1.0) | Every 1200 frames (1 min) |
|
||||
| 351 | `AFTER_HOURS_ALERT` | Current hour (22-5) | After 600 frames (30s) of after-hours presence |
|
||||
| 352 | `UTILIZATION_RATE` | Overall utilization (0.0-1.0) | Every 1200 frames (1 min) |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::bld_energy_audit::EnergyAuditor;
|
||||
|
||||
let mut ea = EnergyAuditor::new();
|
||||
|
||||
// Set real time from host
|
||||
ea.set_time(0, 8); // Monday 8 AM (day 0-6, hour 0-23)
|
||||
|
||||
// Per-frame: presence (0/1), person count
|
||||
let events = ea.process_frame(presence, n_persons);
|
||||
|
||||
// Queries
|
||||
ea.utilization_rate() // -> f32 (overall)
|
||||
ea.hourly_rate(day, hour) // -> f32 (occupancy rate for specific slot)
|
||||
ea.hourly_headcount(day, hour) // -> f32 (average headcount)
|
||||
ea.unoccupied_hours(day) // -> u8 (hours below 10% occupancy)
|
||||
ea.current_time() // -> (day, hour)
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `FRAMES_PER_HOUR` | 72000 | Frames in one hour at 20 Hz |
|
||||
| `SUMMARY_INTERVAL` | 1200 frames (1 min) | How often to emit summaries |
|
||||
| `AFTER_HOURS_START` | 22 (10 PM) | Start of after-hours window |
|
||||
| `AFTER_HOURS_END` | 6 (6 AM) | End of after-hours window |
|
||||
| `USED_THRESHOLD` | 0.1 | Minimum occupancy rate to consider an hour "used" |
|
||||
| `AFTER_HOURS_ALERT_FRAMES` | 600 frames (30s) | Sustained presence before alert |
|
||||
|
||||
#### Example: Energy Optimization Report
|
||||
|
||||
```python
|
||||
# Generate weekly energy optimization report
|
||||
for day in range(7):
|
||||
unused = auditor.unoccupied_hours(day)
|
||||
print(f"{DAY_NAMES[day]}: {unused} hours could have HVAC off")
|
||||
|
||||
for hour in range(24):
|
||||
rate = auditor.hourly_rate(day, hour)
|
||||
if rate < 0.1:
|
||||
print(f" {hour:02d}:00 - unused ({rate:.0%} occupancy)")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Integration Guide
|
||||
|
||||
### Connecting to BACnet / HVAC Systems
|
||||
|
||||
All five building modules emit events via the standard `csi_emit_event()` interface. A typical integration path:
|
||||
|
||||
1. **ESP32 firmware** receives events from the WASM module
|
||||
2. **UDP packet** carries events to the aggregator server (port 5005)
|
||||
3. **Sensing server** (`wifi-densepose-sensing-server`) exposes events via REST API
|
||||
4. **BMS integration script** polls the API and writes BACnet/Modbus objects
|
||||
|
||||
Key BACnet object mappings:
|
||||
|
||||
| Module | BACnet Object Type | Property |
|
||||
|--------|--------------------|----------|
|
||||
| HVAC Presence | Binary Value | Occupancy (310: 1=occupied) |
|
||||
| HVAC Presence | Analog Value | Activity Level (311: 0-1) |
|
||||
| Lighting Zones | Multi-State Value | Zone State (320-322: Off/Dim/On) |
|
||||
| Elevator Count | Analog Value | Occupant Count (330: 0-12) |
|
||||
| Meeting Room | Binary Value | Room In Use (340/343) |
|
||||
| Energy Audit | Analog Value | Utilization Rate (352: 0-1.0) |
|
||||
|
||||
### Lighting Control Integration (DALI, KNX)
|
||||
|
||||
The `bld_lighting_zones` module emits zone-level On/Dim/Off transitions. Map each zone to a DALI address group or KNX group address:
|
||||
|
||||
- Event 320 (LIGHT_ON) -> DALI command `DAPC(254)` or KNX `DPT_Switch ON`
|
||||
- Event 321 (LIGHT_DIM) -> DALI command `DAPC(80)` or KNX `DPT_Scaling 30%`
|
||||
- Event 322 (LIGHT_OFF) -> DALI command `DAPC(0)` or KNX `DPT_Switch OFF`
|
||||
|
||||
### BMS (Building Management System) Integration
|
||||
|
||||
For full BMS integration combining all five modules:
|
||||
|
||||
```
|
||||
ESP32 Nodes (per room/zone)
|
||||
|
|
||||
v UDP events
|
||||
Aggregator Server
|
||||
|
|
||||
v REST API / WebSocket
|
||||
BMS Gateway Script
|
||||
|
|
||||
+-- HVAC Controller (BACnet/Modbus)
|
||||
+-- Lighting Controller (DALI/KNX)
|
||||
+-- Elevator Display Panel
|
||||
+-- Meeting Room Booking System
|
||||
+-- Energy Dashboard
|
||||
```
|
||||
|
||||
### Deployment Considerations
|
||||
|
||||
- **Calibration**: Lighting and Elevator modules require a 10-second calibration with an empty room/cabin. Schedule calibration during known unoccupied periods.
|
||||
- **Clock sync**: The Energy Audit module needs `set_time()` called at startup. Use NTP on the aggregator or pass timestamp via the host API.
|
||||
- **Multiple ESP32s**: For open-plan offices, deploy one ESP32 per zone. Each runs its own HVAC Presence and Lighting Zones instance. The aggregator merges zone-level data.
|
||||
- **Event rate**: All modules throttle events to at most one emission per second (EMIT_INTERVAL = 20 frames). Total bandwidth per module is under 100 bytes/second.
|
||||
@@ -0,0 +1,594 @@
|
||||
# Core Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> The foundation modules that every ESP32 node runs. These handle gesture detection, signal quality monitoring, anomaly detection, zone occupancy, vital sign tracking, intrusion classification, and model packaging.
|
||||
|
||||
All seven modules compile to `wasm32-unknown-unknown` and run inside the WASM3 interpreter on ESP32-S3 after Tier 2 DSP completes (ADR-040). They share a common `no_std`-compatible design: a struct with `const fn new()`, a `process_frame` (or `on_timer`) entry point, and zero heap allocation.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Compute Budget |
|
||||
|--------|------|-------------|----------------|
|
||||
| Gesture Classifier | `gesture.rs` | Recognizes hand gestures from CSI phase sequences using DTW template matching | ~2,400 f32 ops/frame (60x40 cost matrix) |
|
||||
| Coherence Monitor | `coherence.rs` | Measures signal quality via phasor coherence across subcarriers | ~100 trig ops/frame (32 subcarriers) |
|
||||
| Anomaly Detector | `adversarial.rs` | Flags physically impossible signals: phase jumps, flatlines, energy spikes | ~130 f32 ops/frame |
|
||||
| Intrusion Detector | `intrusion.rs` | Detects unauthorized entry via phase velocity and amplitude disturbance | ~130 f32 ops/frame |
|
||||
| Occupancy Detector | `occupancy.rs` | Divides sensing area into spatial zones and reports which are occupied | ~100 f32 ops/frame |
|
||||
| Vital Trend Analyzer | `vital_trend.rs` | Monitors breathing/heart rate over 1-min and 5-min windows for clinical alerts | ~20 f32 ops/timer tick |
|
||||
| RVF Container | `rvf.rs` | Binary container format that packages WASM modules with manifest and signature | Builder only (std), no per-frame cost |
|
||||
|
||||
## Modules
|
||||
|
||||
---
|
||||
|
||||
### Gesture Classifier (`gesture.rs`)
|
||||
|
||||
**What it does**: Recognizes predefined hand gestures from WiFi CSI phase sequences. It compares a sliding window of phase deltas against 4 built-in templates (wave, push, pull, swipe) using Dynamic Time Warping.
|
||||
|
||||
**How it works**: Each incoming frame provides subcarrier phases. The detector computes the phase delta from the previous frame and pushes it into a 60-sample ring buffer. When enough samples accumulate, it runs constrained DTW (with a Sakoe-Chiba band of width 5) between the tail of the observation window and each template. If the best normalized distance falls below the threshold (2.5), the corresponding gesture ID is emitted. A 40-frame cooldown prevents duplicate detections.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `GestureDetector` | struct | Main state holder. Contains ring buffer, templates, and cooldown timer. |
|
||||
| `GestureDetector::new()` | `const fn` | Creates a detector with 4 built-in templates. |
|
||||
| `GestureDetector::process_frame(&mut self, phases: &[f32]) -> Option<u8>` | method | Feed one frame of phase data. Returns `Some(gesture_id)` on match. |
|
||||
| `MAX_TEMPLATE_LEN` | const (40) | Maximum number of samples in a gesture template. |
|
||||
| `MAX_WINDOW_LEN` | const (60) | Maximum observation window length. |
|
||||
| `NUM_TEMPLATES` | const (4) | Number of built-in templates. |
|
||||
| `DTW_THRESHOLD` | const (2.5) | Normalized DTW distance threshold for a match. |
|
||||
| `BAND_WIDTH` | const (5) | Sakoe-Chiba band width (limits warping). |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `DTW_THRESHOLD` | 2.5 | 0.5 -- 10.0 | Lower = stricter matching, fewer false positives but may miss soft gestures |
|
||||
| `BAND_WIDTH` | 5 | 1 -- 20 | Width of the Sakoe-Chiba band. Wider = more flexible time warping but more computation |
|
||||
| Cooldown frames | 40 | 10 -- 200 | Frames to wait before next detection. At 20 Hz, 40 frames = 2 seconds |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 1 | `event_types::GESTURE_DETECTED` | A gesture template matched. Value = gesture ID (1=wave, 2=push, 3=pull, 4=swipe). |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::gesture::GestureDetector;
|
||||
|
||||
let mut detector = GestureDetector::new();
|
||||
|
||||
// Feed frames from CSI data (typically at 20 Hz).
|
||||
let phases: Vec<f32> = get_csi_phases(); // your phase data
|
||||
if let Some(gesture_id) = detector.process_frame(&phases) {
|
||||
println!("Detected gesture {}", gesture_id);
|
||||
// 1 = wave, 2 = push, 3 = pull, 4 = swipe
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Adding a Custom Gesture Template
|
||||
|
||||
1. **Collect reference data**: Record the phase-delta sequence for your gesture by feeding CSI frames through the detector and logging the delta values in the ring buffer.
|
||||
|
||||
2. **Normalize the template**: Scale the phase-delta values so they span roughly -1.0 to 1.0. This ensures consistent DTW distances across different signal strengths.
|
||||
|
||||
3. **Edit the template array**: In `gesture.rs`, increase `NUM_TEMPLATES` by 1 and add a new entry in the `templates` array inside `GestureDetector::new()`:
|
||||
```rust
|
||||
GestureTemplate {
|
||||
values: {
|
||||
let mut v = [0.0f32; MAX_TEMPLATE_LEN];
|
||||
v[0] = 0.2; v[1] = 0.6; // ... your values
|
||||
v
|
||||
},
|
||||
len: 8, // number of valid samples
|
||||
id: 5, // unique gesture ID
|
||||
},
|
||||
```
|
||||
|
||||
4. **Tune the threshold**: Run test data through `dtw_distance()` directly to see the distance between your template and real observations. Adjust `DTW_THRESHOLD` if your gesture is consistently matched at a distance higher than 2.5.
|
||||
|
||||
5. **Test**: Add a unit test that feeds the template values as phase inputs and verifies that `process_frame` returns your new gesture ID.
|
||||
|
||||
---
|
||||
|
||||
### Coherence Monitor (`coherence.rs`)
|
||||
|
||||
**What it does**: Measures the phase coherence of the WiFi signal across subcarriers. High coherence means the signal is stable and sensing is accurate. Low coherence means multipath interference or environmental changes are degrading the signal.
|
||||
|
||||
**How it works**: For each frame, it computes the inter-frame phase delta per subcarrier, converts each delta to a unit phasor (cos + j*sin), and averages them. The magnitude of this mean phasor is the raw coherence (0 = random, 1 = perfectly aligned). This raw value is smoothed with an exponential moving average (alpha = 0.1). A hysteresis gate classifies the result into Accept (>0.7), Warn (0.4--0.7), or Reject (<0.4).
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `CoherenceMonitor` | struct | Tracks phasor sums, EMA score, and gate state. |
|
||||
| `CoherenceMonitor::new()` | `const fn` | Creates a monitor with initial coherence of 1.0 (Accept). |
|
||||
| `process_frame(&mut self, phases: &[f32]) -> f32` | method | Feed one frame of phase data. Returns EMA-smoothed coherence [0, 1]. |
|
||||
| `gate_state(&self) -> GateState` | method | Current gate classification (Accept, Warn, Reject). |
|
||||
| `mean_phasor_angle(&self) -> f32` | method | Dominant phase drift direction in radians. |
|
||||
| `coherence_score(&self) -> f32` | method | Current EMA-smoothed coherence score. |
|
||||
| `GateState` | enum | `Accept`, `Warn`, `Reject` -- signal quality classification. |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `ALPHA` | 0.1 | 0.01 -- 0.5 | EMA smoothing factor. Lower = slower response, more stable. Higher = faster response, more noisy |
|
||||
| `HIGH_THRESHOLD` | 0.7 | 0.5 -- 0.95 | Coherence above this = Accept |
|
||||
| `LOW_THRESHOLD` | 0.4 | 0.1 -- 0.6 | Coherence below this = Reject |
|
||||
| `MAX_SC` | 32 | 1 -- 64 | Maximum subcarriers tracked (compile-time) |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 2 | `event_types::COHERENCE_SCORE` | Emitted every 20 frames with the current coherence score (from the combined pipeline in `lib.rs`). |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::coherence::{CoherenceMonitor, GateState};
|
||||
|
||||
let mut monitor = CoherenceMonitor::new();
|
||||
|
||||
let phases: Vec<f32> = get_csi_phases();
|
||||
let score = monitor.process_frame(&phases);
|
||||
|
||||
match monitor.gate_state() {
|
||||
GateState::Accept => { /* full accuracy */ }
|
||||
GateState::Warn => { /* predictions may be degraded */ }
|
||||
GateState::Reject => { /* sensing unreliable, recalibrate */ }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Anomaly Detector (`adversarial.rs`)
|
||||
|
||||
**What it does**: Detects physically impossible or suspicious CSI signals that may indicate sensor malfunction, RF jamming, replay attacks, or environmental interference. It runs three independent checks on every frame.
|
||||
|
||||
**How it works**: During the first 100 frames it accumulates a baseline (mean amplitude per subcarrier and mean total energy). After calibration, it checks each frame for three anomaly types:
|
||||
|
||||
1. **Phase jump**: If more than 50% of subcarriers show a phase discontinuity greater than 2.5 radians, something non-physical happened.
|
||||
2. **Amplitude flatline**: If amplitude variance across subcarriers is near zero (below 0.001) while the mean is nonzero, the sensor may be stuck.
|
||||
3. **Energy spike**: If total signal energy exceeds 50x the baseline, an external source may be injecting power.
|
||||
|
||||
A 20-frame cooldown prevents event flooding.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `AnomalyDetector` | struct | Tracks baseline, previous phases, cooldown, and anomaly count. |
|
||||
| `AnomalyDetector::new()` | `const fn` | Creates an uncalibrated detector. |
|
||||
| `process_frame(&mut self, phases: &[f32], amplitudes: &[f32]) -> bool` | method | Returns `true` if an anomaly is detected on this frame. |
|
||||
| `total_anomalies(&self) -> u32` | method | Lifetime count of detected anomalies. |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `PHASE_JUMP_THRESHOLD` | 2.5 rad | 1.0 -- pi | Phase jump to flag per subcarrier |
|
||||
| `MIN_AMPLITUDE_VARIANCE` | 0.001 | 0.0001 -- 0.1 | Below this = flatline |
|
||||
| `MAX_ENERGY_RATIO` | 50.0 | 5.0 -- 500.0 | Energy spike threshold vs baseline |
|
||||
| `BASELINE_FRAMES` | 100 | 50 -- 500 | Frames to calibrate baseline |
|
||||
| `ANOMALY_COOLDOWN` | 20 | 5 -- 100 | Frames between anomaly reports |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 3 | `event_types::ANOMALY_DETECTED` | When any anomaly check fires (after cooldown). |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::adversarial::AnomalyDetector;
|
||||
|
||||
let mut detector = AnomalyDetector::new();
|
||||
|
||||
// First 100 frames calibrate the baseline (always returns false).
|
||||
for _ in 0..100 {
|
||||
detector.process_frame(&phases, &litudes);
|
||||
}
|
||||
|
||||
// Now anomalies are reported.
|
||||
if detector.process_frame(&phases, &litudes) {
|
||||
log!("Signal anomaly detected! Total: {}", detector.total_anomalies());
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Intrusion Detector (`intrusion.rs`)
|
||||
|
||||
**What it does**: Detects unauthorized entry into a monitored area. It is designed for security applications with a bias toward low false-negative rate (it would rather alarm falsely than miss a real intrusion).
|
||||
|
||||
**How it works**: The detector goes through four states:
|
||||
|
||||
1. **Calibrating** (200 frames): Learns baseline amplitude mean and variance per subcarrier.
|
||||
2. **Monitoring**: Waits for the environment to be quiet (low disturbance for 100 consecutive frames) before arming.
|
||||
3. **Armed**: Actively watching. Computes a disturbance score combining phase velocity (60% weight) and amplitude deviation (40% weight). If disturbance exceeds 0.8 for 3 consecutive frames, it triggers an alert.
|
||||
4. **Alert**: Intrusion detected. Returns to Armed once disturbance drops below 0.3 for 50 frames.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `IntrusionDetector` | struct | State machine with baseline, debounce, and cooldown. |
|
||||
| `IntrusionDetector::new()` | `const fn` | Creates a detector in Calibrating state. |
|
||||
| `process_frame(&mut self, phases: &[f32], amplitudes: &[f32]) -> &[(i32, f32)]` | method | Returns a slice of events (up to 4 per frame). |
|
||||
| `state(&self) -> DetectorState` | method | Current state machine state. |
|
||||
| `total_alerts(&self) -> u32` | method | Lifetime alert count. |
|
||||
| `DetectorState` | enum | `Calibrating`, `Monitoring`, `Armed`, `Alert`. |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `INTRUSION_VELOCITY_THRESH` | 1.5 rad/frame | 0.5 -- 3.0 | Phase velocity that counts as fast movement |
|
||||
| `AMPLITUDE_CHANGE_THRESH` | 3.0 sigma | 1.0 -- 10.0 | Amplitude deviation in standard deviations |
|
||||
| `ARM_FRAMES` | 100 | 20 -- 500 | Quiet frames needed to arm (at 20 Hz: 5 sec) |
|
||||
| `DETECT_DEBOUNCE` | 3 | 1 -- 10 | Consecutive detection frames before alert |
|
||||
| `ALERT_COOLDOWN` | 100 | 20 -- 500 | Frames between alerts |
|
||||
| `BASELINE_FRAMES` | 200 | 100 -- 1000 | Calibration window |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 200 | `EVENT_INTRUSION_ALERT` | Intrusion detected. Value = disturbance score. |
|
||||
| 201 | `EVENT_INTRUSION_ZONE` | Identifies which subcarrier zone has the most disturbance. |
|
||||
| 202 | `EVENT_INTRUSION_ARMED` | Detector has armed after a quiet period. |
|
||||
| 203 | `EVENT_INTRUSION_DISARMED` | Detector disarmed (not currently emitted). |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::intrusion::{IntrusionDetector, DetectorState};
|
||||
|
||||
let mut detector = IntrusionDetector::new();
|
||||
|
||||
// Calibrate and arm (feed quiet frames).
|
||||
for _ in 0..300 {
|
||||
detector.process_frame(&quiet_phases, &quiet_amps);
|
||||
}
|
||||
assert_eq!(detector.state(), DetectorState::Armed);
|
||||
|
||||
// Now process live data.
|
||||
let events = detector.process_frame(&live_phases, &live_amps);
|
||||
for &(event_type, value) in events {
|
||||
if event_type == 200 {
|
||||
trigger_alarm(value);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Occupancy Detector (`occupancy.rs`)
|
||||
|
||||
**What it does**: Divides the sensing area into spatial zones (based on subcarrier groupings) and determines which zones are currently occupied by people. Useful for smart building applications such as HVAC control and lighting automation.
|
||||
|
||||
**How it works**: Subcarriers are divided into groups of 4, with each group representing a spatial zone (up to 8 zones). For each zone, the detector computes the variance of amplitude values within that group. During calibration (200 frames), it learns the baseline variance. After calibration, it computes the deviation from baseline, applies EMA smoothing (alpha=0.15), and uses a hysteresis threshold to classify each zone as occupied or empty. Events include per-zone occupancy (emitted every 10 frames) and zone transitions (emitted immediately on change).
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `OccupancyDetector` | struct | Per-zone state, calibration accumulators, frame counter. |
|
||||
| `OccupancyDetector::new()` | `const fn` | Creates uncalibrated detector. |
|
||||
| `process_frame(&mut self, phases: &[f32], amplitudes: &[f32]) -> &[(i32, f32)]` | method | Returns events (up to 12 per frame). |
|
||||
| `occupied_count(&self) -> u8` | method | Number of currently occupied zones. |
|
||||
| `is_zone_occupied(&self, zone_id: usize) -> bool` | method | Check a specific zone. |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `MAX_ZONES` | 8 | 1 -- 16 | Maximum number of spatial zones |
|
||||
| `ZONE_THRESHOLD` | 0.02 | 0.005 -- 0.5 | Score above this = occupied. Hysteresis exit at 0.5x |
|
||||
| `ALPHA` | 0.15 | 0.05 -- 0.5 | EMA smoothing factor for zone scores |
|
||||
| `BASELINE_FRAMES` | 200 | 100 -- 1000 | Calibration window length |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 300 | `EVENT_ZONE_OCCUPIED` | Every 10 frames for each occupied zone. Value = `zone_id + confidence`. |
|
||||
| 301 | `EVENT_ZONE_COUNT` | Every 10 frames. Value = total occupied zone count. |
|
||||
| 302 | `EVENT_ZONE_TRANSITION` | Immediately on zone state change. Value = `zone_id + 0.5` (entered) or `zone_id + 0.0` (vacated). |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::occupancy::OccupancyDetector;
|
||||
|
||||
let mut detector = OccupancyDetector::new();
|
||||
|
||||
// Calibrate with empty-room data.
|
||||
for _ in 0..200 {
|
||||
detector.process_frame(&empty_phases, &empty_amps);
|
||||
}
|
||||
|
||||
// Live monitoring.
|
||||
let events = detector.process_frame(&live_phases, &live_amps);
|
||||
println!("Occupied zones: {}", detector.occupied_count());
|
||||
println!("Zone 0 occupied: {}", detector.is_zone_occupied(0));
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Vital Trend Analyzer (`vital_trend.rs`)
|
||||
|
||||
**What it does**: Monitors breathing rate and heart rate over time and alerts on clinically significant conditions. It tracks 1-minute and 5-minute trends and detects apnea, bradypnea, tachypnea, bradycardia, and tachycardia.
|
||||
|
||||
**How it works**: Called at 1 Hz with current vital sign readings (from Tier 2 DSP). It pushes each reading into a 300-sample ring buffer (5-minute history). Each call checks for:
|
||||
|
||||
- **Apnea**: Breathing BPM below 1.0 for 20+ consecutive seconds.
|
||||
- **Bradypnea**: Sustained breathing below 12 BPM (5+ consecutive samples).
|
||||
- **Tachypnea**: Sustained breathing above 25 BPM (5+ consecutive samples).
|
||||
- **Bradycardia**: Sustained heart rate below 50 BPM (5+ consecutive samples).
|
||||
- **Tachycardia**: Sustained heart rate above 120 BPM (5+ consecutive samples).
|
||||
|
||||
Every 60 seconds, it emits 1-minute averages for both breathing and heart rate.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `VitalTrendAnalyzer` | struct | Two ring buffers (breathing, heartrate), debounce counters, apnea counter. |
|
||||
| `VitalTrendAnalyzer::new()` | `const fn` | Creates analyzer with empty history. |
|
||||
| `on_timer(&mut self, breathing_bpm: f32, heartrate_bpm: f32) -> &[(i32, f32)]` | method | Called at 1 Hz. Returns clinical alerts (up to 8). |
|
||||
| `breathing_avg_1m(&self) -> f32` | method | 1-minute breathing rate average. |
|
||||
| `breathing_trend_5m(&self) -> f32` | method | 5-minute breathing trend (positive = increasing). |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `BRADYPNEA_THRESH` | 12.0 BPM | 8 -- 15 | Below this = dangerously slow breathing |
|
||||
| `TACHYPNEA_THRESH` | 25.0 BPM | 20 -- 35 | Above this = dangerously fast breathing |
|
||||
| `BRADYCARDIA_THRESH` | 50.0 BPM | 40 -- 60 | Below this = dangerously slow heart rate |
|
||||
| `TACHYCARDIA_THRESH` | 120.0 BPM | 100 -- 150 | Above this = dangerously fast heart rate |
|
||||
| `APNEA_SECONDS` | 20 | 10 -- 60 | Seconds of near-zero breathing before alert |
|
||||
| `ALERT_DEBOUNCE` | 5 | 2 -- 15 | Consecutive abnormal samples before alert |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|-------------|
|
||||
| 100 | `EVENT_VITAL_TREND` | Reserved for generic trend events. |
|
||||
| 101 | `EVENT_BRADYPNEA` | Sustained slow breathing. Value = current BPM. |
|
||||
| 102 | `EVENT_TACHYPNEA` | Sustained fast breathing. Value = current BPM. |
|
||||
| 103 | `EVENT_BRADYCARDIA` | Sustained slow heart rate. Value = current BPM. |
|
||||
| 104 | `EVENT_TACHYCARDIA` | Sustained fast heart rate. Value = current BPM. |
|
||||
| 105 | `EVENT_APNEA` | Breathing stopped. Value = seconds of apnea. |
|
||||
| 110 | `EVENT_BREATHING_AVG` | 1-minute breathing average. Emitted every 60 seconds. |
|
||||
| 111 | `EVENT_HEARTRATE_AVG` | 1-minute heart rate average. Emitted every 60 seconds. |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::vital_trend::VitalTrendAnalyzer;
|
||||
|
||||
let mut analyzer = VitalTrendAnalyzer::new();
|
||||
|
||||
// Called at 1 Hz from the on_timer WASM export.
|
||||
let events = analyzer.on_timer(breathing_bpm, heartrate_bpm);
|
||||
for &(event_type, value) in events {
|
||||
match event_type {
|
||||
105 => alert_apnea(value as u32),
|
||||
101 => alert_bradypnea(value),
|
||||
104 => alert_tachycardia(value),
|
||||
110 => log_breathing_avg(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Query trend data.
|
||||
let avg = analyzer.breathing_avg_1m();
|
||||
let trend = analyzer.breathing_trend_5m();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### RVF Container (`rvf.rs`)
|
||||
|
||||
**What it does**: Defines the RVF (RuVector Format) binary container that packages a compiled WASM module with its manifest (name, author, capabilities, budget, hash) and an optional Ed25519 signature. This is the file format that gets uploaded to ESP32 nodes via the `/api/wasm/upload` endpoint.
|
||||
|
||||
**How it works**: The format has four sections laid out sequentially:
|
||||
|
||||
```
|
||||
[Header: 32 bytes][Manifest: 96 bytes][WASM: N bytes][Signature: 0|64 bytes]
|
||||
```
|
||||
|
||||
The header contains magic bytes (`RVF\x01`), format version, section sizes, and flags. The manifest describes the module's identity (name, author), resource requirements (max frame time, memory limit), and capability flags (which host APIs it needs). The WASM section is the raw compiled binary. The signature section is optional (indicated by `FLAG_HAS_SIGNATURE`) and covers everything before it.
|
||||
|
||||
The builder (available only with the `std` feature) creates RVF files from WASM binary data and a configuration struct. It automatically computes a SHA-256 hash of the WASM payload and embeds it in the manifest for integrity verification.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `RvfHeader` | `#[repr(C, packed)]` struct | 32-byte header with magic, version, section sizes. |
|
||||
| `RvfManifest` | `#[repr(C, packed)]` struct | 96-byte manifest with module metadata. |
|
||||
| `RvfConfig` | struct (std only) | Builder configuration input. |
|
||||
| `build_rvf(wasm_data: &[u8], config: &RvfConfig) -> Vec<u8>` | function (std only) | Build a complete RVF container. |
|
||||
| `patch_signature(rvf: &mut [u8], signature: &[u8; 64])` | function (std only) | Patch an Ed25519 signature into an existing RVF. |
|
||||
| `RVF_MAGIC` | const (`0x0146_5652`) | Magic bytes: `RVF\x01` as little-endian u32. |
|
||||
| `RVF_FORMAT_VERSION` | const (1) | Current format version. |
|
||||
| `RVF_HEADER_SIZE` | const (32) | Header size in bytes. |
|
||||
| `RVF_MANIFEST_SIZE` | const (96) | Manifest size in bytes. |
|
||||
| `RVF_SIGNATURE_LEN` | const (64) | Ed25519 signature length. |
|
||||
| `RVF_HOST_API_V1` | const (1) | Host API version this crate supports. |
|
||||
|
||||
#### Capability Flags
|
||||
|
||||
| Flag | Value | Description |
|
||||
|------|-------|-------------|
|
||||
| `CAP_READ_PHASE` | `1 << 0` | Module reads phase data |
|
||||
| `CAP_READ_AMPLITUDE` | `1 << 1` | Module reads amplitude data |
|
||||
| `CAP_READ_VARIANCE` | `1 << 2` | Module reads variance data |
|
||||
| `CAP_READ_VITALS` | `1 << 3` | Module reads vital sign data |
|
||||
| `CAP_READ_HISTORY` | `1 << 4` | Module reads phase history |
|
||||
| `CAP_EMIT_EVENTS` | `1 << 5` | Module emits events |
|
||||
| `CAP_LOG` | `1 << 6` | Module uses logging |
|
||||
| `CAP_ALL` | `0x7F` | All capabilities |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::rvf::builder::{build_rvf, RvfConfig, patch_signature};
|
||||
use wifi_densepose_wasm_edge::rvf::*;
|
||||
|
||||
// Read compiled WASM binary.
|
||||
let wasm_data = std::fs::read("target/wasm32-unknown-unknown/release/my_module.wasm")?;
|
||||
|
||||
// Configure the module.
|
||||
let config = RvfConfig {
|
||||
module_name: "my-gesture-v2".into(),
|
||||
author: "team-alpha".into(),
|
||||
capabilities: CAP_READ_PHASE | CAP_EMIT_EVENTS,
|
||||
max_frame_us: 5000, // 5 ms budget per frame
|
||||
max_events_per_sec: 20,
|
||||
memory_limit_kb: 64,
|
||||
min_subcarriers: 8,
|
||||
max_subcarriers: 64,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Build the RVF container.
|
||||
let rvf = build_rvf(&wasm_data, &config);
|
||||
|
||||
// Optionally sign and patch.
|
||||
let signature = sign_with_ed25519(&rvf[..rvf.len() - RVF_SIGNATURE_LEN]);
|
||||
let mut rvf_mut = rvf;
|
||||
patch_signature(&mut rvf_mut, &signature);
|
||||
|
||||
// Upload to ESP32.
|
||||
std::fs::write("my-gesture-v2.rvf", &rvf_mut)?;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
### Running Core Module Tests
|
||||
|
||||
From the crate directory:
|
||||
|
||||
```bash
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm-edge
|
||||
cargo test --features std -- gesture coherence adversarial intrusion occupancy vital_trend rvf
|
||||
```
|
||||
|
||||
This runs all tests whose names contain any of the seven module names. The `--features std` flag is required because the RVF builder tests need `sha2` and `std::io`.
|
||||
|
||||
### Expected Output
|
||||
|
||||
All tests should pass:
|
||||
|
||||
```
|
||||
running 32 tests
|
||||
test adversarial::tests::test_anomaly_detector_init ... ok
|
||||
test adversarial::tests::test_calibration_phase ... ok
|
||||
test adversarial::tests::test_normal_signal_no_anomaly ... ok
|
||||
test adversarial::tests::test_phase_jump_detection ... ok
|
||||
test adversarial::tests::test_amplitude_flatline_detection ... ok
|
||||
test adversarial::tests::test_energy_spike_detection ... ok
|
||||
test adversarial::tests::test_cooldown_prevents_flood ... ok
|
||||
test coherence::tests::test_coherence_monitor_init ... ok
|
||||
test coherence::tests::test_empty_phases_returns_current_score ... ok
|
||||
test coherence::tests::test_first_frame_returns_one ... ok
|
||||
test coherence::tests::test_constant_phases_high_coherence ... ok
|
||||
test coherence::tests::test_incoherent_phases_lower_coherence ... ok
|
||||
test coherence::tests::test_gate_hysteresis ... ok
|
||||
test coherence::tests::test_mean_phasor_angle_zero_for_no_drift ... ok
|
||||
test gesture::tests::test_gesture_detector_init ... ok
|
||||
test gesture::tests::test_empty_phases_returns_none ... ok
|
||||
test gesture::tests::test_first_frame_initializes ... ok
|
||||
test gesture::tests::test_constant_phase_no_gesture_after_cooldown ... ok
|
||||
test gesture::tests::test_dtw_identical_sequences ... ok
|
||||
test gesture::tests::test_dtw_different_sequences ... ok
|
||||
test gesture::tests::test_dtw_empty_input ... ok
|
||||
test gesture::tests::test_cooldown_prevents_duplicate_detection ... ok
|
||||
test gesture::tests::test_window_ring_buffer_wraps ... ok
|
||||
test intrusion::tests::test_intrusion_init ... ok
|
||||
test intrusion::tests::test_calibration_phase ... ok
|
||||
test intrusion::tests::test_arm_after_quiet ... ok
|
||||
test intrusion::tests::test_intrusion_detection ... ok
|
||||
test occupancy::tests::test_occupancy_detector_init ... ok
|
||||
test occupancy::tests::test_occupancy_calibration ... ok
|
||||
test occupancy::tests::test_occupancy_detection ... ok
|
||||
test vital_trend::tests::test_vital_trend_init ... ok
|
||||
test vital_trend::tests::test_normal_vitals_no_alerts ... ok
|
||||
test vital_trend::tests::test_apnea_detection ... ok
|
||||
test vital_trend::tests::test_tachycardia_detection ... ok
|
||||
test vital_trend::tests::test_breathing_average ... ok
|
||||
test rvf::builder::tests::test_build_rvf_roundtrip ... ok
|
||||
test rvf::builder::tests::test_build_hash_integrity ... ok
|
||||
```
|
||||
|
||||
### Test Coverage Notes
|
||||
|
||||
| Module | Tests | Coverage |
|
||||
|--------|-------|----------|
|
||||
| `gesture.rs` | 8 | Init, empty input, first frame, constant input, DTW identical/different/empty, ring buffer wrap, cooldown |
|
||||
| `coherence.rs` | 7 | Init, empty input, first frame, constant phases, incoherent phases, gate hysteresis, phasor angle |
|
||||
| `adversarial.rs` | 7 | Init, calibration, normal signal, phase jump, flatline, energy spike, cooldown |
|
||||
| `intrusion.rs` | 4 | Init, calibration, arming, intrusion detection |
|
||||
| `occupancy.rs` | 3 | Init, calibration, zone detection |
|
||||
| `vital_trend.rs` | 5 | Init, normal vitals, apnea, tachycardia, breathing average |
|
||||
| `rvf.rs` | 2 | Build roundtrip, hash integrity |
|
||||
|
||||
## Common Patterns
|
||||
|
||||
All seven core modules share these design patterns:
|
||||
|
||||
### 1. Const-constructible state
|
||||
|
||||
Every module's main struct can be created with `const fn new()`, which means it can be placed in a `static` variable without runtime initialization. This is essential for WASM modules where there is no allocator.
|
||||
|
||||
```rust
|
||||
static mut STATE: MyModule = MyModule::new();
|
||||
```
|
||||
|
||||
### 2. Calibration-then-detect lifecycle
|
||||
|
||||
Modules that need a baseline (`adversarial`, `intrusion`, `occupancy`) follow the same pattern: accumulate statistics for N frames, compute mean/variance, then switch to detection mode. The calibration frame count is always a compile-time constant.
|
||||
|
||||
### 3. Ring buffer for history
|
||||
|
||||
Both `gesture` (phase deltas) and `vital_trend` (BPM readings) use fixed-size ring buffers with modular index arithmetic. The pattern is:
|
||||
|
||||
```rust
|
||||
self.values[self.idx] = new_value;
|
||||
self.idx = (self.idx + 1) % MAX_SIZE;
|
||||
if self.len < MAX_SIZE { self.len += 1; }
|
||||
```
|
||||
|
||||
### 4. Static event buffers
|
||||
|
||||
Modules that return multiple events per frame (`intrusion`, `occupancy`, `vital_trend`) use `static mut` arrays as return buffers to avoid heap allocation. This is safe in single-threaded WASM but requires `unsafe` blocks. The pattern is:
|
||||
|
||||
```rust
|
||||
static mut EVENTS: [(i32, f32); N] = [(0, 0.0); N];
|
||||
let mut n_events = 0;
|
||||
// ... populate EVENTS[n_events] ...
|
||||
unsafe { &EVENTS[..n_events] }
|
||||
```
|
||||
|
||||
### 5. Cooldown/debounce
|
||||
|
||||
Every detection module uses a cooldown counter to prevent event flooding. After firing an event, the counter is set to a constant value and decremented each frame. No new events are emitted while the counter is positive.
|
||||
|
||||
### 6. EMA smoothing
|
||||
|
||||
Modules that track continuous scores (`coherence`, `occupancy`) use exponential moving average smoothing: `smoothed = alpha * raw + (1 - alpha) * smoothed`. The alpha constant controls responsiveness vs. stability.
|
||||
|
||||
### 7. Hysteresis thresholds
|
||||
|
||||
To prevent oscillation at detection boundaries, modules use different thresholds for entering and exiting a state. For example, the coherence monitor requires a score above 0.7 to enter Accept but only drops to Reject below 0.4.
|
||||
@@ -0,0 +1,78 @@
|
||||
é chip revision: v0.2
|
||||
I (34) boot.esp32s3: Boot SPI Speed : 80MHz
|
||||
I (38) boot.esp32s3: SPI Mode : DIO
|
||||
I (43) boot.esp32s3: SPI Flash Size : 8MB
|
||||
I (48) boot: Enabling RNG early entropy source...
|
||||
I (53) boot: Partition Table:
|
||||
I (57) boot: ## Label Usage Type ST Offset Length
|
||||
I (64) boot: 0 nvs WiFi data 01 02 00009000 00006000
|
||||
I (71) boot: 1 phy_init RF data 01 01 0000f000 00001000
|
||||
I (79) boot: 2 factory factory app 00 00 00010000 00100000
|
||||
I (86) boot: End of partition table
|
||||
I (91) esp_image: segment 0: paddr=00010020 vaddr=3c0b0020 size=2e5ach (189868) map
|
||||
I (133) esp_image: segment 1: paddr=0003e5d4 vaddr=3fc97e00 size=01a44h ( 6724) load
|
||||
I (135) esp_image: segment 2: paddr=00040020 vaddr=42000020 size=a0acch (658124) map
|
||||
I (257) esp_image: segment 3: paddr=000e0af4 vaddr=3fc99844 size=02bbch ( 11196) load
|
||||
I (260) esp_image: segment 4: paddr=000e36b8 vaddr=40374000 size=13d5ch ( 81244) load
|
||||
I (289) boot: Loaded app from partition at offset 0x10000
|
||||
I (289) boot: Disabling RNG early entropy source...
|
||||
I (300) cpu_start: Multicore app
|
||||
I (310) cpu_start: Pro cpu start user code
|
||||
I (310) cpu_start: cpu freq: 160000000 Hz
|
||||
I (310) cpu_start: Application information:
|
||||
I (313) cpu_start: Project name: esp32-csi-node
|
||||
I (319) cpu_start: App version: 1
|
||||
I (323) cpu_start: Compile time: Mar 3 2026 04:15:10
|
||||
I (329) cpu_start: ELF file SHA256: 50c89a9ed...
|
||||
I (334) cpu_start: ESP-IDF: v5.2
|
||||
I (339) cpu_start: Min chip rev: v0.0
|
||||
I (344) cpu_start: Max chip rev: v0.99
|
||||
I (349) cpu_start: Chip rev: v0.2
|
||||
I (353) heap_init: Initializing. RAM available for dynamic allocation:
|
||||
I (361) heap_init: At 3FCA9468 len 000402A8 (256 KiB): RAM
|
||||
I (367) heap_init: At 3FCE9710 len 00005724 (21 KiB): RAM
|
||||
I (373) heap_init: At 3FCF0000 len 00008000 (32 KiB): DRAM
|
||||
I (379) heap_init: At 600FE010 len 00001FD8 (7 KiB): RTCRAM
|
||||
I (386) spi_flash: detected chip: gd
|
||||
I (390) spi_flash: flash io: dio
|
||||
I (394) sleep: Configure to isolate all GPIO pins in sleep state
|
||||
I (400) sleep: Enable automatic switching of GPIO sleep configuration
|
||||
I (408) main_task: Started on CPU0
|
||||
I (412) main_task: Calling app_main()
|
||||
I (441) nvs_config: NVS override: ssid=ruv.net
|
||||
I (442) nvs_config: NVS override: password=***
|
||||
I (443) nvs_config: NVS override: target_ip=192.168.1.20
|
||||
I (448) nvs_config: NVS override: wasm_verify=0
|
||||
I (452) main: ESP32-S3 CSI Node (ADR-018) â?? Node ID: 1
|
||||
I (460) pp: pp rom version: e7ae62f
|
||||
I (462) net80211: net80211 rom version: e7ae62f
|
||||
I (469) wifi:wifi driver task: 3fcb3784, prio:23, stack:6656, core=0
|
||||
I (489) wifi:wifi firmware version: cc1dd81
|
||||
I (489) wifi:wifi certification version: v7.0
|
||||
I (489) wifi:config NVS flash: enabled
|
||||
I (490) wifi:config nano formating: disabled
|
||||
I (494) wifi:Init data frame dynamic rx buffer num: 32
|
||||
I (499) wifi:Init static rx mgmt buffer num: 5
|
||||
I (503) wifi:Init management short buffer num: 32
|
||||
I (507) wifi:Init dynamic tx buffer num: 32
|
||||
I (511) wifi:Init static tx FG buffer num: 2
|
||||
I (515) wifi:Init static rx buffer size: 2212
|
||||
I (519) wifi:Init static rx buffer num: 16
|
||||
I (523) wifi:Init dynamic rx buffer num: 32
|
||||
I (527) wifi_init: rx ba win: 16
|
||||
I (531) wifi_init: tcpip mbox: 32
|
||||
I (535) wifi_init: udp mbox: 32
|
||||
I (538) wifi_init: tcp mbox: 6
|
||||
I (542) wifi_init: tcp tx win: 5760
|
||||
I (546) wifi_init: tcp rx win: 5760
|
||||
I (550) wifi_init: tcp mss: 1440
|
||||
I (554) wifi_init: WiFi IRAM OP enabled
|
||||
I (559) wifi_init: WiFi RX IRAM OP enabled
|
||||
I (566) phy_init: phy_version 620,ec7ec30,Sep 5 2023,13:49:13
|
||||
I (612) wifi:mode : sta (3c:0f:02:ec:c2:28)
|
||||
I (612) wifi:enable tsf
|
||||
I (614) main: WiFi STA initialized, connecting to SSID: ruv.net
|
||||
I (623) wifi:new:<5,0>, old:<1,0>, ap:<255,255>, sta:<5,0>, prof:1
|
||||
I (625) wifi:state: init -> auth (b0)
|
||||
I (656) wifi:state: auth -> assoc (0)
|
||||
I (749) wifi:state: assoc -> run (10)
|
||||
@@ -0,0 +1,645 @@
|
||||
# Exotic & Research Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Experimental sensing applications that push the boundaries of what WiFi
|
||||
> signals can detect. From contactless sleep staging to sign language
|
||||
> recognition, these modules explore novel uses of RF sensing. Some are
|
||||
> highly experimental -- marked with their maturity level.
|
||||
|
||||
## Maturity Levels
|
||||
|
||||
- **Proven**: Based on published research with validated results
|
||||
- **Experimental**: Working implementation, needs real-world validation
|
||||
- **Research**: Proof of concept, exploratory
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Maturity |
|
||||
|--------|------|-------------|-----------|----------|
|
||||
| Sleep Stage Classification | `exo_dream_stage.rs` | Classifies sleep phases from breathing + micro-movements | 600-603 | Experimental |
|
||||
| Emotion Detection | `exo_emotion_detect.rs` | Estimates arousal/stress from physiological proxies | 610-613 | Research |
|
||||
| Sign Language Recognition | `exo_gesture_language.rs` | DTW-based letter recognition from hand/arm CSI patterns | 620-623 | Research |
|
||||
| Music Conductor Tracking | `exo_music_conductor.rs` | Extracts tempo, beat, dynamics from conducting motions | 630-634 | Research |
|
||||
| Plant Growth Detection | `exo_plant_growth.rs` | Detects plant growth drift and circadian leaf movement | 640-643 | Research |
|
||||
| Ghost Hunter (Anomaly) | `exo_ghost_hunter.rs` | Classifies unexplained perturbations in empty rooms | 650-653 | Experimental |
|
||||
| Rain Detection | `exo_rain_detect.rs` | Detects rain from broadband structural vibrations | 660-662 | Experimental |
|
||||
| Breathing Synchronization | `exo_breathing_sync.rs` | Detects phase-locked breathing between multiple people | 670-673 | Research |
|
||||
| Time Crystal Detection | `exo_time_crystal.rs` | Detects period-doubling and temporal coordination | 680-682 | Research |
|
||||
| Hyperbolic Space Embedding | `exo_hyperbolic_space.rs` | Poincare ball location classification with hierarchy | 685-687 | Research |
|
||||
|
||||
## Architecture
|
||||
|
||||
All modules share these design constraints:
|
||||
|
||||
- **`no_std`** -- no heap allocation, runs on WASM3 interpreter on ESP32-S3
|
||||
- **`const fn new()`** -- all state is stack-allocated and const-constructible
|
||||
- **Static event buffer** -- events are returned via `&[(i32, f32)]` from a static array (max 3-5 events per frame)
|
||||
- **Budget-aware** -- each module declares its per-frame time budget (L/S/H)
|
||||
- **Frame rate** -- all modules assume 20 Hz CSI frame rate from the host Tier 2 DSP
|
||||
|
||||
Shared utilities from `vendor_common.rs`:
|
||||
- `CircularBuffer<N>` -- fixed-size ring buffer with O(1) push and indexed access
|
||||
- `Ema` -- exponential moving average with configurable alpha
|
||||
- `WelfordStats` -- online mean/variance computation (Welford's algorithm)
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Sleep Stage Classification (`exo_dream_stage.rs`)
|
||||
|
||||
**What it does**: Classifies sleep phases (Awake, NREM Light, NREM Deep, REM) from breathing patterns, heart rate variability, and micro-movements -- without touching the person.
|
||||
|
||||
**Maturity**: Experimental
|
||||
|
||||
**Research basis**: WiFi-based contactless sleep monitoring has been demonstrated in peer-reviewed research. See [1] for RF-based sleep staging using breathing patterns and body movement.
|
||||
|
||||
#### How It Works
|
||||
|
||||
The module uses a four-feature state machine with hysteresis:
|
||||
|
||||
1. **Breathing regularity** -- Coefficient of variation (CV) of a 64-sample breathing BPM window. Low CV (<0.08) indicates deep sleep; high CV (>0.20) indicates REM or wakefulness.
|
||||
|
||||
2. **Motion energy** -- EMA-smoothed motion from host Tier 2. Below 0.15 = sleep-like; above 0.5 = awake.
|
||||
|
||||
3. **Heart rate variability (HRV)** -- Variance of recent HR BPM values. High HRV (>8.0) correlates with REM; very low HRV (<2.0) with deep sleep.
|
||||
|
||||
4. **Phase micro-movements** -- High-pass energy of the phase signal (successive differences). Captures muscle atonia disruption during REM.
|
||||
|
||||
Stage transitions require 10 consecutive frames of the candidate stage (hysteresis), preventing jittery classification.
|
||||
|
||||
#### Sleep Stages
|
||||
|
||||
| Stage | Code | Conditions |
|
||||
|-------|------|-----------|
|
||||
| Awake | 0 | No presence, high motion, or moderate motion + irregular breathing |
|
||||
| NREM Light | 1 | Low motion, moderate breathing regularity, default sleep state |
|
||||
| NREM Deep | 2 | Very low motion, very regular breathing (CV < 0.08), low HRV (< 2.0) |
|
||||
| REM | 3 | Very low motion, high HRV (> 8.0), micro-movements above threshold |
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `SLEEP_STAGE` | 600 | 0-3 (Awake/Light/Deep/REM) | Every frame (after warmup) |
|
||||
| `SLEEP_QUALITY` | 601 | Sleep efficiency [0, 100] | Every 20 frames |
|
||||
| `REM_EPISODE` | 602 | Current/last REM episode length (frames) | When REM active or just ended |
|
||||
| `DEEP_SLEEP_RATIO` | 603 | Deep/total sleep ratio [0, 1] | Every 20 frames |
|
||||
|
||||
#### Quality Metrics
|
||||
|
||||
- **Efficiency** = (sleep_frames / total_frames) * 100
|
||||
- **Deep ratio** = deep_frames / sleep_frames
|
||||
- **REM ratio** = rem_frames / sleep_frames
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Parameter | Default | Description |
|
||||
|-----------|---------|-------------|
|
||||
| `BREATH_HIST_LEN` | 64 | Rolling window for breathing BPM history |
|
||||
| `HR_HIST_LEN` | 64 | Rolling window for heart rate history |
|
||||
| `PHASE_BUF_LEN` | 128 | Phase buffer for micro-movement detection |
|
||||
| `MOTION_ALPHA` | 0.1 | Motion EMA smoothing factor |
|
||||
| `MIN_WARMUP` | 40 | Minimum frames before classification begins |
|
||||
| `STAGE_HYSTERESIS` | 10 | Consecutive frames required for stage transition |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = DreamStageDetector::new();
|
||||
let events = detector.process_frame(
|
||||
breathing_bpm, // f32: from Tier 2 DSP
|
||||
heart_rate_bpm, // f32: from Tier 2 DSP
|
||||
motion_energy, // f32: from Tier 2 DSP
|
||||
phase, // f32: representative subcarrier phase
|
||||
variance, // f32: representative subcarrier variance
|
||||
presence, // i32: 1 if person detected, 0 otherwise
|
||||
);
|
||||
// events: &[(i32, f32)] -- event ID + value pairs
|
||||
|
||||
let stage = detector.stage(); // SleepStage enum
|
||||
let eff = detector.efficiency(); // f32 [0, 100]
|
||||
let deep = detector.deep_ratio(); // f32 [0, 1]
|
||||
let rem = detector.rem_ratio(); // f32 [0, 1]
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up Contactless Sleep Tracking
|
||||
|
||||
1. **Placement**: Mount the WiFi transmitter and receiver so the line of sight crosses the bed at chest height. Place the ESP32 node 1-3 meters from the bed.
|
||||
|
||||
2. **Calibration**: Let the system run for 40+ frames (2 seconds at 20 Hz) with the person in bed before expecting valid stage classifications.
|
||||
|
||||
3. **Interpreting Results**: Monitor `SLEEP_STAGE` events. A healthy sleep cycle progresses through Light -> Deep -> Light -> REM, repeating in ~90 minute cycles. The `SLEEP_QUALITY` event (601) gives an overall efficiency percentage -- above 85% is considered good.
|
||||
|
||||
4. **Limitations**: The module requires the Tier 2 DSP to provide valid `breathing_bpm` and `heart_rate_bpm`. If the person is too far from the WiFi path or behind thick walls, these vitals may not be detectable.
|
||||
|
||||
---
|
||||
|
||||
### Emotion Detection (`exo_emotion_detect.rs`)
|
||||
|
||||
**What it does**: Estimates continuous arousal level and discrete stress/calm/agitation states from WiFi CSI without cameras or microphones. Uses physiological proxies: breathing rate, heart rate, fidgeting, and phase variance.
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Limitations**: This module does NOT detect emotions directly. It detects physiological arousal -- elevated heart rate, rapid breathing, and fidgeting. These correlate with stress and anxiety but can also be caused by exercise, caffeine, or excitement. The module cannot distinguish between positive and negative arousal. It is a research tool for exploring the feasibility of affect sensing via RF, not a clinical instrument.
|
||||
|
||||
#### How It Works
|
||||
|
||||
The arousal level is a weighted sum of four normalized features:
|
||||
|
||||
| Feature | Weight | Source | Score = 0 | Score = 1 |
|
||||
|---------|--------|--------|-----------|-----------|
|
||||
| Breathing rate | 0.30 | Host Tier 2 | 6-10 BPM (calm) | >= 20 BPM (stressed) |
|
||||
| Heart rate | 0.20 | Host Tier 2 | <= 70 BPM (baseline) | 100+ BPM (elevated) |
|
||||
| Fidget energy | 0.30 | Motion successive diffs | No fidgeting | Continuous fidgeting |
|
||||
| Phase variance | 0.20 | Subcarrier variance | Stable signal | Sharp body movements |
|
||||
|
||||
The stress index uses different weights (0.4/0.3/0.2/0.1) emphasizing breathing and heart rate over fidgeting.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `AROUSAL_LEVEL` | 610 | Continuous arousal [0, 1] | Every frame |
|
||||
| `STRESS_INDEX` | 611 | Stress index [0, 1] | Every frame |
|
||||
| `CALM_DETECTED` | 612 | 1.0 when calm state detected | When conditions met |
|
||||
| `AGITATION_DETECTED` | 613 | 1.0 when agitation detected | When conditions met |
|
||||
|
||||
#### Discrete State Detection
|
||||
|
||||
- **Calm**: arousal < 0.25 AND motion < 0.08 AND breathing 6-10 BPM AND breath CV < 0.08
|
||||
- **Agitation**: arousal > 0.75 AND (motion > 0.6 OR fidget > 0.15 OR breath CV > 0.25)
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = EmotionDetector::new();
|
||||
let events = detector.process_frame(
|
||||
breathing_bpm, // f32
|
||||
heart_rate_bpm, // f32
|
||||
motion_energy, // f32
|
||||
phase, // f32 (unused in current implementation)
|
||||
variance, // f32
|
||||
);
|
||||
|
||||
let arousal = detector.arousal(); // f32 [0, 1]
|
||||
let stress = detector.stress_index(); // f32 [0, 1]
|
||||
let calm = detector.is_calm(); // bool
|
||||
let agitated = detector.is_agitated(); // bool
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Sign Language Recognition (`exo_gesture_language.rs`)
|
||||
|
||||
**What it does**: Classifies hand/arm movements into sign language letter groups using WiFi CSI phase and amplitude patterns. Uses DTW (Dynamic Time Warping) template matching on compact 6D feature sequences.
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Limitations**: Full 26-letter ASL alphabet recognition via WiFi is extremely challenging. This module provides a proof-of-concept framework. Real-world accuracy depends heavily on: (a) template quality and diversity, (b) environmental stability, (c) person-to-person variation. Expect proof-of-concept accuracy, not production ASL translation.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Feature extraction**: Per frame, compute 6 features: mean phase, phase spread, mean amplitude, amplitude spread, motion energy, variance. These are accumulated in a gesture window (max 32 frames).
|
||||
|
||||
2. **Gesture segmentation**: Active gestures are bounded by pauses (low motion for 15+ frames). When a pause is detected, the accumulated gesture window is matched against templates.
|
||||
|
||||
3. **DTW matching**: Each template is a reference feature sequence. Multivariate DTW with Sakoe-Chiba band (width=4) computes the alignment distance. The best match below threshold (0.5) is accepted.
|
||||
|
||||
4. **Word boundaries**: Extended pauses (15+ low-motion frames) emit word boundary events.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `LETTER_RECOGNIZED` | 620 | Letter index (0=A, ..., 25=Z) | On match after pause |
|
||||
| `LETTER_CONFIDENCE` | 621 | Inverse DTW distance [0, 1] | With recognized letter |
|
||||
| `WORD_BOUNDARY` | 622 | 1.0 | After extended pause |
|
||||
| `GESTURE_REJECTED` | 623 | 1.0 | When gesture does not match |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = GestureLanguageDetector::new();
|
||||
|
||||
// Load templates (required before recognition works)
|
||||
detector.load_synthetic_templates(); // 26 ramp-pattern templates for testing
|
||||
// OR load custom templates:
|
||||
detector.set_template(0, &features_for_letter_a); // 0 = 'A'
|
||||
|
||||
let events = detector.process_frame(
|
||||
&phases, // &[f32]: per-subcarrier phase
|
||||
&litudes, // &[f32]: per-subcarrier amplitude
|
||||
variance, // f32
|
||||
motion_energy, // f32
|
||||
presence, // i32
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Music Conductor Tracking (`exo_music_conductor.rs`)
|
||||
|
||||
**What it does**: Extracts musical conducting parameters from WiFi CSI motion signatures: tempo (BPM), beat position (1-4 in 4/4 time), dynamic level (MIDI velocity 0-127), and special gestures (cutoff and fermata).
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Research basis**: Gesture tracking via WiFi CSI has been demonstrated for coarse arm movements. Conductor tracking extends this to periodic rhythmic motion analysis.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Tempo detection**: Autocorrelation of a 128-point motion energy buffer at lags 4-64. The dominant peak determines the period, converted to BPM: `BPM = 60 * 20 / lag` (at 20 Hz frame rate). Valid range: 30-240 BPM.
|
||||
|
||||
2. **Beat position**: A modular frame counter relative to the detected period maps to beats 1-4 in 4/4 time.
|
||||
|
||||
3. **Dynamic level**: Motion energy relative to the EMA-smoothed peak, scaled to MIDI velocity [0, 127].
|
||||
|
||||
4. **Cutoff detection**: Sharp drop in motion energy (ratio < 0.2 of recent peak) with high preceding motion.
|
||||
|
||||
5. **Fermata detection**: Sustained low motion (< 0.05) for 10+ consecutive frames.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `CONDUCTOR_BPM` | 630 | Detected tempo in BPM | After tempo lock |
|
||||
| `BEAT_POSITION` | 631 | Beat number (1-4) | After tempo lock |
|
||||
| `DYNAMIC_LEVEL` | 632 | MIDI velocity [0, 127] | Every frame |
|
||||
| `GESTURE_CUTOFF` | 633 | 1.0 | On cutoff gesture |
|
||||
| `GESTURE_FERMATA` | 634 | 1.0 | During fermata hold |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = MusicConductorDetector::new();
|
||||
let events = detector.process_frame(
|
||||
phase, // f32 (unused)
|
||||
amplitude, // f32 (unused)
|
||||
motion_energy, // f32: from Tier 2 DSP
|
||||
variance, // f32 (unused)
|
||||
);
|
||||
|
||||
let bpm = detector.tempo_bpm(); // f32
|
||||
let fermata = detector.is_fermata(); // bool
|
||||
let cutoff = detector.is_cutoff(); // bool
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Plant Growth Detection (`exo_plant_growth.rs`)
|
||||
|
||||
**What it does**: Detects plant growth and leaf movement from micro-CSI changes over hours/days. Plants cause extremely slow, monotonic drift in CSI amplitude (growth) and diurnal phase oscillations (circadian leaf movement -- nyctinasty).
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Requirements**: Room must be empty (`presence == 0`) to isolate plant-scale perturbations from human motion. This module is designed for long-running monitoring (hours to days).
|
||||
|
||||
#### How It Works
|
||||
|
||||
- **Growth rate**: Tracks the slow drift of amplitude baseline via a very slow EWMA (alpha=0.0001, half-life ~175 seconds). Plant growth produces continuous ~0.01 dB/hour amplitude decrease as new leaf area intercepts RF energy.
|
||||
|
||||
- **Circadian phase**: Tracks peak-to-trough oscillation in phase EWMA over a rolling window. Nyctinastic leaf movement (folding at night) produces ~24-hour oscillations.
|
||||
|
||||
- **Wilting detection**: Short-term amplitude rises above baseline (less absorption) combined with reduced phase variance.
|
||||
|
||||
- **Watering event**: Abrupt amplitude drop (more water = more RF absorption) followed by recovery.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `GROWTH_RATE` | 640 | Amplitude drift rate (scaled) | Every 100 empty-room frames |
|
||||
| `CIRCADIAN_PHASE` | 641 | Oscillation magnitude [0, 1] | When oscillation detected |
|
||||
| `WILT_DETECTED` | 642 | 1.0 | When wilting signature seen |
|
||||
| `WATERING_EVENT` | 643 | 1.0 | When watering signature seen |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = PlantGrowthDetector::new();
|
||||
let events = detector.process_frame(
|
||||
&litudes, // &[f32]: per-subcarrier amplitudes (up to 32)
|
||||
&phases, // &[f32]: per-subcarrier phases (up to 32)
|
||||
&variance, // &[f32]: per-subcarrier variance (up to 32)
|
||||
presence, // i32: 0 = empty room (required for detection)
|
||||
);
|
||||
|
||||
let calibrated = detector.is_calibrated(); // true after MIN_EMPTY_FRAMES
|
||||
let empty = detector.empty_frames(); // frames of empty-room data
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Ghost Hunter -- Environmental Anomaly Detector (`exo_ghost_hunter.rs`)
|
||||
|
||||
**What it does**: Monitors CSI when no humans are detected for any perturbation above the noise floor. When the room should be empty but CSI changes are detected, something unexplained is happening. Classifies anomalies by their temporal signature.
|
||||
|
||||
**Maturity**: Experimental
|
||||
|
||||
**Practical applications**: Despite the playful name, this module has serious uses: detecting HVAC compressor cycling, pest/animal movement, structural settling, gas leaks (which alter dielectric properties), hidden intruders who evade the primary presence detector, and electromagnetic interference.
|
||||
|
||||
#### Anomaly Classification
|
||||
|
||||
| Class | Code | Signature | Typical Sources |
|
||||
|-------|------|-----------|----------------|
|
||||
| Impulsive | 1 | < 5 frames, sharp transient | Object falling, thermal cracking |
|
||||
| Periodic | 2 | Recurring, detectable autocorrelation peak | HVAC, appliances, pest movement |
|
||||
| Drift | 3 | 30+ frames same-sign amplitude delta | Temperature change, humidity, gas leak |
|
||||
| Random | 4 | Stochastic, no pattern | EMI, co-channel WiFi interference |
|
||||
|
||||
#### Hidden Presence Detection
|
||||
|
||||
A sub-detector looks for breathing signatures in the phase signal: periodic oscillation at 0.2-2.0 Hz via autocorrelation at lags 5-15 (at 20 Hz frame rate). This can detect a motionless person who evades the main presence detector.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `ANOMALY_DETECTED` | 650 | Energy level [0, 1] | When anomaly active |
|
||||
| `ANOMALY_CLASS` | 651 | 1-4 (see table above) | With anomaly detection |
|
||||
| `HIDDEN_PRESENCE` | 652 | Confidence [0, 1] | When breathing signature found |
|
||||
| `ENVIRONMENTAL_DRIFT` | 653 | Drift magnitude | When sustained drift detected |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = GhostHunterDetector::new();
|
||||
let events = detector.process_frame(
|
||||
&phases, // &[f32]
|
||||
&litudes, // &[f32]
|
||||
&variance, // &[f32]
|
||||
presence, // i32: must be 0 for detection
|
||||
motion_energy, // f32
|
||||
);
|
||||
|
||||
let class = detector.anomaly_class(); // AnomalyClass enum
|
||||
let hidden = detector.hidden_presence_confidence(); // f32 [0, 1]
|
||||
let energy = detector.anomaly_energy(); // f32
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Rain Detection (`exo_rain_detect.rs`)
|
||||
|
||||
**What it does**: Detects rain from broadband CSI phase variance perturbations caused by raindrop impacts on building surfaces. Classifies intensity as light, moderate, or heavy.
|
||||
|
||||
**Maturity**: Experimental
|
||||
|
||||
**Research basis**: Raindrops impacting surfaces produce broadband impulse vibrations that propagate through building structure and modulate CSI phase. These are distinguishable from human motion by their broadband nature (all subcarrier groups affected equally), stochastic timing, and small amplitude.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Requires empty room** (`presence == 0`) to avoid confounding with human motion.
|
||||
2. **Broadband criterion**: Compute per-group variance ratio (short-term / baseline). If >= 75% of groups (6/8) have elevated variance (ratio > 2.5x), the signal is broadband -- consistent with rain.
|
||||
3. **Hysteresis state machine**: Onset requires 10 consecutive broadband frames; cessation requires 20 consecutive quiet frames.
|
||||
4. **Intensity classification**: Based on smoothed excess energy above baseline.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `RAIN_ONSET` | 660 | 1.0 | On rain start |
|
||||
| `RAIN_INTENSITY` | 661 | 1=light, 2=moderate, 3=heavy | While raining |
|
||||
| `RAIN_CESSATION` | 662 | 1.0 | On rain stop |
|
||||
|
||||
#### Intensity Thresholds
|
||||
|
||||
| Level | Code | Energy Range |
|
||||
|-------|------|-------------|
|
||||
| None | 0 | (not raining) |
|
||||
| Light | 1 | energy < 0.3 |
|
||||
| Moderate | 2 | 0.3 <= energy < 0.7 |
|
||||
| Heavy | 3 | energy >= 0.7 |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = RainDetector::new();
|
||||
let events = detector.process_frame(
|
||||
&phases, // &[f32]
|
||||
&variance, // &[f32]
|
||||
&litudes, // &[f32]
|
||||
presence, // i32: must be 0
|
||||
);
|
||||
|
||||
let raining = detector.is_raining(); // bool
|
||||
let intensity = detector.intensity(); // RainIntensity enum
|
||||
let energy = detector.energy(); // f32 [0, 1]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Breathing Synchronization (`exo_breathing_sync.rs`)
|
||||
|
||||
**What it does**: Detects when multiple people's breathing patterns synchronize. Extracts per-person breathing components via subcarrier group decomposition and computes pairwise normalized cross-correlation.
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Research basis**: Breathing synchronization (interpersonal physiological synchrony) is a known phenomenon in couples, parent-infant pairs, and close social groups. This module attempts to detect it contactlessly via WiFi CSI.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Per-person decomposition**: With N persons, the 8 subcarrier groups are divided among persons (e.g., 2 persons = 4 groups each). Each person's phase signal is bandpass-filtered to the breathing band using dual EWMA (DC removal + low-pass).
|
||||
|
||||
2. **Pairwise correlation**: For each pair, compute normalized zero-lag cross-correlation over a 64-sample buffer: `rho = sum(x_i * x_j) / sqrt(sum(x_i^2) * sum(x_j^2))`
|
||||
|
||||
3. **Synchronization state machine**: High correlation (|rho| > 0.6) for 20+ consecutive frames declares synchronization. Low correlation for 15+ frames declares sync lost.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `SYNC_DETECTED` | 670 | 1.0 | On sync onset |
|
||||
| `SYNC_PAIR_COUNT` | 671 | Number of synced pairs | On count change |
|
||||
| `GROUP_COHERENCE` | 672 | Average coherence [0, 1] | Every 10 frames |
|
||||
| `SYNC_LOST` | 673 | 1.0 | On sync loss |
|
||||
|
||||
#### Constraints
|
||||
|
||||
- Maximum 4 persons (6 pairwise comparisons)
|
||||
- Requires >= 8 subcarriers and >= 2 persons
|
||||
- 64-frame warmup before analysis begins
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = BreathingSyncDetector::new();
|
||||
let events = detector.process_frame(
|
||||
&phases, // &[f32]: per-subcarrier phases
|
||||
&variance, // &[f32]: per-subcarrier variance
|
||||
breathing_bpm, // f32: host aggregate (unused internally)
|
||||
n_persons, // i32: number of persons detected
|
||||
);
|
||||
|
||||
let synced = detector.is_synced(); // bool
|
||||
let coherence = detector.group_coherence(); // f32 [0, 1]
|
||||
let persons = detector.active_persons(); // usize
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Time Crystal Detection (`exo_time_crystal.rs`)
|
||||
|
||||
**What it does**: Detects temporal symmetry breaking patterns -- specifically period doubling -- in motion energy. A "time crystal" in this context is when the system oscillates at a sub-harmonic of the driving frequency. Also counts independent non-harmonic periodic components as a "coordination index" for multi-person temporal coordination.
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Background**: In condensed matter physics, discrete time crystals exhibit period doubling under periodic driving. This module applies the same mathematical criterion (autocorrelation peak at lag L AND lag 2L) to human motion patterns. Two people walking at different cadences produce independent periodic peaks at non-harmonic ratios.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Autocorrelation**: 256-point motion energy buffer, autocorrelation at lags 1-128. Pre-linearized for performance (eliminates modulus ops in inner loop).
|
||||
|
||||
2. **Period doubling**: Search for peaks where a strong autocorrelation at lag L is accompanied by a strong peak at lag 2L (+/- 2 frame tolerance).
|
||||
|
||||
3. **Coordination index**: Count peaks whose lag ratios are not integer multiples of any other peak (within 5% tolerance). These represent independent periodic motions.
|
||||
|
||||
4. **Stability tracking**: Crystal detection is tracked over 200-frame windows. The stability score is the fraction of frames where the crystal was detected, EMA-smoothed.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `CRYSTAL_DETECTED` | 680 | Period multiplier (2 = doubling) | When detected |
|
||||
| `CRYSTAL_STABILITY` | 681 | Stability score [0, 1] | Every frame |
|
||||
| `COORDINATION_INDEX` | 682 | Non-harmonic peak count | When > 0 |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut detector = TimeCrystalDetector::new();
|
||||
let events = detector.process_frame(motion_energy);
|
||||
|
||||
let detected = detector.is_detected(); // bool
|
||||
let multiplier = detector.multiplier(); // u8 (0 or 2)
|
||||
let stability = detector.stability(); // f32 [0, 1]
|
||||
let coordination = detector.coordination_index(); // u8
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Hyperbolic Space Embedding (`exo_hyperbolic_space.rs`)
|
||||
|
||||
**What it does**: Embeds CSI fingerprints into a 2D Poincare disk to exploit the natural hierarchy of indoor spaces (rooms contain zones). Hyperbolic geometry provides exponentially more representational capacity near the boundary, ideal for tree-structured location taxonomies.
|
||||
|
||||
**Maturity**: Research
|
||||
|
||||
**Research basis**: Hyperbolic embeddings have been shown to outperform Euclidean embeddings for hierarchical data (Nickel & Kiela, 2017). This module applies the concept to indoor localization.
|
||||
|
||||
#### How It Works
|
||||
|
||||
1. **Feature extraction**: 8D vector from mean amplitude across 8 subcarrier groups.
|
||||
2. **Linear projection**: 2x8 matrix maps features to 2D Poincare disk coordinates.
|
||||
3. **Normalization**: If the projected point exceeds the disk boundary, scale to radius 0.95.
|
||||
4. **Nearest reference**: Compute Poincare distance to 16 reference points and find the closest.
|
||||
5. **Hierarchy level**: Points near the center (radius < 0.5) are room-level; near the boundary are zone-level.
|
||||
|
||||
#### Poincare Distance
|
||||
|
||||
```
|
||||
d(x, y) = acosh(1 + 2 * ||x-y||^2 / ((1 - ||x||^2) * (1 - ||y||^2)))
|
||||
```
|
||||
|
||||
This metric respects the hyperbolic geometry: distances near the boundary grow exponentially.
|
||||
|
||||
#### Default Reference Layout
|
||||
|
||||
| Index | Label | Radius | Description |
|
||||
|-------|-------|--------|-------------|
|
||||
| 0-3 | Rooms | 0.3 | Bathroom, Kitchen, Living room, Bedroom |
|
||||
| 4-6 | Zone 0a-c | 0.7 | Bathroom sub-zones |
|
||||
| 7-9 | Zone 1a-c | 0.7 | Kitchen sub-zones |
|
||||
| 10-12 | Zone 2a-c | 0.7 | Living room sub-zones |
|
||||
| 13-15 | Zone 3a-c | 0.7 | Bedroom sub-zones |
|
||||
|
||||
#### Events
|
||||
|
||||
| Event | ID | Value | Frequency |
|
||||
|-------|-----|-------|-----------|
|
||||
| `HIERARCHY_LEVEL` | 685 | 0 = room, 1 = zone | Every frame |
|
||||
| `HYPERBOLIC_RADIUS` | 686 | Disk radius [0, 1) | Every frame |
|
||||
| `LOCATION_LABEL` | 687 | Nearest reference (0-15) | Every frame |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
let mut embedder = HyperbolicEmbedder::new();
|
||||
let events = embedder.process_frame(&litudes);
|
||||
|
||||
let label = embedder.label(); // u8 (0-15)
|
||||
let pos = embedder.position(); // &[f32; 2]
|
||||
|
||||
// Custom calibration:
|
||||
embedder.set_reference(0, [0.2, 0.1]);
|
||||
embedder.set_projection_row(0, [0.05, 0.03, 0.02, 0.01, -0.01, -0.02, -0.03, -0.04]);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Event ID Registry (600-699)
|
||||
|
||||
| Range | Module | Events |
|
||||
|-------|--------|--------|
|
||||
| 600-603 | Dream Stage | SLEEP_STAGE, SLEEP_QUALITY, REM_EPISODE, DEEP_SLEEP_RATIO |
|
||||
| 610-613 | Emotion Detect | AROUSAL_LEVEL, STRESS_INDEX, CALM_DETECTED, AGITATION_DETECTED |
|
||||
| 620-623 | Gesture Language | LETTER_RECOGNIZED, LETTER_CONFIDENCE, WORD_BOUNDARY, GESTURE_REJECTED |
|
||||
| 630-634 | Music Conductor | CONDUCTOR_BPM, BEAT_POSITION, DYNAMIC_LEVEL, GESTURE_CUTOFF, GESTURE_FERMATA |
|
||||
| 640-643 | Plant Growth | GROWTH_RATE, CIRCADIAN_PHASE, WILT_DETECTED, WATERING_EVENT |
|
||||
| 650-653 | Ghost Hunter | ANOMALY_DETECTED, ANOMALY_CLASS, HIDDEN_PRESENCE, ENVIRONMENTAL_DRIFT |
|
||||
| 660-662 | Rain Detect | RAIN_ONSET, RAIN_INTENSITY, RAIN_CESSATION |
|
||||
| 670-673 | Breathing Sync | SYNC_DETECTED, SYNC_PAIR_COUNT, GROUP_COHERENCE, SYNC_LOST |
|
||||
| 680-682 | Time Crystal | CRYSTAL_DETECTED, CRYSTAL_STABILITY, COORDINATION_INDEX |
|
||||
| 685-687 | Hyperbolic Space | HIERARCHY_LEVEL, HYPERBOLIC_RADIUS, LOCATION_LABEL |
|
||||
|
||||
## Code Quality Notes
|
||||
|
||||
All 10 modules have been reviewed for:
|
||||
|
||||
- **Edge cases**: Division by zero is guarded everywhere (explicit checks before division, EPSILON constants). Negative variance from floating-point rounding is clamped to zero. Empty buffers return safe defaults.
|
||||
- **NaN protection**: All computations use `libm` functions (`sqrtf`, `acoshf`, `sinf`) which are well-defined for valid inputs. Inputs are validated before reaching math functions.
|
||||
- **Buffer safety**: All `CircularBuffer` accesses use the `get(i)` method which returns 0.0 for out-of-bounds indices. Fixed-size arrays prevent overflow.
|
||||
- **Range clamping**: All outputs that represent ratios or probabilities are clamped to [0, 1]. MIDI velocity is clamped to [0, 127]. Poincare disk coordinates are normalized to radius < 1.
|
||||
- **Test coverage**: Each module has 7-10 tests covering: construction, warmup period, happy path detection, edge cases (no presence, insufficient data), range validation, and reset.
|
||||
|
||||
## Research References
|
||||
|
||||
1. Liu, J., et al. "Monitoring Vital Signs and Postures During Sleep Using WiFi Signals." IEEE Internet of Things Journal, 2018. -- WiFi-based sleep monitoring using CSI breathing patterns.
|
||||
2. Zhao, M., et al. "Through-Wall Human Pose Estimation Using Radio Signals." CVPR 2018. -- RF-based pose estimation foundations.
|
||||
3. Wang, H., et al. "RT-Fall: A Real-Time and Contactless Fall Detection System with Commodity WiFi Devices." IEEE Transactions on Mobile Computing, 2017. -- WiFi CSI for human activity recognition.
|
||||
4. Li, H., et al. "WiFinger: Talk to Your Smart Devices with Finger Gesture." UbiComp 2016. -- WiFi-based gesture recognition using CSI.
|
||||
5. Ma, Y., et al. "SignFi: Sign Language Recognition Using WiFi." ACM IMWUT, 2018. -- WiFi CSI for sign language.
|
||||
6. Nickel, M. & Kiela, D. "Poincare Embeddings for Learning Hierarchical Representations." NeurIPS 2017. -- Hyperbolic embedding foundations.
|
||||
7. Wang, W., et al. "Understanding and Modeling of WiFi Signal Based Human Activity Recognition." MobiCom 2015. -- CSI-based activity recognition.
|
||||
8. Adib, F., et al. "Smart Homes that Monitor Breathing and Heart Rate." CHI 2015. -- Contactless vital sign monitoring via RF signals.
|
||||
|
||||
## Contributing New Research Modules
|
||||
|
||||
### Adding a New Exotic Module
|
||||
|
||||
1. **Choose an event ID range**: Use the next available range in the 600-699 block. Check `lib.rs` event_types for allocated IDs.
|
||||
|
||||
2. **Create the source file**: Name it `exo_<name>.rs` in `src/`. Follow the existing pattern:
|
||||
- Module-level doc comment with algorithm description, events, and budget
|
||||
- `const fn new()` constructor
|
||||
- `process_frame()` returning `&[(i32, f32)]` via static buffer
|
||||
- Public accessor methods for key state
|
||||
- `reset()` method
|
||||
|
||||
3. **Register in `lib.rs`**: Add `pub mod exo_<name>;` in the Category 6 section.
|
||||
|
||||
4. **Register event constants**: Add entries to `event_types` in `lib.rs`.
|
||||
|
||||
5. **Update this document**: Add the module to the overview table and write its section.
|
||||
|
||||
6. **Testing requirements**:
|
||||
- At minimum: `test_const_new`, `test_warmup_no_events`, one happy-path detection test, `test_reset`
|
||||
- Test edge cases: empty input, extreme values, insufficient data
|
||||
- Verify all output values are in their documented ranges
|
||||
- Run: `cargo test --features std -- exo_` (from within the wasm-edge crate directory)
|
||||
|
||||
### Design Constraints
|
||||
|
||||
- **`no_std`**: No heap allocation. Use `CircularBuffer`, `Ema`, `WelfordStats` from `vendor_common`.
|
||||
- **Stack budget**: Keep total struct size reasonable. The ESP32-S3 WASM3 stack is limited.
|
||||
- **Time budget**: Stay within your declared budget (L < 2ms, S < 5ms, H < 10ms at 20 Hz).
|
||||
- **Static events**: Use a `static mut EVENTS` array for zero-allocation event returns.
|
||||
- **Input validation**: Always check array lengths, handle missing data gracefully.
|
||||
@@ -0,0 +1,832 @@
|
||||
# Industrial & Specialized Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Worker safety and compliance monitoring using WiFi CSI signals. Works through
|
||||
> dust, smoke, shelving, and walls where cameras fail. Designed for warehouses,
|
||||
> factories, clean rooms, farms, and construction sites.
|
||||
|
||||
**ADR-041 Category 5 | Event IDs 500--599 | Crate `wifi-densepose-wasm-edge`**
|
||||
|
||||
## Safety Warning
|
||||
|
||||
These modules are **supplementary monitoring tools**. They do NOT replace:
|
||||
|
||||
- Certified safety systems (SIL-rated controllers, safety PLCs)
|
||||
- Gas detectors, O2 monitors, or LEL sensors
|
||||
- OSHA-required personal protective equipment
|
||||
- Physical barriers, guardrails, or interlocks
|
||||
- Trained safety attendants or rescue teams
|
||||
|
||||
Always deploy alongside certified primary safety systems. WiFi CSI sensing is
|
||||
susceptible to environmental changes (new metal objects, humidity, temperature)
|
||||
that can cause false negatives. Calibrate regularly and validate against ground
|
||||
truth.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|---|---|---|---|---|
|
||||
| Forklift Proximity | `ind_forklift_proximity.rs` | Warns when pedestrians are near moving forklifts/AGVs | 500--502 | S (<5 ms) |
|
||||
| Confined Space | `ind_confined_space.rs` | Monitors worker vitals in tanks, manholes, vessels | 510--514 | L (<2 ms) |
|
||||
| Clean Room | `ind_clean_room.rs` | Personnel count and turbulent motion for ISO 14644 | 520--523 | L (<2 ms) |
|
||||
| Livestock Monitor | `ind_livestock_monitor.rs` | Animal health monitoring in pens, barns, enclosures | 530--533 | L (<2 ms) |
|
||||
| Structural Vibration | `ind_structural_vibration.rs` | Seismic, resonance, and structural drift detection | 540--543 | H (<10 ms) |
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Forklift Proximity Warning (`ind_forklift_proximity.rs`)
|
||||
|
||||
**What it does**: Warns when a person is too close to a moving forklift, AGV,
|
||||
or mobile robot, even around blind corners and through shelving racks.
|
||||
|
||||
**How it works**: The module separates forklift signatures from human
|
||||
signatures using three CSI features:
|
||||
|
||||
1. **Amplitude ratio**: Large metal bodies (forklifts) produce 2--5x amplitude
|
||||
increases across all subcarriers relative to an empty-warehouse baseline.
|
||||
2. **Low-frequency phase dominance**: Forklifts move slowly (<0.3 Hz phase
|
||||
modulation) compared to walking humans (0.5--2 Hz). The module computes
|
||||
the ratio of low-frequency energy to total phase energy.
|
||||
3. **Motor vibration**: Electric forklift motors produce elevated, uniform
|
||||
variance across subcarriers (>0.08 threshold).
|
||||
|
||||
When all three conditions are met for 4 consecutive frames (debounced), the
|
||||
module declares a vehicle present. If a human signature (host-reported
|
||||
presence + motion energy >0.15) co-occurs, a proximity warning is emitted
|
||||
with a distance category derived from amplitude ratio.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
pub struct ForkliftProximityDetector { /* ... */ }
|
||||
|
||||
impl ForkliftProximityDetector {
|
||||
/// Create a new detector. Requires 100-frame calibration (~5 s at 20 Hz).
|
||||
pub const fn new() -> Self;
|
||||
|
||||
/// Process one CSI frame. Returns events as (event_id, value) pairs.
|
||||
pub fn process_frame(
|
||||
&mut self,
|
||||
phases: &[f32], // per-subcarrier phase values
|
||||
amplitudes: &[f32], // per-subcarrier amplitude values
|
||||
variance: &[f32], // per-subcarrier variance values
|
||||
motion_energy: f32, // host-reported motion energy
|
||||
presence: i32, // host-reported presence flag (0/1)
|
||||
n_persons: i32, // host-reported person count
|
||||
) -> &[(i32, f32)];
|
||||
|
||||
/// Whether a vehicle is currently detected.
|
||||
pub fn is_vehicle_present(&self) -> bool;
|
||||
|
||||
/// Current amplitude ratio (proxy for vehicle proximity).
|
||||
pub fn amplitude_ratio(&self) -> f32;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Meaning |
|
||||
|---|---|---|---|
|
||||
| 500 | `EVENT_PROXIMITY_WARNING` | Distance category: 0.0 = critical, 1.0 = warning, 2.0 = caution | Person dangerously close to vehicle |
|
||||
| 501 | `EVENT_VEHICLE_DETECTED` | Amplitude ratio (float) | Forklift/AGV entered sensor zone |
|
||||
| 502 | `EVENT_HUMAN_NEAR_VEHICLE` | Motion energy (float) | Human detected in vehicle zone (fires once on transition) |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
+-----------+
|
||||
| |
|
||||
+-------->| No Vehicle|<---------+
|
||||
| | | |
|
||||
| +-----+-----+ |
|
||||
| | |
|
||||
| amp_ratio > 2.5 AND |
|
||||
| low_freq_dominant AND | debounce drops
|
||||
| vibration > 0.08 | below threshold
|
||||
| (4 frames debounce) |
|
||||
| | |
|
||||
| +-----v-----+ |
|
||||
| | |----------+
|
||||
+---------| Vehicle |
|
||||
| Present |
|
||||
+-----+-----+
|
||||
|
|
||||
human present | (presence + motion > 0.15)
|
||||
+ debounce |
|
||||
+-----v-----+
|
||||
| Proximity |----> EVENT 500 (cooldown 40 frames)
|
||||
| Warning |----> EVENT 502 (once on transition)
|
||||
+-----------+
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Safety Implication |
|
||||
|---|---|---|---|
|
||||
| `FORKLIFT_AMP_RATIO` | 2.5 | 1.5--5.0 | Lower = more sensitive, more false positives |
|
||||
| `HUMAN_MOTION_THRESH` | 0.15 | 0.05--0.5 | Lower = catches slow-moving workers |
|
||||
| `VEHICLE_DEBOUNCE` | 4 frames | 2--10 | Higher = fewer false alarms, slower response |
|
||||
| `PROXIMITY_DEBOUNCE` | 2 frames | 1--5 | Higher = fewer false alarms, slower response |
|
||||
| `ALERT_COOLDOWN` | 40 frames (2 s) | 10--200 | Lower = more frequent warnings |
|
||||
| `DIST_CRITICAL` | amp ratio > 4.0 | -- | Very close proximity |
|
||||
| `DIST_WARNING` | amp ratio > 3.0 | -- | Close proximity |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ind_forklift_proximity::ForkliftProximityDetector;
|
||||
|
||||
let mut detector = ForkliftProximityDetector::new();
|
||||
|
||||
// Calibration phase: feed 100 frames of empty warehouse
|
||||
for _ in 0..100 {
|
||||
detector.process_frame(&phases, &s, &variance, 0.0, 0, 0);
|
||||
}
|
||||
|
||||
// Normal operation
|
||||
let events = detector.process_frame(&phases, &s, &variance, 0.5, 1, 1);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
500 => {
|
||||
let category = match value as i32 {
|
||||
0 => "CRITICAL -- stop forklift immediately",
|
||||
1 => "WARNING -- reduce speed",
|
||||
_ => "CAUTION -- be alert",
|
||||
};
|
||||
trigger_alarm(category);
|
||||
}
|
||||
501 => log("Vehicle detected, amplitude ratio: {}", value),
|
||||
502 => log("Human entered vehicle zone"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up Warehouse Proximity Alerts
|
||||
|
||||
1. **Sensor placement**: Mount one ESP32 WiFi sensor per aisle, at shelf
|
||||
height (1.5--2 m). Each sensor covers approximately one aisle width
|
||||
(3--4 m) and 10--15 m of aisle length.
|
||||
|
||||
2. **Calibration**: Power on during a quiet period (no forklifts, no
|
||||
workers). The module auto-calibrates over the first 100 frames (5 s
|
||||
at 20 Hz). The baseline amplitude represents the empty aisle.
|
||||
|
||||
3. **Threshold tuning**: If false alarms occur due to hand trucks or
|
||||
pallet jacks, increase `FORKLIFT_AMP_RATIO` from 2.5 to 3.0. If
|
||||
forklifts are missed, decrease to 2.0.
|
||||
|
||||
4. **Integration**: Connect `EVENT_PROXIMITY_WARNING` (500) to a warning
|
||||
light (amber for caution/warning, red for critical) and audible alarm.
|
||||
Connect to the facility SCADA system for logging.
|
||||
|
||||
5. **Validation**: Walk through the aisle while a forklift operates.
|
||||
Verify all three distance categories trigger at appropriate ranges.
|
||||
|
||||
---
|
||||
|
||||
### Confined Space Monitor (`ind_confined_space.rs`)
|
||||
|
||||
**What it does**: Monitors workers inside tanks, manholes, vessels, or any
|
||||
enclosed space. Confirms they are breathing and alerts if they stop moving
|
||||
or breathing.
|
||||
|
||||
**Compliance**: Designed to support OSHA 29 CFR 1910.146 confined space
|
||||
entry requirements. The module provides continuous proof-of-life monitoring
|
||||
to supplement (not replace) the required safety attendant.
|
||||
|
||||
**How it works**: Uses debounced presence detection to track entry/exit
|
||||
transitions. While a worker is inside, the module continuously monitors
|
||||
two vital indicators:
|
||||
|
||||
1. **Breathing**: Host-reported breathing BPM must stay above 4.0 BPM.
|
||||
If breathing is not detected for 300 frames (15 seconds at 20 Hz),
|
||||
an extraction alert is emitted.
|
||||
2. **Motion**: Host-reported motion energy must stay above 0.02. If no
|
||||
motion is detected for 1200 frames (60 seconds), an immobility alert
|
||||
is emitted.
|
||||
|
||||
The module transitions between `Empty`, `Present`, `BreathingCeased`, and
|
||||
`Immobile` states. When breathing or motion resumes, the state recovers
|
||||
back to `Present`.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
pub enum WorkerState {
|
||||
Empty, // No worker in the space
|
||||
Present, // Worker present, vitals normal
|
||||
BreathingCeased, // No breathing detected (danger)
|
||||
Immobile, // No motion detected (danger)
|
||||
}
|
||||
|
||||
pub struct ConfinedSpaceMonitor { /* ... */ }
|
||||
|
||||
impl ConfinedSpaceMonitor {
|
||||
pub const fn new() -> Self;
|
||||
|
||||
/// Process one frame.
|
||||
pub fn process_frame(
|
||||
&mut self,
|
||||
presence: i32, // host-reported presence (0/1)
|
||||
breathing_bpm: f32, // host-reported breathing rate
|
||||
motion_energy: f32, // host-reported motion energy
|
||||
variance: f32, // mean CSI variance
|
||||
) -> &[(i32, f32)];
|
||||
|
||||
/// Current worker state.
|
||||
pub fn state(&self) -> WorkerState;
|
||||
|
||||
/// Whether a worker is inside the space.
|
||||
pub fn is_worker_inside(&self) -> bool;
|
||||
|
||||
/// Seconds since last confirmed breathing.
|
||||
pub fn seconds_since_breathing(&self) -> f32;
|
||||
|
||||
/// Seconds since last detected motion.
|
||||
pub fn seconds_since_motion(&self) -> f32;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Meaning |
|
||||
|---|---|---|---|
|
||||
| 510 | `EVENT_WORKER_ENTRY` | 1.0 | Worker entered the confined space |
|
||||
| 511 | `EVENT_WORKER_EXIT` | 1.0 | Worker exited the confined space |
|
||||
| 512 | `EVENT_BREATHING_OK` | BPM (float) | Periodic breathing confirmation (~every 5 s) |
|
||||
| 513 | `EVENT_EXTRACTION_ALERT` | Seconds since last breath | No breathing for >15 s -- initiate rescue |
|
||||
| 514 | `EVENT_IMMOBILE_ALERT` | Seconds without motion | No motion for >60 s -- check on worker |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
+---------+
|
||||
| Empty |<----------+
|
||||
+----+----+ |
|
||||
| |
|
||||
presence | | absence (10 frames)
|
||||
(10 frames) | |
|
||||
v |
|
||||
+---------+ |
|
||||
+------>| Present |-----------+
|
||||
| +----+----+
|
||||
| | |
|
||||
| breathing | no | no motion
|
||||
| resumes | breathing| (1200 frames)
|
||||
| | (300 |
|
||||
| | frames) |
|
||||
| +----v------+ |
|
||||
+-------|Breathing | |
|
||||
| | Ceased | |
|
||||
| +-----------+ |
|
||||
| |
|
||||
| +-----------+ |
|
||||
+-------| Immobile |<--+
|
||||
+-----------+
|
||||
motion resumes -> Present
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Safety Implication |
|
||||
|---|---|---|---|
|
||||
| `BREATHING_CEASE_FRAMES` | 300 (15 s) | 100--600 | Lower = faster alert, more false positives |
|
||||
| `IMMOBILE_FRAMES` | 1200 (60 s) | 400--3600 | Lower = catches slower collapses |
|
||||
| `MIN_BREATHING_BPM` | 4.0 | 2.0--8.0 | Lower = more tolerant of slow breathing |
|
||||
| `MIN_MOTION_ENERGY` | 0.02 | 0.005--0.1 | Lower = catches subtle movements |
|
||||
| `ENTRY_EXIT_DEBOUNCE` | 10 frames | 5--30 | Higher = fewer false entry/exits |
|
||||
| `MIN_PRESENCE_VAR` | 0.005 | 0.001--0.05 | Noise rejection for empty space |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ind_confined_space::{
|
||||
ConfinedSpaceMonitor, WorkerState,
|
||||
EVENT_EXTRACTION_ALERT, EVENT_IMMOBILE_ALERT,
|
||||
};
|
||||
|
||||
let mut monitor = ConfinedSpaceMonitor::new();
|
||||
|
||||
// Process each CSI frame
|
||||
let events = monitor.process_frame(presence, breathing_bpm, motion_energy, variance);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
513 => { // EXTRACTION_ALERT
|
||||
activate_rescue_alarm();
|
||||
notify_safety_attendant(value); // seconds since last breath
|
||||
}
|
||||
514 => { // IMMOBILE_ALERT
|
||||
notify_safety_attendant(value); // seconds without motion
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Query state for dashboard display
|
||||
match monitor.state() {
|
||||
WorkerState::Empty => display_green("Space empty"),
|
||||
WorkerState::Present => display_green("Worker OK"),
|
||||
WorkerState::BreathingCeased => display_red("NO BREATHING"),
|
||||
WorkerState::Immobile => display_amber("Worker immobile"),
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Clean Room Monitor (`ind_clean_room.rs`)
|
||||
|
||||
**What it does**: Tracks personnel count and movement patterns in cleanrooms
|
||||
to enforce ISO 14644 occupancy limits and detect turbulent motion that could
|
||||
disturb laminar airflow.
|
||||
|
||||
**How it works**: Uses the host-reported person count with debounced
|
||||
violation detection. Turbulent motion (rapid movement with energy >0.6) is
|
||||
flagged because it disrupts the laminar airflow that keeps particulate counts
|
||||
low. The module maintains a running compliance percentage for audit reporting.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
pub struct CleanRoomMonitor { /* ... */ }
|
||||
|
||||
impl CleanRoomMonitor {
|
||||
/// Create with default max occupancy of 4.
|
||||
pub const fn new() -> Self;
|
||||
|
||||
/// Create with custom maximum occupancy.
|
||||
pub const fn with_max_occupancy(max: u8) -> Self;
|
||||
|
||||
/// Process one frame.
|
||||
pub fn process_frame(
|
||||
&mut self,
|
||||
n_persons: i32, // host-reported person count
|
||||
presence: i32, // host-reported presence (0/1)
|
||||
motion_energy: f32, // host-reported motion energy
|
||||
) -> &[(i32, f32)];
|
||||
|
||||
/// Current occupancy count.
|
||||
pub fn current_count(&self) -> u8;
|
||||
|
||||
/// Maximum allowed occupancy.
|
||||
pub fn max_occupancy(&self) -> u8;
|
||||
|
||||
/// Whether currently in violation.
|
||||
pub fn is_in_violation(&self) -> bool;
|
||||
|
||||
/// Compliance percentage (0--100).
|
||||
pub fn compliance_percent(&self) -> f32;
|
||||
|
||||
/// Total number of violation events.
|
||||
pub fn total_violations(&self) -> u32;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Meaning |
|
||||
|---|---|---|---|
|
||||
| 520 | `EVENT_OCCUPANCY_COUNT` | Person count (float) | Occupancy changed |
|
||||
| 521 | `EVENT_OCCUPANCY_VIOLATION` | Current count (float) | Count exceeds max allowed |
|
||||
| 522 | `EVENT_TURBULENT_MOTION` | Motion energy (float) | Rapid movement detected (airflow risk) |
|
||||
| 523 | `EVENT_COMPLIANCE_REPORT` | Compliance % (0--100) | Periodic compliance summary (~30 s) |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
+------------------+
|
||||
| Monitoring |
|
||||
| (count <= max) |
|
||||
+--------+---------+
|
||||
| count > max
|
||||
| (10 frames debounce)
|
||||
+--------v---------+
|
||||
| Violation |----> EVENT 521 (cooldown 200 frames)
|
||||
| (count > max) |
|
||||
+--------+---------+
|
||||
| count <= max
|
||||
|
|
||||
+--------v---------+
|
||||
| Monitoring |
|
||||
+------------------+
|
||||
|
||||
Parallel:
|
||||
motion_energy > 0.6 (3 frames) ----> EVENT 522 (cooldown 100 frames)
|
||||
Every 600 frames (~30 s) ----------> EVENT 523 (compliance %)
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Safety Implication |
|
||||
|---|---|---|---|
|
||||
| `DEFAULT_MAX_OCCUPANCY` | 4 | 1--255 | Per ISO 14644 room class |
|
||||
| `TURBULENT_MOTION_THRESH` | 0.6 | 0.3--0.9 | Lower = stricter movement control |
|
||||
| `VIOLATION_DEBOUNCE` | 10 frames | 3--20 | Higher = tolerates brief over-counts |
|
||||
| `VIOLATION_COOLDOWN` | 200 frames (10 s) | 40--600 | Alert repeat interval |
|
||||
| `COMPLIANCE_REPORT_INTERVAL` | 600 frames (30 s) | 200--6000 | Audit report frequency |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ind_clean_room::{
|
||||
CleanRoomMonitor, EVENT_OCCUPANCY_VIOLATION, EVENT_COMPLIANCE_REPORT,
|
||||
};
|
||||
|
||||
// ISO Class 5 cleanroom: max 3 personnel
|
||||
let mut monitor = CleanRoomMonitor::with_max_occupancy(3);
|
||||
|
||||
let events = monitor.process_frame(n_persons, presence, motion_energy);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
521 => alert_cleanroom_supervisor(value as u8),
|
||||
522 => alert_turbulent_motion(),
|
||||
523 => log_compliance_audit(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Dashboard
|
||||
println!("Occupancy: {}/{}", monitor.current_count(), monitor.max_occupancy());
|
||||
println!("Compliance: {:.1}%", monitor.compliance_percent());
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Livestock Monitor (`ind_livestock_monitor.rs`)
|
||||
|
||||
**What it does**: Monitors animal presence and health in pens, barns, and
|
||||
enclosures. Detects abnormal stillness (possible illness), labored breathing,
|
||||
and escape events.
|
||||
|
||||
**How it works**: Tracks presence with debounced entry/exit detection.
|
||||
Monitors breathing rate against species-specific normal ranges. Detects
|
||||
prolonged stillness (>5 minutes) as a sign of illness, and sudden absence
|
||||
after confirmed presence as an escape event.
|
||||
|
||||
Species-specific breathing ranges:
|
||||
|
||||
| Species | Normal BPM | Labored: below | Labored: above |
|
||||
|---|---|---|---|
|
||||
| Cattle | 12--30 | 8.4 (0.7x min) | 39.0 (1.3x max) |
|
||||
| Sheep | 12--20 | 8.4 (0.7x min) | 26.0 (1.3x max) |
|
||||
| Poultry | 15--30 | 10.5 (0.7x min) | 39.0 (1.3x max) |
|
||||
| Custom | configurable | 0.7x min | 1.3x max |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
pub enum Species {
|
||||
Cattle,
|
||||
Sheep,
|
||||
Poultry,
|
||||
Custom { min_bpm: f32, max_bpm: f32 },
|
||||
}
|
||||
|
||||
pub struct LivestockMonitor { /* ... */ }
|
||||
|
||||
impl LivestockMonitor {
|
||||
/// Create with default species (Cattle).
|
||||
pub const fn new() -> Self;
|
||||
|
||||
/// Create with a specific species.
|
||||
pub const fn with_species(species: Species) -> Self;
|
||||
|
||||
/// Process one frame.
|
||||
pub fn process_frame(
|
||||
&mut self,
|
||||
presence: i32, // host-reported presence (0/1)
|
||||
breathing_bpm: f32, // host-reported breathing rate
|
||||
motion_energy: f32, // host-reported motion energy
|
||||
variance: f32, // mean CSI variance (unused, reserved)
|
||||
) -> &[(i32, f32)];
|
||||
|
||||
/// Whether an animal is currently detected.
|
||||
pub fn is_animal_present(&self) -> bool;
|
||||
|
||||
/// Configured species.
|
||||
pub fn species(&self) -> Species;
|
||||
|
||||
/// Minutes of stillness.
|
||||
pub fn stillness_minutes(&self) -> f32;
|
||||
|
||||
/// Last observed breathing BPM.
|
||||
pub fn last_breathing_bpm(&self) -> f32;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Meaning |
|
||||
|---|---|---|---|
|
||||
| 530 | `EVENT_ANIMAL_PRESENT` | BPM (float) | Periodic presence report (~10 s) |
|
||||
| 531 | `EVENT_ABNORMAL_STILLNESS` | Minutes still (float) | No motion for >5 minutes |
|
||||
| 532 | `EVENT_LABORED_BREATHING` | BPM (float) | Breathing outside normal range |
|
||||
| 533 | `EVENT_ESCAPE_ALERT` | Minutes present before escape (float) | Animal suddenly absent after confirmed presence |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
+---------+
|
||||
| Empty |<---------+
|
||||
+----+----+ |
|
||||
| |
|
||||
presence | absence >= 20 frames
|
||||
(10 frames) | (after >= 200 frames presence
|
||||
v | -> EVENT 533 escape alert)
|
||||
+---------+ |
|
||||
| Present |----------+
|
||||
+----+----+
|
||||
|
|
||||
no motion (6000 frames = 5 min) -> EVENT 531 (once)
|
||||
breathing outside range (20 frames) -> EVENT 532 (repeating)
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Safety Implication |
|
||||
|---|---|---|---|
|
||||
| `STILLNESS_FRAMES` | 6000 (5 min) | 1200--12000 | Lower = earlier illness detection |
|
||||
| `MIN_PRESENCE_FOR_ESCAPE` | 200 (10 s) | 60--600 | Minimum presence before escape counts |
|
||||
| `ESCAPE_ABSENCE_FRAMES` | 20 (1 s) | 10--100 | Brief absences tolerated |
|
||||
| `LABORED_DEBOUNCE` | 20 frames (1 s) | 5--60 | Lower = faster breathing alerts |
|
||||
| `MIN_MOTION_ACTIVE` | 0.03 | 0.01--0.1 | Sensitivity to subtle movement |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ind_livestock_monitor::{
|
||||
LivestockMonitor, Species, EVENT_ESCAPE_ALERT, EVENT_LABORED_BREATHING,
|
||||
};
|
||||
|
||||
// Dairy barn: monitor cows
|
||||
let mut monitor = LivestockMonitor::with_species(Species::Cattle);
|
||||
|
||||
let events = monitor.process_frame(presence, breathing_bpm, motion_energy, variance);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
532 => alert_veterinarian(value), // labored breathing BPM
|
||||
533 => alert_farm_security(value), // escape: minutes present before loss
|
||||
531 => log_health_concern(value), // minutes of stillness
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Structural Vibration Monitor (`ind_structural_vibration.rs`)
|
||||
|
||||
**What it does**: Detects building vibration, seismic activity, and structural
|
||||
stress using CSI phase stability. Only operates when the monitored space is
|
||||
unoccupied (human movement masks structural signals).
|
||||
|
||||
**How it works**: When no humans are present, WiFi CSI phase is highly stable
|
||||
(noise floor ~0.02 rad). The module detects three types of structural events:
|
||||
|
||||
1. **Seismic**: Broadband energy increase (>60% of subcarriers affected,
|
||||
RMS >0.15 rad). Indicates earthquake, heavy vehicle pass-by, or
|
||||
construction activity.
|
||||
2. **Mechanical resonance**: Narrowband peaks detected via autocorrelation
|
||||
of the mean-phase time series. A peak-to-mean ratio >3.0 with RMS above
|
||||
2x noise floor indicates periodic mechanical vibration (HVAC, pumps,
|
||||
rotating equipment).
|
||||
3. **Structural drift**: Slow monotonic phase change across >50% of
|
||||
subcarriers for >30 seconds. Indicates material stress, foundation
|
||||
settlement, or thermal expansion.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
pub struct StructuralVibrationMonitor { /* ... */ }
|
||||
|
||||
impl StructuralVibrationMonitor {
|
||||
/// Create a new monitor. Requires 100-frame calibration when empty.
|
||||
pub const fn new() -> Self;
|
||||
|
||||
/// Process one CSI frame.
|
||||
pub fn process_frame(
|
||||
&mut self,
|
||||
phases: &[f32], // per-subcarrier phase values
|
||||
amplitudes: &[f32], // per-subcarrier amplitude values
|
||||
variance: &[f32], // per-subcarrier variance values
|
||||
presence: i32, // 0 = empty (analyze), 1 = occupied (skip)
|
||||
) -> &[(i32, f32)];
|
||||
|
||||
/// Current RMS vibration level.
|
||||
pub fn rms_vibration(&self) -> f32;
|
||||
|
||||
/// Whether baseline has been established.
|
||||
pub fn is_calibrated(&self) -> bool;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Meaning |
|
||||
|---|---|---|---|
|
||||
| 540 | `EVENT_SEISMIC_DETECTED` | RMS vibration level (rad) | Broadband seismic activity |
|
||||
| 541 | `EVENT_MECHANICAL_RESONANCE` | Dominant frequency (Hz) | Narrowband mechanical vibration |
|
||||
| 542 | `EVENT_STRUCTURAL_DRIFT` | Drift rate (rad/s) | Slow structural deformation |
|
||||
| 543 | `EVENT_VIBRATION_SPECTRUM` | RMS level (rad) | Periodic spectrum report (~5 s) |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
+--------------+
|
||||
| Calibrating | (100 frames, presence=0 required)
|
||||
+------+-------+
|
||||
|
|
||||
+------v-------+
|
||||
| Idle | (presence=1: skip analysis, reset drift)
|
||||
| (Occupied) |
|
||||
+------+-------+
|
||||
| presence=0
|
||||
+------v-------+
|
||||
| Analyzing |
|
||||
+------+-------+
|
||||
|
|
||||
+-----> RMS > 0.15 + broadband -------> EVENT 540 (seismic)
|
||||
+-----> autocorr peak ratio > 3.0 ----> EVENT 541 (resonance)
|
||||
+-----> monotonic drift > 30 s -------> EVENT 542 (drift)
|
||||
+-----> every 100 frames -------------> EVENT 543 (spectrum)
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Safety Implication |
|
||||
|---|---|---|---|
|
||||
| `SEISMIC_THRESH` | 0.15 rad RMS | 0.05--0.5 | Lower = more sensitive to tremors |
|
||||
| `RESONANCE_PEAK_RATIO` | 3.0 | 2.0--5.0 | Lower = detects weaker resonances |
|
||||
| `DRIFT_RATE_THRESH` | 0.0005 rad/frame | 0.0001--0.005 | Lower = detects slower drift |
|
||||
| `DRIFT_MIN_FRAMES` | 600 (30 s) | 200--2400 | Minimum drift duration before alert |
|
||||
| `SEISMIC_DEBOUNCE` | 4 frames | 2--10 | Higher = fewer false seismic alerts |
|
||||
| `SEISMIC_COOLDOWN` | 200 frames (10 s) | 40--600 | Alert repeat interval |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ind_structural_vibration::{
|
||||
StructuralVibrationMonitor, EVENT_SEISMIC_DETECTED, EVENT_STRUCTURAL_DRIFT,
|
||||
};
|
||||
|
||||
let mut monitor = StructuralVibrationMonitor::new();
|
||||
|
||||
// Calibrate during unoccupied period
|
||||
for _ in 0..100 {
|
||||
monitor.process_frame(&phases, &s, &variance, 0);
|
||||
}
|
||||
assert!(monitor.is_calibrated());
|
||||
|
||||
// Normal operation
|
||||
let events = monitor.process_frame(&phases, &s, &variance, presence);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
540 => {
|
||||
trigger_building_alarm();
|
||||
log_seismic_event(value); // RMS vibration level
|
||||
}
|
||||
542 => {
|
||||
notify_structural_engineer(value); // drift rate rad/s
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## OSHA Compliance Notes
|
||||
|
||||
### Forklift Proximity (OSHA 29 CFR 1910.178)
|
||||
|
||||
- **Standard**: Powered Industrial Trucks -- operator must warn others.
|
||||
- **Module supports**: Automated proximity detection supplements horn/light
|
||||
warnings. Does NOT replace operator training, seat belts, or speed limits.
|
||||
- **Additional equipment required**: Physical barriers, floor markings,
|
||||
traffic mirrors, operator training program.
|
||||
|
||||
### Confined Space (OSHA 29 CFR 1910.146)
|
||||
|
||||
- **Standard**: Permit-Required Confined Spaces.
|
||||
- **Module supports**: Continuous proof-of-life monitoring (breathing and
|
||||
motion confirmation). Assists the required safety attendant.
|
||||
- **Additional equipment required**:
|
||||
- Atmospheric monitoring (O2, H2S, CO, LEL) -- the WiFi module cannot
|
||||
detect gas hazards.
|
||||
- Communication system between entrant and attendant.
|
||||
- Rescue equipment (retrieval system, harness, tripod).
|
||||
- Entry permit documenting hazards and controls.
|
||||
- **Audit trail**: `EVENT_BREATHING_OK` (512) provides timestamped
|
||||
proof-of-life records for compliance documentation.
|
||||
|
||||
### Clean Room (ISO 14644)
|
||||
|
||||
- **Standard**: Cleanrooms and associated controlled environments.
|
||||
- **Module supports**: Real-time occupancy enforcement and turbulent motion
|
||||
detection for particulate control.
|
||||
- **Additional equipment required**: Particle counters, differential pressure
|
||||
monitors, HEPA/ULPA filtration systems.
|
||||
- **Documentation**: `EVENT_COMPLIANCE_REPORT` (523) provides periodic
|
||||
compliance percentages for audit records.
|
||||
|
||||
### Livestock (no direct OSHA standard; see USDA Animal Welfare Act)
|
||||
|
||||
- **Module supports**: Automated health monitoring reduces manual inspection
|
||||
burden. Escape detection supports perimeter security.
|
||||
- **Additional equipment required**: Veterinary monitoring systems, proper
|
||||
fencing, temperature/humidity sensors.
|
||||
|
||||
### Structural Vibration (OSHA 29 CFR 1926 Subpart P, Excavations)
|
||||
|
||||
- **Standard**: Structural stability requirements for construction.
|
||||
- **Module supports**: Continuous vibration monitoring during unoccupied
|
||||
periods. Seismic detection provides early warning.
|
||||
- **Additional equipment required**: Certified structural inspection,
|
||||
accelerometers for critical structures, tilt sensors.
|
||||
|
||||
---
|
||||
|
||||
## Deployment Guide
|
||||
|
||||
### Sensor Placement for Warehouse Coverage
|
||||
|
||||
```
|
||||
+---+---+---+---+---+
|
||||
| S | | | | S | S = WiFi sensor (ESP32)
|
||||
+---+ Aisle 1 +---+ Mounted at shelf height (1.5-2 m)
|
||||
| | | | One sensor per aisle intersection
|
||||
+---+ Aisle 2 +---+
|
||||
| S | | S | Coverage: ~15 m range per sensor
|
||||
+---+---+---+---+---+ For proximity: sensor every 10 m along aisle
|
||||
```
|
||||
|
||||
- Mount sensors at shelf height (1.5--2 m) for best human/forklift separation.
|
||||
- Place at aisle intersections for blind-corner coverage.
|
||||
- Each sensor covers approximately 10--15 m of aisle length.
|
||||
- For critical zones (loading docks, charging areas), use overlapping sensors.
|
||||
|
||||
### Multi-Sensor Setup for Confined Spaces
|
||||
|
||||
```
|
||||
Ground Level
|
||||
+-----------+
|
||||
| Sensor A | <-- Entry point monitoring
|
||||
+-----+-----+
|
||||
|
|
||||
| Manhole / Hatch
|
||||
|
|
||||
+-----v-----+
|
||||
| Sensor B | <-- Inside space (if possible)
|
||||
+-----------+
|
||||
```
|
||||
|
||||
- Sensor A at the entry point detects worker entry/exit.
|
||||
- Sensor B inside the confined space (if safely mountable) provides
|
||||
breathing and motion monitoring.
|
||||
- If only one sensor is available, mount at the entry facing into the space.
|
||||
- WiFi signals penetrate metal walls poorly -- use multiple sensors for
|
||||
large vessels.
|
||||
|
||||
### Integration with Safety PLCs
|
||||
|
||||
Connect ESP32 event output to safety PLCs via:
|
||||
|
||||
1. **UDP**: The sensing server receives ESP32 CSI data and emits events
|
||||
via REST API. Poll `/api/v1/events` for real-time alerts.
|
||||
2. **Modbus TCP**: Use a gateway to convert UDP events to Modbus registers
|
||||
for direct PLC integration.
|
||||
3. **GPIO**: For hard-wired safety circuits, connect ESP32 GPIO outputs
|
||||
to PLC safety inputs. Configure the ESP32 firmware to assert GPIO on
|
||||
specific event IDs.
|
||||
|
||||
### Calibration Checklist
|
||||
|
||||
1. Ensure the monitored space is in its normal empty state.
|
||||
2. Power on the sensor and wait for calibration to complete:
|
||||
- Forklift Proximity: 100 frames (5 seconds)
|
||||
- Structural Vibration: 100 frames (5 seconds)
|
||||
- Confined Space: No calibration needed (uses host presence)
|
||||
- Clean Room: No calibration needed (uses host person count)
|
||||
- Livestock: No calibration needed (uses host presence)
|
||||
3. Validate by walking through the space and confirming presence detection.
|
||||
4. For forklift proximity, drive a forklift through and verify vehicle
|
||||
detection and proximity warnings at appropriate distances.
|
||||
5. Document calibration date, sensor position, and firmware version.
|
||||
|
||||
---
|
||||
|
||||
## Event ID Registry (Category 5)
|
||||
|
||||
| Range | Module | Events |
|
||||
|---|---|---|
|
||||
| 500--502 | Forklift Proximity | `PROXIMITY_WARNING`, `VEHICLE_DETECTED`, `HUMAN_NEAR_VEHICLE` |
|
||||
| 510--514 | Confined Space | `WORKER_ENTRY`, `WORKER_EXIT`, `BREATHING_OK`, `EXTRACTION_ALERT`, `IMMOBILE_ALERT` |
|
||||
| 520--523 | Clean Room | `OCCUPANCY_COUNT`, `OCCUPANCY_VIOLATION`, `TURBULENT_MOTION`, `COMPLIANCE_REPORT` |
|
||||
| 530--533 | Livestock Monitor | `ANIMAL_PRESENT`, `ABNORMAL_STILLNESS`, `LABORED_BREATHING`, `ESCAPE_ALERT` |
|
||||
| 540--543 | Structural Vibration | `SEISMIC_DETECTED`, `MECHANICAL_RESONANCE`, `STRUCTURAL_DRIFT`, `VIBRATION_SPECTRUM` |
|
||||
|
||||
Total: 20 event types across 5 modules.
|
||||
@@ -0,0 +1,688 @@
|
||||
# Medical & Health Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Contactless health monitoring using WiFi signals. No wearables, no cameras -- just an ESP32 sensor reading WiFi reflections off a person's body to detect breathing problems, heart rhythm issues, walking difficulties, and seizures.
|
||||
|
||||
## Important Disclaimer
|
||||
|
||||
These modules are **research tools, not FDA-approved medical devices**. They should supplement -- not replace -- professional medical monitoring. WiFi CSI-derived vital signs are inherently noisier than clinical instruments (ECG, pulse oximetry, respiratory belts). False positives and false negatives will occur. Always validate findings against clinical-grade equipment before acting on alerts.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|-------------|-----------|--------|
|
||||
| Sleep Apnea Detection | `med_sleep_apnea.rs` | Detects apnea episodes when breathing ceases for >10s; tracks AHI score | 100-102 | L (< 2 ms) |
|
||||
| Cardiac Arrhythmia | `med_cardiac_arrhythmia.rs` | Detects tachycardia, bradycardia, missed beats, HRV anomalies | 110-113 | S (< 5 ms) |
|
||||
| Respiratory Distress | `med_respiratory_distress.rs` | Detects tachypnea, labored breathing, Cheyne-Stokes, composite distress score | 120-123 | H (< 10 ms) |
|
||||
| Gait Analysis | `med_gait_analysis.rs` | Extracts step cadence, asymmetry, shuffling, festination, fall-risk score | 130-134 | H (< 10 ms) |
|
||||
| Seizure Detection | `med_seizure_detect.rs` | Detects tonic-clonic seizures with phase discrimination (fall vs tremor) | 140-143 | S (< 5 ms) |
|
||||
|
||||
All modules:
|
||||
- Compile to `no_std` for WASM (ESP32 WASM3 runtime)
|
||||
- Use `const fn new()` for zero-cost initialization
|
||||
- Return events via `&[(i32, f32)]` slices (no heap allocation)
|
||||
- Include NaN and division-by-zero protections
|
||||
- Implement cooldown timers to prevent event flooding
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Sleep Apnea Detection (`med_sleep_apnea.rs`)
|
||||
|
||||
**What it does**: Monitors breathing rate from the host CSI pipeline and detects when breathing drops below 4 BPM for more than 10 consecutive seconds, indicating an apnea episode. It tracks all episodes and computes the Apnea-Hypopnea Index (AHI) -- the number of apnea events per hour of monitored sleep time. AHI is the standard clinical metric for sleep apnea severity.
|
||||
|
||||
**Clinical basis**: Obstructive and central sleep apnea are defined by cessation of airflow for 10 seconds or more. The module uses a breathing rate threshold of 4 BPM (essentially near-zero breathing) with a 10-second onset delay to confirm cessation is sustained. AHI severity classification: < 5 normal, 5-15 mild, 15-30 moderate, > 30 severe.
|
||||
|
||||
**How it works**:
|
||||
1. Each second, checks if breathing BPM is below 4.0
|
||||
2. Increments a consecutive-low-breath counter
|
||||
3. After 10 consecutive seconds, declares apnea onset (backdated to when breathing first dropped)
|
||||
4. When breathing resumes above 4 BPM, records the episode with its duration
|
||||
5. Every 5 minutes, computes AHI = (total episodes) / (monitoring hours)
|
||||
6. Only monitors when presence is detected; if subject leaves during apnea, the episode is ended
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `SleepApneaDetector` | struct | Main detector state |
|
||||
| `SleepApneaDetector::new()` | `const fn` | Create detector with zeroed state |
|
||||
| `process_frame(breathing_bpm, presence, variance)` | method | Process one frame at ~1 Hz; returns event slice |
|
||||
| `ahi()` | method | Current AHI value |
|
||||
| `episode_count()` | method | Total recorded apnea episodes |
|
||||
| `monitoring_seconds()` | method | Total seconds with presence active |
|
||||
| `in_apnea()` | method | Whether currently in an apnea episode |
|
||||
| `APNEA_BPM_THRESH` | const | 4.0 BPM -- below this counts as apnea |
|
||||
| `APNEA_ONSET_SECS` | const | 10 seconds -- minimum duration to declare apnea |
|
||||
| `AHI_REPORT_INTERVAL` | const | 300 seconds (5 min) -- how often AHI is recalculated |
|
||||
| `MAX_EPISODES` | const | 256 -- maximum episodes stored per session |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Clinical Meaning |
|
||||
|----------|----------|-------|-----------------|
|
||||
| 100 | `EVENT_APNEA_START` | Current breathing BPM | Breathing has ceased or dropped below 4 BPM for >10 seconds |
|
||||
| 101 | `EVENT_APNEA_END` | Duration in seconds | Breathing has resumed after an apnea episode |
|
||||
| 102 | `EVENT_AHI_UPDATE` | AHI score (events/hour) | Periodic severity metric; >5 = mild, >15 = moderate, >30 = severe |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
presence lost
|
||||
[Monitoring] -----> [Not Monitoring] (no events, counter paused)
|
||||
| |
|
||||
| bpm < 4.0 | presence regained
|
||||
v v
|
||||
[Low Breath Counter] [Monitoring]
|
||||
|
|
||||
| count >= 10s
|
||||
v
|
||||
[In Apnea] ---------> [Episode End] (bpm >= 4.0 or presence lost)
|
||||
| |
|
||||
| v
|
||||
| [Record Episode, emit APNEA_END]
|
||||
|
|
||||
+-- emit APNEA_START (once)
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Clinical Range | Description |
|
||||
|-----------|---------|----------------|-------------|
|
||||
| `APNEA_BPM_THRESH` | 4.0 | 0-6 BPM | Breathing rate below which apnea is suspected |
|
||||
| `APNEA_ONSET_SECS` | 10 | 10-20 s | Seconds of low breathing before apnea is declared |
|
||||
| `AHI_REPORT_INTERVAL` | 300 | 60-3600 s | How often AHI is recalculated and emitted |
|
||||
| `MAX_EPISODES` | 256 | -- | Fixed buffer size for episode history |
|
||||
| `PRESENCE_ACTIVE` | 1 | -- | Minimum presence flag value for monitoring |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::med_sleep_apnea::*;
|
||||
|
||||
let mut detector = SleepApneaDetector::new();
|
||||
|
||||
// Normal breathing -- no events
|
||||
let events = detector.process_frame(14.0, 1, 0.1);
|
||||
assert!(events.is_empty());
|
||||
|
||||
// Simulate apnea: feed low BPM for 15 seconds
|
||||
for _ in 0..15 {
|
||||
let events = detector.process_frame(1.0, 1, 0.1);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_APNEA_START => println!("Apnea detected! BPM: {}", value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(detector.in_apnea());
|
||||
|
||||
// Resume normal breathing
|
||||
let events = detector.process_frame(14.0, 1, 0.1);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_APNEA_END => println!("Apnea ended after {} seconds", value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Episodes: {}", detector.episode_count());
|
||||
println!("AHI: {:.1}", detector.ahi());
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up Bedroom Sleep Monitoring
|
||||
|
||||
1. **ESP32 placement**: Mount the ESP32-S3 on the wall or ceiling 1-2 meters from the bed, at chest height. The sensor should have line-of-sight to the sleeping area. Avoid placing near metal objects or moving fans that create CSI interference.
|
||||
|
||||
2. **WiFi router**: Ensure a stable WiFi AP is within range. The ESP32 monitors the CSI (Channel State Information) of WiFi signals reflected off the person's body. The AP should be on the opposite side of the bed from the sensor for best body reflection capture.
|
||||
|
||||
3. **Firmware configuration**: Flash the ESP32 firmware with Tier 2 edge processing enabled (provides breathing BPM). The sleep apnea WASM module runs as a Tier 3 algorithm on top of the Tier 2 vitals output.
|
||||
|
||||
4. **Threshold tuning**: The default 4 BPM threshold is conservative (near-complete cessation). For a more sensitive detector, lower to 6-8 BPM, but expect more false positives from shallow breathing. The 10-second onset delay matches clinical apnea definitions.
|
||||
|
||||
5. **Reading AHI results**: AHI is emitted every 5 minutes. After a full night (7-8 hours), the final AHI value represents the overnight severity. Compare against clinical thresholds: < 5 (normal), 5-15 (mild), 15-30 (moderate), > 30 (severe).
|
||||
|
||||
6. **Limitations**: WiFi-based breathing detection works best when the subject is relatively still (sleeping). Tossing and turning may cause momentary breathing detection loss, which could either mask or falsely trigger apnea events. A single-night study should always be confirmed with clinical polysomnography.
|
||||
|
||||
---
|
||||
|
||||
### Cardiac Arrhythmia Detection (`med_cardiac_arrhythmia.rs`)
|
||||
|
||||
**What it does**: Monitors heart rate from the host CSI pipeline and detects four types of cardiac rhythm abnormalities: tachycardia (sustained fast heart rate), bradycardia (sustained slow heart rate), missed beats (sudden HR drops), and HRV anomalies (heart rate variability outside normal bounds).
|
||||
|
||||
**Clinical basis**: Tachycardia is defined as HR > 100 BPM sustained for 10+ seconds. Bradycardia is HR < 50 BPM sustained for 10+ seconds (the 50 BPM threshold is used instead of the typical 60 BPM to account for CSI measurement noise and to avoid false positives in athletes with naturally low resting HR). Missed beats are detected as a >30% drop from the running average. HRV is assessed via RMSSD (root mean square of successive differences) with a widened normal band (10-120 ms equivalent) to account for the coarser CSI-derived HR measurement compared to ECG.
|
||||
|
||||
**How it works**:
|
||||
1. Maintains an exponential moving average (EMA) of heart rate with alpha=0.1
|
||||
2. Tracks consecutive seconds above 100 BPM (tachycardia) or below 50 BPM (bradycardia)
|
||||
3. After 10 consecutive seconds in an abnormal range, emits the corresponding alert
|
||||
4. Computes fractional drop from EMA to detect missed beats
|
||||
5. Maintains a 30-second ring buffer of successive HR differences for RMSSD calculation
|
||||
6. RMSSD is converted from BPM units to approximate ms-equivalent (scale factor ~17)
|
||||
7. All alerts have a 30-second cooldown to prevent event flooding
|
||||
8. Invalid readings (< 1 BPM or NaN) are silently ignored to prevent contamination
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `CardiacArrhythmiaDetector` | struct | Main detector state |
|
||||
| `CardiacArrhythmiaDetector::new()` | `const fn` | Create detector with zeroed state |
|
||||
| `process_frame(hr_bpm, phase)` | method | Process one frame at ~1 Hz; returns event slice |
|
||||
| `hr_ema()` | method | Current EMA heart rate |
|
||||
| `frame_count()` | method | Total frames processed |
|
||||
| `TACHY_THRESH` | const | 100.0 BPM |
|
||||
| `BRADY_THRESH` | const | 50.0 BPM |
|
||||
| `SUSTAINED_SECS` | const | 10 seconds |
|
||||
| `MISSED_BEAT_DROP` | const | 0.30 (30% drop from EMA) |
|
||||
| `HRV_WINDOW` | const | 30 seconds |
|
||||
| `RMSSD_LOW` / `RMSSD_HIGH` | const | 10.0 / 120.0 ms (widened for CSI) |
|
||||
| `COOLDOWN_SECS` | const | 30 seconds |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Clinical Meaning |
|
||||
|----------|----------|-------|-----------------|
|
||||
| 110 | `EVENT_TACHYCARDIA` | Current HR in BPM | Heart rate sustained above 100 BPM for 10+ seconds |
|
||||
| 111 | `EVENT_BRADYCARDIA` | Current HR in BPM | Heart rate sustained below 50 BPM for 10+ seconds |
|
||||
| 112 | `EVENT_MISSED_BEAT` | Current HR in BPM | Sudden HR drop >30% from running average |
|
||||
| 113 | `EVENT_HRV_ANOMALY` | RMSSD value (ms) | Heart rate variability outside 10-120 ms normal range |
|
||||
|
||||
#### State Machine
|
||||
|
||||
The cardiac module does not have a formal state machine -- it uses independent detectors with cooldown timers:
|
||||
|
||||
```
|
||||
For each frame:
|
||||
1. Tick cooldowns (4 independent timers)
|
||||
2. Reject invalid inputs (< 1 BPM or NaN)
|
||||
3. Update EMA (alpha = 0.1)
|
||||
4. Update RR-diff ring buffer
|
||||
5. Check tachycardia (HR > 100 for 10+ consecutive seconds)
|
||||
6. Check bradycardia (HR < 50 for 10+ consecutive seconds)
|
||||
7. Check missed beat (>30% drop from EMA)
|
||||
8. Check HRV anomaly (RMSSD outside 10-120 ms, requires full 30s window)
|
||||
9. Each check respects its own 30-second cooldown
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Clinical Range | Description |
|
||||
|-----------|---------|----------------|-------------|
|
||||
| `TACHY_THRESH` | 100.0 | 90-120 BPM | HR threshold for tachycardia |
|
||||
| `BRADY_THRESH` | 50.0 | 40-60 BPM | HR threshold for bradycardia |
|
||||
| `SUSTAINED_SECS` | 10 | 5-30 s | Consecutive seconds required for alert |
|
||||
| `MISSED_BEAT_DROP` | 0.30 | 0.20-0.40 | Fractional HR drop to flag missed beat |
|
||||
| `RMSSD_LOW` | 10.0 | 5-20 ms | Minimum normal RMSSD |
|
||||
| `RMSSD_HIGH` | 120.0 | 80-150 ms | Maximum normal RMSSD |
|
||||
| `EMA_ALPHA` | 0.1 | 0.05-0.2 | EMA smoothing coefficient |
|
||||
| `COOLDOWN_SECS` | 30 | 10-60 s | Minimum time between repeated alerts |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::med_cardiac_arrhythmia::*;
|
||||
|
||||
let mut detector = CardiacArrhythmiaDetector::new();
|
||||
|
||||
// Normal heart rate -- no events
|
||||
for _ in 0..60 {
|
||||
let events = detector.process_frame(72.0, 0.0);
|
||||
assert!(events.is_empty() || events.iter().all(|&(t, _)| t == EVENT_HRV_ANOMALY));
|
||||
}
|
||||
|
||||
// Sustained tachycardia
|
||||
for _ in 0..15 {
|
||||
let events = detector.process_frame(120.0, 0.0);
|
||||
for &(event_id, value) in events {
|
||||
if event_id == EVENT_TACHYCARDIA {
|
||||
println!("Tachycardia alert! HR: {} BPM", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Respiratory Distress Detection (`med_respiratory_distress.rs`)
|
||||
|
||||
**What it does**: Detects four types of respiratory abnormalities from the host CSI pipeline: tachypnea (fast breathing), labored breathing (high amplitude variance), Cheyne-Stokes respiration (a crescendo-decrescendo breathing pattern), and a composite respiratory distress severity score from 0-100.
|
||||
|
||||
**Clinical basis**: Tachypnea is defined clinically as > 20 BPM in adults. This module uses a threshold of 25 BPM (more conservative) to reduce false positives from the inherently noisier CSI-derived breathing rate. Labored breathing is detected as a 3x increase in amplitude variance relative to a learned baseline. Cheyne-Stokes respiration is a pathological breathing pattern with 30-90 second periodicity, commonly associated with heart failure and neurological conditions. The module detects it via autocorrelation of the breathing amplitude envelope.
|
||||
|
||||
**How it works**:
|
||||
1. Maintains a 120-second ring buffer of breathing BPM for autocorrelation analysis
|
||||
2. Maintains a 60-second ring buffer of amplitude variance
|
||||
3. Learns a baseline variance over the first 60 seconds (Welford online mean)
|
||||
4. Checks for tachypnea: breathing rate > 25 BPM sustained for 8+ seconds
|
||||
5. Checks for labored breathing: current variance > 3x baseline variance
|
||||
6. Checks for Cheyne-Stokes: significant autocorrelation peak in 30-90s lag range
|
||||
7. Computes composite distress score (0-100) every 30 seconds based on: rate deviation from normal (16 BPM center), variance ratio, tachypnea flag, and recent Cheyne-Stokes detection
|
||||
8. NaN inputs are excluded from ring buffers to prevent contamination
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `RespiratoryDistressDetector` | struct | Main detector state |
|
||||
| `RespiratoryDistressDetector::new()` | `const fn` | Create detector with zeroed state |
|
||||
| `process_frame(breathing_bpm, phase, variance)` | method | Process one frame at ~1 Hz; returns event slice |
|
||||
| `last_distress_score()` | method | Most recent composite score (0-100) |
|
||||
| `frame_count()` | method | Total frames processed |
|
||||
| `TACHYPNEA_THRESH` | const | 25.0 BPM (conservative; clinical is 20 BPM) |
|
||||
| `SUSTAINED_SECS` | const | 8 seconds |
|
||||
| `LABORED_VAR_RATIO` | const | 3.0x baseline |
|
||||
| `CS_LAG_MIN` / `CS_LAG_MAX` | const | 30 / 90 seconds (Cheyne-Stokes period range) |
|
||||
| `CS_PEAK_THRESH` | const | 0.35 (normalized autocorrelation) |
|
||||
| `BASELINE_SECS` | const | 60 seconds (learning period) |
|
||||
| `COOLDOWN_SECS` | const | 20 seconds |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Clinical Meaning |
|
||||
|----------|----------|-------|-----------------|
|
||||
| 120 | `EVENT_TACHYPNEA` | Current breathing BPM | Breathing rate sustained above 25 BPM for 8+ seconds |
|
||||
| 121 | `EVENT_LABORED_BREATHING` | Variance ratio | Breathing effort > 3x baseline; possible respiratory distress |
|
||||
| 122 | `EVENT_CHEYNE_STOKES` | Period in seconds | Crescendo-decrescendo breathing pattern; associated with heart failure |
|
||||
| 123 | `EVENT_RESP_DISTRESS_LEVEL` | Score 0-100 | Composite severity: 0-20 normal, 20-50 mild, 50-80 moderate, 80-100 severe |
|
||||
|
||||
#### State Machine
|
||||
|
||||
The respiratory distress module uses independent detector tracks with cooldowns rather than a single state machine:
|
||||
|
||||
```
|
||||
For each frame:
|
||||
1. Tick cooldowns (3 independent timers)
|
||||
2. Skip NaN inputs for ring buffer updates
|
||||
3. Update breathing BPM ring buffer (120s) and variance ring buffer (60s)
|
||||
4. Learn baseline variance during first 60 seconds (Welford)
|
||||
5. Tachypnea check: BPM > 25 for 8+ consecutive seconds
|
||||
6. Labored breathing: current variance mean > 3x baseline (after baseline period)
|
||||
7. Cheyne-Stokes: autocorrelation peak > 0.35 in 30-90s lag range (needs full 120s buffer)
|
||||
8. Composite distress score emitted every 30 seconds
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Clinical Range | Description |
|
||||
|-----------|---------|----------------|-------------|
|
||||
| `TACHYPNEA_THRESH` | 25.0 | 20-30 BPM | Breathing rate for tachypnea alert |
|
||||
| `SUSTAINED_SECS` | 8 | 5-15 s | Debounce period for tachypnea |
|
||||
| `LABORED_VAR_RATIO` | 3.0 | 2.0-5.0 | Variance ratio above baseline |
|
||||
| `AC_WINDOW` | 120 | 90-180 s | Autocorrelation buffer for Cheyne-Stokes |
|
||||
| `CS_PEAK_THRESH` | 0.35 | 0.25-0.50 | Autocorrelation peak threshold |
|
||||
| `CS_LAG_MIN` / `CS_LAG_MAX` | 30 / 90 | 20-120 s | Cheyne-Stokes period search range |
|
||||
| `BASELINE_SECS` | 60 | 30-120 s | Duration to learn baseline variance |
|
||||
| `DISTRESS_REPORT_INTERVAL` | 30 | 10-60 s | How often composite score is emitted |
|
||||
| `COOLDOWN_SECS` | 20 | 10-60 s | Minimum time between repeated alerts |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::med_respiratory_distress::*;
|
||||
|
||||
let mut detector = RespiratoryDistressDetector::new();
|
||||
|
||||
// Build baseline with normal breathing (60 seconds)
|
||||
for _ in 0..60 {
|
||||
detector.process_frame(16.0, 0.0, 0.5);
|
||||
}
|
||||
|
||||
// Simulate respiratory distress: high rate + high variance
|
||||
for _ in 0..30 {
|
||||
let events = detector.process_frame(30.0, 0.0, 3.0);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_TACHYPNEA => println!("Tachypnea! Rate: {} BPM", value),
|
||||
EVENT_LABORED_BREATHING => println!("Labored breathing! Variance ratio: {:.1}x", value),
|
||||
EVENT_RESP_DISTRESS_LEVEL => println!("Distress score: {:.0}/100", value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up ICU/Ward Monitoring
|
||||
|
||||
1. **Placement**: Mount the ESP32 at the foot of the bed or on the ceiling directly above the patient. The sensor needs clear WiFi signal reflection from the patient's torso.
|
||||
|
||||
2. **Baseline learning**: The module automatically learns a 60-second baseline variance when first activated. Ensure the patient is breathing normally during this calibration period. If the patient is already in distress at module start, the baseline will be skewed and labored-breathing detection will be unreliable.
|
||||
|
||||
3. **Cheyne-Stokes detection**: Requires at least 120 seconds of data to begin autocorrelation analysis. The 30-90 second periodicity search range covers the clinically documented Cheyne-Stokes cycle range. In practice, detection typically becomes reliable after 3-4 minutes of monitoring.
|
||||
|
||||
4. **Distress score interpretation**: The composite score (0-100) combines four factors: rate deviation from normal, variance ratio, tachypnea presence, and Cheyne-Stokes detection. A score above 50 warrants clinical attention. Above 80 suggests acute distress.
|
||||
|
||||
---
|
||||
|
||||
### Gait Analysis (`med_gait_analysis.rs`)
|
||||
|
||||
**What it does**: Extracts gait parameters from CSI phase variance periodicity to assess mobility and fall risk. Detects step cadence, gait asymmetry (limping), stride variability, shuffling gait patterns (associated with Parkinson's disease), festination (involuntary acceleration), and computes a composite fall-risk score from 0-100.
|
||||
|
||||
**Clinical basis**: Normal walking cadence is 80-120 steps/min for healthy adults. Shuffling gait (>140 steps/min with low energy) is characteristic of Parkinson's disease and other neurological conditions. Festination (involuntary cadence acceleration) is a Parkinsonian feature. Gait asymmetry (left/right step interval ratio deviating from 1.0 by >15%) indicates limping or musculoskeletal issues. High stride variability (coefficient of variation) is a strong predictor of fall risk in elderly patients.
|
||||
|
||||
**How it works**:
|
||||
1. Maintains a 60-second ring buffer of phase variance and motion energy
|
||||
2. Detects steps as local maxima in the phase variance signal (peak-to-trough ratio > 1.5)
|
||||
3. Records step intervals in a 64-entry buffer
|
||||
4. Every 10 seconds, computes: cadence (60 / mean step interval), asymmetry (odd/even step interval ratio), variability (coefficient of variation)
|
||||
5. Tracks cadence history over 6 reporting periods for festination detection
|
||||
6. Shuffling is flagged when cadence > 140 and motion energy is low
|
||||
7. Festination is detected as cadence accelerating by > 1.5 steps/min/sec
|
||||
8. Fall-risk score (0-100) is a weighted composite of: abnormal cadence (25%), asymmetry (25%), variability (25%), low energy (15%), festination (10%)
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `GaitAnalyzer` | struct | Main analyzer state |
|
||||
| `GaitAnalyzer::new()` | `const fn` | Create analyzer with zeroed state |
|
||||
| `process_frame(phase, amplitude, variance, motion_energy)` | method | Process one frame at ~1 Hz; returns event slice |
|
||||
| `last_cadence()` | method | Most recent cadence (steps/min) |
|
||||
| `last_asymmetry()` | method | Most recent asymmetry ratio (1.0 = symmetric) |
|
||||
| `last_fall_risk()` | method | Most recent fall-risk score (0-100) |
|
||||
| `frame_count()` | method | Total frames processed |
|
||||
| `NORMAL_CADENCE_LOW` / `HIGH` | const | 80.0 / 120.0 steps/min |
|
||||
| `SHUFFLE_CADENCE_HIGH` | const | 140.0 steps/min |
|
||||
| `ASYMMETRY_THRESH` | const | 0.15 (15% deviation from 1.0) |
|
||||
| `FESTINATION_ACCEL` | const | 1.5 steps/min/sec |
|
||||
| `REPORT_INTERVAL` | const | 10 seconds |
|
||||
| `COOLDOWN_SECS` | const | 15 seconds |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Clinical Meaning |
|
||||
|----------|----------|-------|-----------------|
|
||||
| 130 | `EVENT_STEP_CADENCE` | Steps/min | Detected walking cadence; <80 or >120 is abnormal |
|
||||
| 131 | `EVENT_GAIT_ASYMMETRY` | Ratio (1.0=symmetric) | Step interval asymmetry; >1.15 or <0.85 indicates limping |
|
||||
| 132 | `EVENT_FALL_RISK_SCORE` | Score 0-100 | Composite: 0-25 low, 25-50 moderate, 50-75 high, 75-100 critical |
|
||||
| 133 | `EVENT_SHUFFLING_DETECTED` | Cadence (steps/min) | High-frequency, low-amplitude gait; Parkinson's indicator |
|
||||
| 134 | `EVENT_FESTINATION` | Cadence (steps/min) | Involuntary cadence acceleration; Parkinsonian feature |
|
||||
|
||||
#### State Machine
|
||||
|
||||
The gait analyzer operates on a periodic reporting cycle:
|
||||
|
||||
```
|
||||
Continuous (every frame):
|
||||
- Push variance and energy into ring buffers
|
||||
- Detect step peaks (local max in variance > 1.5x neighbors)
|
||||
- Record step intervals
|
||||
|
||||
Every REPORT_INTERVAL (10s), if >= 4 steps detected:
|
||||
1. Compute cadence, asymmetry, variability
|
||||
2. Emit EVENT_STEP_CADENCE
|
||||
3. If asymmetry > threshold: emit EVENT_GAIT_ASYMMETRY
|
||||
4. If cadence > 140 and energy < 0.3: emit EVENT_SHUFFLING_DETECTED
|
||||
5. If cadence accelerating > 1.5/s over 3 periods: emit EVENT_FESTINATION
|
||||
6. Compute and emit EVENT_FALL_RISK_SCORE
|
||||
7. Reset step buffer for next window
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Clinical Range | Description |
|
||||
|-----------|---------|----------------|-------------|
|
||||
| `GAIT_WINDOW` | 60 | 30-120 s | Ring buffer size for phase variance |
|
||||
| `STEP_PEAK_RATIO` | 1.5 | 1.2-2.0 | Min peak-to-trough ratio for step detection |
|
||||
| `NORMAL_CADENCE_LOW` | 80.0 | 70-90 steps/min | Lower bound of normal cadence |
|
||||
| `NORMAL_CADENCE_HIGH` | 120.0 | 110-130 steps/min | Upper bound of normal cadence |
|
||||
| `SHUFFLE_CADENCE_HIGH` | 140.0 | 120-160 steps/min | Cadence threshold for shuffling |
|
||||
| `SHUFFLE_ENERGY_LOW` | 0.3 | 0.1-0.5 | Energy ceiling for shuffling detection |
|
||||
| `FESTINATION_ACCEL` | 1.5 | 1.0-3.0 steps/min/s | Cadence acceleration threshold |
|
||||
| `ASYMMETRY_THRESH` | 0.15 | 0.10-0.25 | Asymmetry ratio deviation from 1.0 |
|
||||
| `REPORT_INTERVAL` | 10 | 5-30 s | Gait analysis reporting period |
|
||||
| `MIN_MOTION_ENERGY` | 0.1 | 0.05-0.3 | Minimum energy for step detection |
|
||||
| `COOLDOWN_SECS` | 15 | 10-30 s | Cooldown for shuffling/festination alerts |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::med_gait_analysis::*;
|
||||
|
||||
let mut analyzer = GaitAnalyzer::new();
|
||||
|
||||
// Simulate walking with alternating high/low variance (steps)
|
||||
for i in 0..30 {
|
||||
let variance = if i % 2 == 0 { 5.0 } else { 0.5 };
|
||||
let events = analyzer.process_frame(0.0, 1.0, variance, 1.0);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_STEP_CADENCE => println!("Cadence: {:.0} steps/min", value),
|
||||
EVENT_FALL_RISK_SCORE => println!("Fall risk: {:.0}/100", value),
|
||||
EVENT_GAIT_ASYMMETRY => println!("Asymmetry: {:.2}", value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up Hallway Gait Monitoring
|
||||
|
||||
1. **Placement**: Mount the ESP32 in a hallway or corridor at waist height on the wall. The walking path should be 3-5 meters long within the sensor's field of view. Position the WiFi AP at the opposite end of the hallway for optimal body reflection.
|
||||
|
||||
2. **Calibration**: The step detector relies on periodic peaks in phase variance. The `STEP_PEAK_RATIO` of 1.5 works well for most flooring surfaces. On carpet (which dampens impact signals), consider lowering to 1.2. On hard floors with shoes, 1.5-2.0 is appropriate.
|
||||
|
||||
3. **Clinical context**: The fall-risk score is most useful for longitudinal monitoring. A single reading provides a snapshot, but tracking trends over days/weeks reveals progressive mobility decline. A rising fall-risk score (e.g., from 20 to 40 over a month) warrants clinical assessment even if individual readings are below the "high risk" threshold.
|
||||
|
||||
4. **Limitations**: At a 1 Hz timer rate, the module cannot detect cadences above ~60 steps/min via direct peak counting. For higher cadences, the step detection relies on the host's higher-rate CSI processing to pre-compute variance peaks. Shuffling detection at >140 steps/min requires the host to be providing step-level variance data at higher than 1 Hz.
|
||||
|
||||
---
|
||||
|
||||
### Seizure Detection (`med_seizure_detect.rs`)
|
||||
|
||||
**What it does**: Detects tonic-clonic (grand mal) seizures by identifying sustained high-energy rhythmic motion in the 3-8 Hz band. Discriminates seizures from falls (single impulse followed by stillness) and tremor (lower amplitude, higher regularity). Tracks seizure phases: tonic (sustained muscle rigidity), clonic (rhythmic jerking), and post-ictal (sudden cessation of movement).
|
||||
|
||||
**Clinical basis**: Tonic-clonic seizures have a characteristic progression: (1) tonic phase with sustained muscle rigidity causing high motion energy with low variance, lasting 10-20 seconds; (2) clonic phase with rhythmic jerking at 3-8 Hz, lasting 30-60 seconds; (3) post-ictal phase with sudden cessation of movement and deep unresponsiveness. Falls produce a brief (<10 frame) high-energy spike followed by stillness. Tremors have lower amplitude than seizure-grade jerking.
|
||||
|
||||
**How it works**:
|
||||
1. Operates at ~20 Hz frame rate (higher than other modules) for rhythm detection
|
||||
2. Maintains 100-frame ring buffers for motion energy and amplitude
|
||||
3. State machine progresses: Monitoring -> PossibleOnset -> Tonic/Clonic -> PostIctal -> Cooldown
|
||||
4. Onset requires 10+ consecutive frames of high motion energy (>2.0 normalized)
|
||||
5. Fall discrimination: if high energy lasts < 10 frames then drops, it is classified as a fall and ignored
|
||||
6. Tonic phase: high energy with low variance (< 0.5)
|
||||
7. Clonic phase: detected via autocorrelation of amplitude buffer for 2-7 frame period (3-8 Hz at 20 Hz sampling)
|
||||
8. Post-ictal: motion drops below 0.2 for 40+ consecutive frames
|
||||
9. After an episode, 200-frame cooldown prevents re-triggering
|
||||
10. Presence must be active; loss of presence resets the state machine
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `SeizureDetector` | struct | Main detector state |
|
||||
| `SeizureDetector::new()` | `const fn` | Create detector with zeroed state |
|
||||
| `process_frame(phase, amplitude, motion_energy, presence)` | method | Process at ~20 Hz; returns event slice |
|
||||
| `phase()` | method | Current `SeizurePhase` enum value |
|
||||
| `seizure_count()` | method | Total seizure episodes detected |
|
||||
| `frame_count()` | method | Total frames processed |
|
||||
| `SeizurePhase` | enum | Monitoring, PossibleOnset, Tonic, Clonic, PostIctal, Cooldown |
|
||||
| `HIGH_ENERGY_THRESH` | const | 2.0 (normalized) |
|
||||
| `TONIC_MIN_FRAMES` | const | 20 frames (1 second at 20 Hz) |
|
||||
| `CLONIC_PERIOD_MIN` / `MAX` | const | 2 / 7 frames (3-8 Hz at 20 Hz) |
|
||||
| `POST_ICTAL_MIN_FRAMES` | const | 40 frames (2 seconds at 20 Hz) |
|
||||
| `COOLDOWN_FRAMES` | const | 200 frames (10 seconds at 20 Hz) |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | Value | Clinical Meaning |
|
||||
|----------|----------|-------|-----------------|
|
||||
| 140 | `EVENT_SEIZURE_ONSET` | Motion energy | Seizure activity detected; immediate clinical attention needed |
|
||||
| 141 | `EVENT_SEIZURE_TONIC` | Duration in frames | Tonic phase identified; sustained rigidity |
|
||||
| 142 | `EVENT_SEIZURE_CLONIC` | Period in frames | Clonic phase identified; rhythmic jerking with detected periodicity |
|
||||
| 143 | `EVENT_POST_ICTAL` | 1.0 | Post-ictal phase; movement has ceased after seizure |
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
presence lost (from any active state)
|
||||
+-----------------------------------------+
|
||||
v |
|
||||
[Monitoring] --> [PossibleOnset] --> [Tonic] --> [Clonic] --> [PostIctal] --> [Cooldown]
|
||||
^ | | | | |
|
||||
| | | +------> [PostIctal] -----+ |
|
||||
| | | (direct if energy drops) |
|
||||
| | +--------> [Clonic] |
|
||||
| | (skip tonic) |
|
||||
| | |
|
||||
| +-- timeout (200 frames) --> [Monitoring] |
|
||||
| +-- fall (<10 frames) -----> [Monitoring] |
|
||||
| |
|
||||
+------ cooldown expires (200 frames) ------------------------------------+
|
||||
```
|
||||
|
||||
Transitions:
|
||||
- **Monitoring -> PossibleOnset**: 10+ frames of motion energy > 2.0
|
||||
- **PossibleOnset -> Tonic**: Low energy variance + high energy (muscle rigidity pattern)
|
||||
- **PossibleOnset -> Clonic**: Rhythmic autocorrelation peak + amplitude above tremor floor
|
||||
- **PossibleOnset -> Monitoring**: Energy drop within 10 frames (fall) or timeout at 200 frames
|
||||
- **Tonic -> Clonic**: Energy variance increases and rhythm is detected
|
||||
- **Tonic -> PostIctal**: Motion energy drops below 0.2 for 40+ frames
|
||||
- **Clonic -> PostIctal**: Motion energy drops below 0.2 for 40+ frames
|
||||
- **PostIctal -> Cooldown**: After 40 frames in post-ictal
|
||||
- **Cooldown -> Monitoring**: After 200 frames (10 seconds)
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Clinical Range | Description |
|
||||
|-----------|---------|----------------|-------------|
|
||||
| `ENERGY_WINDOW` / `PHASE_WINDOW` | 100 | 60-200 frames | Ring buffer sizes for analysis |
|
||||
| `HIGH_ENERGY_THRESH` | 2.0 | 1.5-3.0 | Motion energy threshold for onset |
|
||||
| `TONIC_ENERGY_THRESH` | 1.5 | 1.0-2.0 | Energy threshold during tonic phase |
|
||||
| `TONIC_VAR_CEIL` | 0.5 | 0.3-1.0 | Max energy variance for tonic classification |
|
||||
| `TONIC_MIN_FRAMES` | 20 | 10-40 frames | Min frames to confirm tonic phase |
|
||||
| `CLONIC_PERIOD_MIN` / `MAX` | 2 / 7 | 2-10 frames | Period range for 3-8 Hz rhythm |
|
||||
| `CLONIC_AUTOCORR_THRESH` | 0.30 | 0.20-0.50 | Autocorrelation threshold for rhythm |
|
||||
| `CLONIC_MIN_FRAMES` | 30 | 20-60 frames | Min frames to confirm clonic phase |
|
||||
| `POST_ICTAL_ENERGY_THRESH` | 0.2 | 0.1-0.5 | Energy threshold for cessation |
|
||||
| `POST_ICTAL_MIN_FRAMES` | 40 | 20-80 frames | Min frames of low energy |
|
||||
| `FALL_MAX_DURATION` | 10 | 5-20 frames | Max high-energy duration classified as fall |
|
||||
| `TREMOR_AMPLITUDE_FLOOR` | 0.8 | 0.5-1.5 | Min amplitude to distinguish from tremor |
|
||||
| `COOLDOWN_FRAMES` | 200 | 100-400 frames | Cooldown after episode completes |
|
||||
| `ONSET_MIN_FRAMES` | 10 | 5-20 frames | Min high-energy frames before onset |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::med_seizure_detect::*;
|
||||
|
||||
let mut detector = SeizureDetector::new();
|
||||
|
||||
// Normal motion -- no seizure
|
||||
for _ in 0..200 {
|
||||
let events = detector.process_frame(0.0, 0.5, 0.3, 1);
|
||||
assert!(events.is_empty());
|
||||
}
|
||||
assert_eq!(detector.phase(), SeizurePhase::Monitoring);
|
||||
|
||||
// Tonic phase: sustained high energy, low variance
|
||||
for _ in 0..50 {
|
||||
let events = detector.process_frame(0.0, 2.0, 3.0, 1);
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_SEIZURE_ONSET => println!("SEIZURE ONSET! Energy: {}", value),
|
||||
EVENT_SEIZURE_TONIC => println!("Tonic phase: {} frames", value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Post-ictal: sudden cessation
|
||||
for _ in 0..100 {
|
||||
let events = detector.process_frame(0.0, 0.05, 0.05, 1);
|
||||
for &(event_id, _) in events {
|
||||
if event_id == EVENT_POST_ICTAL {
|
||||
println!("Post-ictal phase detected -- patient needs immediate assessment");
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up Seizure Monitoring
|
||||
|
||||
1. **Placement**: Mount the ESP32 on the ceiling directly above the bed or monitoring area. Seizure detection requires the highest sensitivity to body motion, so minimize distance to the patient. Ensure no other people or moving objects are in the sensor's field of view (pets, curtains, fans).
|
||||
|
||||
2. **Frame rate**: Unlike other medical modules that operate at 1 Hz, the seizure detector expects ~20 Hz frame input for accurate rhythm detection in the 3-8 Hz band. Ensure the host firmware is configured for high-rate CSI processing when this module is loaded.
|
||||
|
||||
3. **Sensitivity tuning**: The `HIGH_ENERGY_THRESH` of 2.0 and `ONSET_MIN_FRAMES` of 10 balance sensitivity against false positives. In a quiet bedroom environment, these defaults work well. In noisier environments (shared ward, nearby equipment vibration), consider raising `HIGH_ENERGY_THRESH` to 2.5-3.0.
|
||||
|
||||
4. **Fall vs seizure discrimination**: The module automatically distinguishes falls (brief energy spike < 10 frames) from seizures (sustained energy). If the patient is known to be a fall risk, consider running the gait analysis module in parallel for complementary monitoring.
|
||||
|
||||
5. **Response protocol**: When `EVENT_SEIZURE_ONSET` fires, immediately notify clinical staff. The `EVENT_POST_ICTAL` event indicates the active seizure has ended and the patient is entering post-ictal state -- they need assessment but are no longer in the convulsive phase.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
All medical modules include comprehensive unit tests covering initialization, normal operation, clinical scenario detection, edge cases, and cooldown behavior.
|
||||
|
||||
```bash
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm-edge
|
||||
cargo test --features std -- med_
|
||||
```
|
||||
|
||||
Expected output: **38 tests passed, 0 failed**.
|
||||
|
||||
### Test Coverage by Module
|
||||
|
||||
| Module | Tests | Scenarios Covered |
|
||||
|--------|-------|-------------------|
|
||||
| Sleep Apnea | 7 | Init, normal breathing, apnea onset/end, no monitoring without presence, AHI update, multiple episodes, presence-loss during apnea |
|
||||
| Cardiac Arrhythmia | 7 | Init, normal HR, tachycardia, bradycardia, missed beat, HRV anomaly (low variability), cooldown flood prevention, EMA convergence |
|
||||
| Respiratory Distress | 6 | Init, normal breathing, tachypnea, labored breathing, distress score emission, Cheyne-Stokes detection, distress score range |
|
||||
| Gait Analysis | 7 | Init, no events without steps, cadence extraction, fall-risk score range, asymmetry detection, shuffling detection, variability (uniform + varied) |
|
||||
| Seizure Detection | 7 | Init, normal motion, fall discrimination, seizure onset with sustained energy, post-ictal detection, no detection without presence, energy variance, cooldown after episode |
|
||||
|
||||
---
|
||||
|
||||
## Clinical Thresholds Reference
|
||||
|
||||
| Condition | Normal Range | Module Threshold | Clinical Standard | Notes |
|
||||
|-----------|-------------|------------------|-------------------|-------|
|
||||
| Breathing rate | 12-20 BPM | -- | -- | Normal adult at rest |
|
||||
| Bradypnea | < 12 BPM | Not directly detected | < 12 BPM | Gap: covered implicitly by distress score |
|
||||
| Tachypnea | > 20 BPM | > 25 BPM | > 20 BPM | Conservative threshold for CSI noise tolerance |
|
||||
| Apnea | 0 BPM | < 4 BPM for > 10s | Cessation > 10s | 4 BPM threshold accounts for CSI noise floor |
|
||||
| Bradycardia | < 60 BPM | < 50 BPM | < 60 BPM | Lower threshold avoids false positives in athletes |
|
||||
| Tachycardia | > 100 BPM | > 100 BPM | > 100 BPM | Matches clinical standard |
|
||||
| Heart rate (normal) | 60-100 BPM | -- | 60-100 BPM | -- |
|
||||
| AHI (mild apnea) | -- | > 5 events/hr | > 5 events/hr | Matches clinical standard |
|
||||
| AHI (moderate) | -- | > 15 events/hr | > 15 events/hr | Matches clinical standard |
|
||||
| AHI (severe) | -- | > 30 events/hr | > 30 events/hr | Matches clinical standard |
|
||||
| RMSSD (normal HRV) | 20-80 ms | 10-120 ms | 19-75 ms | Widened band for CSI-derived HR |
|
||||
| Gait cadence (normal) | 80-120 steps/min | 80-120 steps/min | 90-120 steps/min | Slightly wider range |
|
||||
| Gait asymmetry | 1.0 ratio | > 0.15 deviation | > 0.10 deviation | Slightly higher threshold for CSI |
|
||||
| Cheyne-Stokes period | 30-90 s | 30-90 s lag search | 30-100 s | Matches clinical range |
|
||||
| Seizure clonic frequency | 3-8 Hz | 3-8 Hz (period 2-7 frames at 20 Hz) | 3-8 Hz | Matches clinical standard |
|
||||
|
||||
### Threshold Rationale
|
||||
|
||||
Several thresholds differ from strict clinical standards. This is intentional:
|
||||
|
||||
- **WiFi CSI is not ECG/pulse oximetry.** The signal-to-noise ratio is lower, so thresholds are widened to reduce false positives while maintaining clinical relevance.
|
||||
- **Conservative thresholds favor specificity over sensitivity.** A missed alert is preferable to alert fatigue in a non-clinical-grade system.
|
||||
- **All thresholds are compile-time constants.** To adjust for a specific deployment, modify the constants at the top of each module file and recompile.
|
||||
|
||||
---
|
||||
|
||||
## Safety Considerations
|
||||
|
||||
1. **Not a substitute for medical devices.** These modules are research/assistive tools. They have not been validated through clinical trials and are not FDA/CE cleared. Never rely on them as the sole source of patient monitoring.
|
||||
|
||||
2. **False positive rates.** WiFi CSI is affected by environmental factors: moving objects (fans, pets, curtains), multipath changes (opening doors, people walking nearby), and electromagnetic interference. Expect false positive rates of 5-15% in typical home environments and 1-5% in controlled clinical settings.
|
||||
|
||||
3. **False negative rates.** The conservative thresholds mean some borderline conditions may not trigger alerts. Specifically:
|
||||
- Bradypnea (12-20 BPM dropping to 12-4 BPM) is not directly flagged -- only sub-4 BPM apnea is detected
|
||||
- Mild tachycardia (100-120 BPM) is detected, but the 10-second sustained requirement means brief episodes are missed
|
||||
- Low-amplitude seizures without strong motor components may not exceed the energy threshold
|
||||
|
||||
4. **Environmental factors affecting accuracy:**
|
||||
- **Multi-person environments**: All modules assume a single subject. Multiple people in the sensor's field of view will corrupt readings.
|
||||
- **Distance**: CSI sensitivity drops with distance. Place sensor within 2 meters of the subject.
|
||||
- **Obstructions**: Thick walls, metal furniture, and large water bodies (aquariums) between sensor and subject degrade performance.
|
||||
- **WiFi congestion**: Heavy WiFi traffic on the same channel increases noise in CSI measurements.
|
||||
|
||||
5. **Power and connectivity**: The ESP32 must maintain continuous WiFi connectivity for CSI monitoring. Power loss or WiFi disconnection will silently stop all monitoring. Consider UPS power and redundant AP placement for critical applications.
|
||||
|
||||
6. **Data privacy**: These modules process health-related data. Ensure compliance with HIPAA, GDPR, or local health data regulations when deploying in clinical or home care settings. CSI data and emitted events should be encrypted in transit and at rest.
|
||||
@@ -0,0 +1,482 @@
|
||||
# Retail & Hospitality Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Understand customer behavior without cameras or consent forms. Count queues, map foot traffic, track table turnover, measure shelf engagement -- all from WiFi signals that are already there.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Frame Budget |
|
||||
|--------|------|--------------|-----------|--------------|
|
||||
| Queue Length | `ret_queue_length.rs` | Estimates queue length and wait time using Little's Law | 400-403 | ~0.5 us/frame |
|
||||
| Dwell Heatmap | `ret_dwell_heatmap.rs` | Tracks dwell time per spatial zone (3x3 grid) | 410-413 | ~1 us/frame |
|
||||
| Customer Flow | `ret_customer_flow.rs` | Directional foot traffic counting (ingress/egress) | 420-423 | ~1.5 us/frame |
|
||||
| Table Turnover | `ret_table_turnover.rs` | Restaurant table lifecycle tracking with turnover rate | 430-433 | ~0.3 us/frame |
|
||||
| Shelf Engagement | `ret_shelf_engagement.rs` | Detects and classifies customer shelf interaction | 440-443 | ~1 us/frame |
|
||||
|
||||
All modules target the ESP32-S3 running WASM3 (ADR-040 Tier 3). They receive pre-processed CSI signals from Tier 2 DSP and emit structured events via `csi_emit_event()`.
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Queue Length Estimation (`ret_queue_length.rs`)
|
||||
|
||||
**What it does**: Estimates the number of people waiting in a queue, computes arrival and service rates, estimates wait time using Little's Law (L = lambda x W), and fires alerts when the queue exceeds a configurable threshold.
|
||||
|
||||
**How it works**: The module tracks person count changes frame-to-frame to detect arrivals (count increased or new presence with variance spike) and departures (count decreased or presence edge with low motion). Over 30-second windows, it computes arrival rate (lambda) and service rate (mu) in persons-per-minute. The queue length is smoothed via EMA on the raw person count. Wait time is estimated as `queue_length / (arrival_rate / 60)`.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 400 | `QUEUE_LENGTH` | Estimated queue length (0-20) | Every 20 frames (1s) |
|
||||
| 401 | `WAIT_TIME_ESTIMATE` | Estimated wait in seconds | Every 600 frames (30s window) |
|
||||
| 402 | `SERVICE_RATE` | Service rate (persons/min, smoothed) | Every 600 frames (30s window) |
|
||||
| 403 | `QUEUE_ALERT` | Current queue length | When queue >= 5 (once, resets below 4) |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ret_queue_length::QueueLengthEstimator;
|
||||
|
||||
let mut q = QueueLengthEstimator::new();
|
||||
|
||||
// Per-frame: presence (0/1), person count, variance, motion energy
|
||||
let events = q.process_frame(presence, n_persons, variance, motion_energy);
|
||||
|
||||
// Queries
|
||||
q.queue_length() // -> u8 (0-20, smoothed)
|
||||
q.arrival_rate() // -> f32 (persons/minute, EMA-smoothed)
|
||||
q.service_rate() // -> f32 (persons/minute, EMA-smoothed)
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `REPORT_INTERVAL` | 20 frames (1s) | Queue length report interval |
|
||||
| `SERVICE_WINDOW_FRAMES` | 600 frames (30s) | Window for rate computation |
|
||||
| `QUEUE_EMA_ALPHA` | 0.1 | EMA smoothing for queue length |
|
||||
| `RATE_EMA_ALPHA` | 0.05 | EMA smoothing for arrival/service rates |
|
||||
| `JOIN_VARIANCE_THRESH` | 0.05 | Variance spike threshold for join detection |
|
||||
| `DEPART_MOTION_THRESH` | 0.02 | Motion threshold for departure detection |
|
||||
| `QUEUE_ALERT_THRESH` | 5.0 | Queue length that triggers alert |
|
||||
| `MAX_QUEUE` | 20 | Maximum tracked queue length |
|
||||
|
||||
#### Example: Retail Queue Management
|
||||
|
||||
```python
|
||||
# React to queue events
|
||||
if event_id == 400: # QUEUE_LENGTH
|
||||
queue_len = int(value)
|
||||
dashboard.update_queue(register_id, queue_len)
|
||||
|
||||
elif event_id == 401: # WAIT_TIME_ESTIMATE
|
||||
wait_seconds = value
|
||||
signage.show(f"Estimated wait: {int(wait_seconds / 60)} min")
|
||||
|
||||
elif event_id == 403: # QUEUE_ALERT
|
||||
staff_pager.send(f"Register {register_id}: {int(value)} in queue")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Dwell Heatmap (`ret_dwell_heatmap.rs`)
|
||||
|
||||
**What it does**: Divides the sensing area into a 3x3 grid (9 zones) and tracks how long customers spend in each zone. Identifies "hot zones" (highest dwell time) and "cold zones" (lowest dwell time). Emits session summaries when the space empties, enabling store layout optimization.
|
||||
|
||||
**How it works**: Subcarriers are divided into 9 groups, one per zone. Each zone's variance is smoothed via EMA and compared against a threshold. When variance exceeds the threshold and presence is detected, dwell time accumulates at 0.05 seconds per frame. Sessions start when someone enters and end after 100 frames (5 seconds) of empty space.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value Encoding | When Emitted |
|
||||
|----------|------|----------------|--------------|
|
||||
| 410 | `DWELL_ZONE_UPDATE` | `zone_id * 1000 + dwell_seconds` | Every 600 frames (30s) per occupied zone |
|
||||
| 411 | `HOT_ZONE` | `zone_id + dwell_seconds/1000` | Every 600 frames (30s) |
|
||||
| 412 | `COLD_ZONE` | `zone_id + dwell_seconds/1000` | Every 600 frames (30s) |
|
||||
| 413 | `SESSION_SUMMARY` | Session duration in seconds | When space empties after occupancy |
|
||||
|
||||
**Value decoding for DWELL_ZONE_UPDATE**: The zone ID is encoded in the thousands place. For example, `value = 2015.5` means zone 2 with 15.5 seconds of dwell time.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ret_dwell_heatmap::DwellHeatmapTracker;
|
||||
|
||||
let mut t = DwellHeatmapTracker::new();
|
||||
|
||||
// Per-frame: presence (0/1), per-subcarrier variances, motion energy, person count
|
||||
let events = t.process_frame(presence, &variances, motion_energy, n_persons);
|
||||
|
||||
// Queries
|
||||
t.zone_dwell(zone_id) // -> f32 (seconds in current session)
|
||||
t.zone_total_dwell(zone_id) // -> f32 (seconds across all sessions)
|
||||
t.is_zone_occupied(zone_id) // -> bool
|
||||
t.is_session_active() // -> bool
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `NUM_ZONES` | 9 | Spatial zones (3x3 grid) |
|
||||
| `REPORT_INTERVAL` | 600 frames (30s) | Heatmap update interval |
|
||||
| `ZONE_OCCUPIED_THRESH` | 0.015 | Variance threshold for zone occupancy |
|
||||
| `ZONE_EMA_ALPHA` | 0.12 | EMA smoothing for zone variance |
|
||||
| `EMPTY_FRAMES_FOR_SUMMARY` | 100 frames (5s) | Vacancy duration before session end |
|
||||
| `MAX_EVENTS` | 12 | Maximum events per frame |
|
||||
|
||||
#### Zone Layout
|
||||
|
||||
The 3x3 grid maps to the physical space:
|
||||
|
||||
```
|
||||
+-------+-------+-------+
|
||||
| Z0 | Z1 | Z2 |
|
||||
| | | |
|
||||
+-------+-------+-------+
|
||||
| Z3 | Z4 | Z5 |
|
||||
| | | |
|
||||
+-------+-------+-------+
|
||||
| Z6 | Z7 | Z8 |
|
||||
| | | |
|
||||
+-------+-------+-------+
|
||||
Near Mid Far
|
||||
```
|
||||
|
||||
Subcarriers are divided evenly: with 27 subcarriers, each zone gets 3 subcarriers. Lower-index subcarriers correspond to nearer Fresnel zones.
|
||||
|
||||
---
|
||||
|
||||
### Customer Flow Counting (`ret_customer_flow.rs`)
|
||||
|
||||
**What it does**: Counts people entering and exiting through a doorway or passage using directional phase gradient analysis. Maintains cumulative ingress/egress counts and reports net occupancy (in - out, clamped to zero). Emits hourly traffic summaries.
|
||||
|
||||
**How it works**: Subcarriers are split into two groups: low-index (near entrance) and high-index (far side). A person walking through the sensing area causes an asymmetric phase velocity pattern -- the near-side group's phase changes before the far-side group for ingress, and vice versa for egress. The directional gradient (low_gradient - high_gradient) is smoothed via EMA and thresholded. Combined with motion energy and amplitude spike detection, this discriminates genuine crossings from noise.
|
||||
|
||||
```
|
||||
Ingress: positive smoothed gradient (low-side phase leads)
|
||||
Egress: negative smoothed gradient (high-side phase leads)
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 420 | `INGRESS` | Cumulative ingress count | On each detected entry |
|
||||
| 421 | `EGRESS` | Cumulative egress count | On each detected exit |
|
||||
| 422 | `NET_OCCUPANCY` | Current net occupancy (>= 0) | On crossing + every 100 frames |
|
||||
| 423 | `HOURLY_TRAFFIC` | `ingress * 1000 + egress` | Every 72000 frames (1 hour) |
|
||||
|
||||
**Decoding HOURLY_TRAFFIC**: `ingress = int(value / 1000)`, `egress = int(value % 1000)`.
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ret_customer_flow::CustomerFlowTracker;
|
||||
|
||||
let mut cf = CustomerFlowTracker::new();
|
||||
|
||||
// Per-frame: per-subcarrier phases, amplitudes, variance, motion energy
|
||||
let events = cf.process_frame(&phases, &litudes, variance, motion_energy);
|
||||
|
||||
// Queries
|
||||
cf.net_occupancy() // -> i32 (ingress - egress, clamped to 0)
|
||||
cf.total_ingress() // -> u32 (cumulative entries)
|
||||
cf.total_egress() // -> u32 (cumulative exits)
|
||||
cf.current_gradient() // -> f32 (smoothed directional gradient)
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `PHASE_GRADIENT_THRESH` | 0.15 | Minimum gradient magnitude for crossing |
|
||||
| `MOTION_THRESH` | 0.03 | Minimum motion energy for valid crossing |
|
||||
| `AMPLITUDE_SPIKE_THRESH` | 1.5 | Amplitude change scale factor |
|
||||
| `CROSSING_DEBOUNCE` | 10 frames (0.5s) | Debounce between crossing events |
|
||||
| `GRADIENT_EMA_ALPHA` | 0.2 | EMA smoothing for gradient |
|
||||
| `OCCUPANCY_REPORT_INTERVAL` | 100 frames (5s) | Net occupancy report interval |
|
||||
|
||||
#### Example: Store Occupancy Display
|
||||
|
||||
```python
|
||||
# Real-time occupancy counter at store entrance
|
||||
if event_id == 422: # NET_OCCUPANCY
|
||||
occupancy = int(value)
|
||||
display.show(f"Currently in store: {occupancy}")
|
||||
|
||||
if occupancy >= max_capacity:
|
||||
door_signal.set("WAIT")
|
||||
else:
|
||||
door_signal.set("ENTER")
|
||||
|
||||
elif event_id == 423: # HOURLY_TRAFFIC
|
||||
ingress = int(value / 1000)
|
||||
egress = int(value % 1000)
|
||||
analytics.log_hourly(hour, ingress, egress)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Table Turnover Tracking (`ret_table_turnover.rs`)
|
||||
|
||||
**What it does**: Tracks the full lifecycle of a restaurant table -- from guests sitting down, through eating, to departing and cleanup. Measures seating duration and computes a rolling turnover rate (turnovers per hour). Designed for one ESP32 node per table or table group.
|
||||
|
||||
**How it works**: A five-state machine processes presence, motion energy, and person count:
|
||||
|
||||
```
|
||||
Empty --> Eating --> Departing --> Cooldown --> Empty
|
||||
| (2s (motion (30s |
|
||||
| debounce) increase) cleanup) |
|
||||
| |
|
||||
+----------------------------------------------+
|
||||
(brief absence: stays in Eating)
|
||||
```
|
||||
|
||||
The `Seating` state exists in the enum for completeness but transitions are handled directly (Empty -> Eating after debounce). The `Departing` state detects when guests show increased motion and reduced person count. Vacancy requires 5 seconds of confirmed absence to avoid false triggers from brief bathroom breaks.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 430 | `TABLE_SEATED` | Person count at seating | After 40-frame debounce |
|
||||
| 431 | `TABLE_VACATED` | Seating duration in seconds | After 100-frame absence debounce |
|
||||
| 432 | `TABLE_AVAILABLE` | 1.0 | After 30-second cleanup cooldown |
|
||||
| 433 | `TURNOVER_RATE` | Turnovers per hour (rolling) | Every 6000 frames (5 min) |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ret_table_turnover::TableTurnoverTracker;
|
||||
|
||||
let mut tt = TableTurnoverTracker::new();
|
||||
|
||||
// Per-frame: presence (0/1), motion energy, person count
|
||||
let events = tt.process_frame(presence, motion_energy, n_persons);
|
||||
|
||||
// Queries
|
||||
tt.state() // -> TableState (Empty|Seating|Eating|Departing|Cooldown)
|
||||
tt.total_turnovers() // -> u32 (cumulative turnovers)
|
||||
tt.session_duration_s() // -> f32 (current session length in seconds)
|
||||
tt.turnover_rate() // -> f32 (turnovers/hour, rolling window)
|
||||
```
|
||||
|
||||
#### State Machine
|
||||
|
||||
| State | Entry Condition | Exit Condition |
|
||||
|-------|----------------|----------------|
|
||||
| `Empty` | Table is free | 40 frames (2s) of continuous presence |
|
||||
| `Eating` | Guests confirmed seated | 100 frames (5s) of absence -> Cooldown; high motion + fewer people -> Departing |
|
||||
| `Departing` | High motion with dropping count | 100 frames absence -> Cooldown; motion settles -> back to Eating |
|
||||
| `Cooldown` | Table vacated, cleanup period | 600 frames (30s) -> Empty; presence during cooldown -> Eating (fast re-seat) |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `SEATED_DEBOUNCE_FRAMES` | 40 frames (2s) | Confirmation before marking seated |
|
||||
| `VACATED_DEBOUNCE_FRAMES` | 100 frames (5s) | Absence confirmation before vacating |
|
||||
| `AVAILABLE_COOLDOWN_FRAMES` | 600 frames (30s) | Cleanup time before marking available |
|
||||
| `EATING_MOTION_THRESH` | 0.1 | Motion below this = settled/eating |
|
||||
| `ACTIVE_MOTION_THRESH` | 0.3 | Motion above this = arriving/departing |
|
||||
| `TURNOVER_REPORT_INTERVAL` | 6000 frames (5 min) | Rate report interval |
|
||||
| `MAX_TURNOVERS` | 50 | Rolling window buffer for rate |
|
||||
|
||||
#### Example: Restaurant Operations Dashboard
|
||||
|
||||
```python
|
||||
# Restaurant table management
|
||||
if event_id == 430: # TABLE_SEATED
|
||||
party_size = int(value)
|
||||
kitchen.notify(f"Table {table_id}: {party_size} guests seated")
|
||||
pos.start_timer(table_id)
|
||||
|
||||
elif event_id == 431: # TABLE_VACATED
|
||||
duration_s = value
|
||||
analytics.log_seating(table_id, duration_s, peak_persons)
|
||||
staff.alert(f"Table {table_id}: needs bussing ({duration_s/60:.0f} min use)")
|
||||
|
||||
elif event_id == 432: # TABLE_AVAILABLE
|
||||
hostess_display.mark_available(table_id)
|
||||
|
||||
elif event_id == 433: # TURNOVER_RATE
|
||||
rate = value
|
||||
manager_dashboard.update(table_id, turnovers_per_hour=rate)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Shelf Engagement Detection (`ret_shelf_engagement.rs`)
|
||||
|
||||
**What it does**: Detects when a customer stops in front of a shelf and classifies their engagement level: Browse (under 5 seconds), Consider (5-30 seconds), or Deep Engagement (over 30 seconds). Also detects reaching gestures (hand/arm movement toward the shelf). Uses the principle that a person standing still but interacting with products produces high-frequency phase perturbations with low translational motion.
|
||||
|
||||
**How it works**: The key insight is distinguishing two types of CSI phase changes:
|
||||
- **Translational motion** (walking): Large uniform phase shifts across all subcarriers
|
||||
- **Localized interaction** (reaching, examining): High spatial variance in frame-to-frame phase differences
|
||||
|
||||
The module computes the standard deviation of per-subcarrier phase differences. High std-dev with low overall motion indicates shelf interaction. A reach gesture produces a burst of high-frequency perturbation exceeding a higher threshold.
|
||||
|
||||
#### Engagement Classification
|
||||
|
||||
| Level | Duration | Description | Event ID |
|
||||
|-------|----------|-------------|----------|
|
||||
| None | -- | No engagement (absent or walking) | -- |
|
||||
| Browse | < 5s | Brief glance, passing interest | 440 |
|
||||
| Consider | 5-30s | Examining, reading label, comparing | 441 |
|
||||
| Deep Engage | > 30s | Extended interaction, decision-making | 442 |
|
||||
|
||||
The `REACH_DETECTED` event (443) fires independently whenever a sudden high-frequency phase burst is detected while the customer is standing still.
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Name | Value | When Emitted |
|
||||
|----------|------|-------|--------------|
|
||||
| 440 | `SHELF_BROWSE` | Engagement duration in seconds | On classification (with cooldown) |
|
||||
| 441 | `SHELF_CONSIDER` | Engagement duration in seconds | On level upgrade |
|
||||
| 442 | `SHELF_ENGAGE` | Engagement duration in seconds | On level upgrade |
|
||||
| 443 | `REACH_DETECTED` | Phase perturbation magnitude | Per reach burst |
|
||||
|
||||
#### API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::ret_shelf_engagement::ShelfEngagementDetector;
|
||||
|
||||
let mut se = ShelfEngagementDetector::new();
|
||||
|
||||
// Per-frame: presence (0/1), motion energy, variance, per-subcarrier phases
|
||||
let events = se.process_frame(presence, motion_energy, variance, &phases);
|
||||
|
||||
// Queries
|
||||
se.engagement_level() // -> EngagementLevel (None|Browse|Consider|DeepEngage)
|
||||
se.engagement_duration_s() // -> f32 (seconds)
|
||||
se.total_browse_events() // -> u32
|
||||
se.total_consider_events() // -> u32
|
||||
se.total_engage_events() // -> u32
|
||||
se.total_reach_events() // -> u32
|
||||
```
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Description |
|
||||
|----------|-------|-------------|
|
||||
| `BROWSE_THRESH_S` | 5.0s (100 frames) | Engagement time for Browse |
|
||||
| `CONSIDER_THRESH_S` | 30.0s (600 frames) | Engagement time for Consider |
|
||||
| `STILL_MOTION_THRESH` | 0.08 | Motion below this = standing still |
|
||||
| `PHASE_PERTURBATION_THRESH` | 0.04 | Phase variance for interaction |
|
||||
| `REACH_BURST_THRESH` | 0.15 | Phase burst for reach detection |
|
||||
| `STILL_DEBOUNCE` | 10 frames (0.5s) | Stillness confirmation before counting |
|
||||
| `ENGAGEMENT_COOLDOWN` | 60 frames (3s) | Cooldown between engagement events |
|
||||
|
||||
#### Example: Planogram Analytics
|
||||
|
||||
```python
|
||||
# Shelf performance analytics
|
||||
shelf_stats = defaultdict(lambda: {"browse": 0, "consider": 0, "engage": 0, "reaches": 0})
|
||||
|
||||
if event_id == 440: # SHELF_BROWSE
|
||||
shelf_stats[shelf_id]["browse"] += 1
|
||||
elif event_id == 441: # SHELF_CONSIDER
|
||||
shelf_stats[shelf_id]["consider"] += 1
|
||||
elif event_id == 442: # SHELF_ENGAGE
|
||||
shelf_stats[shelf_id]["engage"] += 1
|
||||
duration_s = value
|
||||
if duration_s > 60:
|
||||
analytics.flag_decision_difficulty(shelf_id)
|
||||
elif event_id == 443: # REACH_DETECTED
|
||||
shelf_stats[shelf_id]["reaches"] += 1
|
||||
|
||||
# Conversion funnel: Browse -> Consider -> Engage
|
||||
# Low consider-to-engage ratio = poor shelf placement or pricing
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Retail Store Layout Optimization
|
||||
|
||||
Deploy ESP32 nodes at key locations:
|
||||
- **Entrance**: Customer Flow module counts foot traffic and peak hours
|
||||
- **Checkout lanes**: Queue Length module monitors wait times, triggers "open register" alerts
|
||||
- **Aisles**: Dwell Heatmap identifies high-traffic zones for premium product placement
|
||||
- **Endcaps/displays**: Shelf Engagement measures which displays convert attention to interaction
|
||||
|
||||
```
|
||||
Entrance
|
||||
(CustomerFlow)
|
||||
|
|
||||
+--------------+--------------+
|
||||
| | |
|
||||
Aisle 1 Aisle 2 Aisle 3
|
||||
(DwellHeatmap) (DwellHeatmap) (DwellHeatmap)
|
||||
| | |
|
||||
[Shelf A] [Shelf B] [Shelf C]
|
||||
(ShelfEngage) (ShelfEngage) (ShelfEngage)
|
||||
| | |
|
||||
+--------------+--------------+
|
||||
|
|
||||
Checkout Area
|
||||
(QueueLength x3)
|
||||
```
|
||||
|
||||
### Restaurant Operations
|
||||
|
||||
Deploy per-table ESP32 nodes plus entrance/exit nodes:
|
||||
|
||||
- **Entrance**: Customer Flow tracks customer arrivals
|
||||
- **Each table**: Table Turnover monitors seating lifecycle
|
||||
- **Host stand**: Queue Length estimates wait time for walk-ins
|
||||
- **Kitchen view**: Dwell Heatmap identifies server traffic patterns
|
||||
|
||||
Key metrics:
|
||||
- Average seating duration per table
|
||||
- Turnovers per hour (efficiency)
|
||||
- Peak vs. off-peak utilization
|
||||
- Wait time vs. party size correlation
|
||||
|
||||
### Shopping Mall Analytics
|
||||
|
||||
Multi-floor, multi-zone deployment:
|
||||
|
||||
- **Mall entrances** (4-8 nodes): Customer Flow for total foot traffic + directionality
|
||||
- **Food court**: Table Turnover + Queue Length per restaurant
|
||||
- **Anchor store entrances**: Customer Flow per store
|
||||
- **Common areas**: Dwell Heatmap for seating area utilization
|
||||
- **Kiosks/pop-ups**: Shelf Engagement for promotional display effectiveness
|
||||
|
||||
### Event Venue Management
|
||||
|
||||
- **Gates**: Customer Flow for entry/exit counting, capacity monitoring
|
||||
- **Concession stands**: Queue Length with staff dispatch alerts
|
||||
- **Seating sections**: Dwell Heatmap for section utilization
|
||||
- **Merchandise areas**: Shelf Engagement for product interest
|
||||
|
||||
---
|
||||
|
||||
## Integration Architecture
|
||||
|
||||
```
|
||||
ESP32 Nodes (per zone)
|
||||
|
|
||||
v UDP events (port 5005)
|
||||
Sensing Server (wifi-densepose-sensing-server)
|
||||
|
|
||||
v REST API + WebSocket
|
||||
+---+---+---+---+
|
||||
| | | | |
|
||||
v v v v v
|
||||
POS Dashboard Staff Analytics
|
||||
Pager Backend
|
||||
```
|
||||
|
||||
### Event Packet Format
|
||||
|
||||
Each event is a `(event_type: i32, value: f32)` pair. Multiple events per frame are packed into a single UDP packet. The sensing server deserializes and exposes them via:
|
||||
|
||||
- `GET /api/v1/sensing/latest` -- latest raw events
|
||||
- `GET /api/v1/sensing/events?type=400-403` -- filtered by event type
|
||||
- WebSocket `/ws/events` -- real-time stream
|
||||
|
||||
### Privacy Considerations
|
||||
|
||||
These modules process WiFi CSI data (channel amplitude and phase), not video or personally identifiable information. No MAC addresses, device identifiers, or individual tracking data leaves the ESP32. All output is aggregate metrics: counts, durations, zone labels. This makes WiFi sensing suitable for jurisdictions with strict privacy requirements (GDPR, CCPA) where camera-based analytics would require consent forms or impact assessments.
|
||||
@@ -0,0 +1,615 @@
|
||||
# Security & Safety Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Perimeter monitoring and threat detection using WiFi Channel State Information (CSI).
|
||||
> Works through walls, in complete darkness, without visible cameras.
|
||||
> Each module runs on an $8 ESP32-S3 chip at 20 Hz frame rate.
|
||||
> All modules are `no_std`-compatible and compile to WASM for hot-loading via ADR-040 Tier 3.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| Intrusion Detection | `intrusion.rs` | Phase/amplitude anomaly intrusion alarm with arm/disarm | 200-203 | S (<5 ms) |
|
||||
| Perimeter Breach | `sec_perimeter_breach.rs` | Multi-zone perimeter crossing with approach/departure | 210-213 | S (<5 ms) |
|
||||
| Weapon Detection | `sec_weapon_detect.rs` | Concealed metallic object detection via RF reflectivity ratio | 220-222 | S (<5 ms) |
|
||||
| Tailgating Detection | `sec_tailgating.rs` | Double-peak motion envelope for unauthorized following | 230-232 | L (<2 ms) |
|
||||
| Loitering Detection | `sec_loitering.rs` | Prolonged stationary presence with 4-state machine | 240-242 | L (<2 ms) |
|
||||
| Panic Motion | `sec_panic_motion.rs` | Erratic motion, struggle, and fleeing patterns | 250-252 | S (<5 ms) |
|
||||
|
||||
Budget key: **S** = Standard (<5 ms per frame), **L** = Light (<2 ms per frame).
|
||||
|
||||
## Shared Design Patterns
|
||||
|
||||
All security modules follow these conventions:
|
||||
|
||||
- **`const fn new()`**: Zero-allocation constructor, no heap, suitable for `static mut` on ESP32.
|
||||
- **`process_frame(...) -> &[(i32, f32)]`**: Returns event tuples `(event_id, value)` via a static buffer (safe in single-threaded WASM).
|
||||
- **Calibration phase**: First N frames (typically 100-200 at 20 Hz = 5-10 seconds) learn ambient baseline. No events during calibration.
|
||||
- **Debounce**: Consecutive-frame counters prevent single-frame noise from triggering alerts.
|
||||
- **Cooldown**: After emitting an event, a cooldown window suppresses duplicate emissions (40-100 frames = 2-5 seconds).
|
||||
- **Hysteresis**: Debounce counters use `saturating_sub(1)` for gradual decay rather than hard reset, reducing flap on borderline signals.
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Intrusion Detection (`intrusion.rs`)
|
||||
|
||||
**What it does**: Monitors a previously-empty space and triggers an alarm when someone enters. Works like a traditional motion alarm -- the environment must settle before the system arms itself.
|
||||
|
||||
**How it works**: During calibration (200 frames), the detector learns per-subcarrier amplitude mean and variance. After calibration, it waits for the environment to be quiet (100 consecutive frames with low disturbance) before arming. Once armed, it computes a composite disturbance score from phase velocity (sudden phase jumps between frames) and amplitude deviation (amplitude departing from baseline by more than 3 sigma). If the disturbance exceeds 0.8 for 3+ consecutive frames, an alert fires.
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
Calibrating --> Monitoring --> Armed --> Alert
|
||||
^ |
|
||||
| (quiet for |
|
||||
| 50 frames) |
|
||||
+---- Armed <----------+
|
||||
```
|
||||
|
||||
- **Calibrating**: Accumulates baseline amplitude statistics for 200 frames.
|
||||
- **Monitoring**: Waits for 100 consecutive quiet frames before arming.
|
||||
- **Armed**: Active detection. Triggers alert on 3+ consecutive high-disturbance frames.
|
||||
- **Alert**: Active alert. Returns to Armed after 50 consecutive quiet frames. 100-frame cooldown prevents re-triggering.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `IntrusionDetector::new()` | `const fn` | Create detector in Calibrating state |
|
||||
| `process_frame(phases, amplitudes)` | `fn` | Process one CSI frame, returns events |
|
||||
| `state()` | `fn -> DetectorState` | Current state (Calibrating/Monitoring/Armed/Alert) |
|
||||
| `total_alerts()` | `fn -> u32` | Cumulative alert count |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 200 | `EVENT_INTRUSION_ALERT` | Intrusion detected (disturbance score as value) |
|
||||
| 201 | `EVENT_INTRUSION_ZONE` | Zone index of highest disturbance |
|
||||
| 202 | `EVENT_INTRUSION_ARMED` | System transitioned to Armed state |
|
||||
| 203 | `EVENT_INTRUSION_DISARMED` | System disarmed (currently unused -- reserved) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `INTRUSION_VELOCITY_THRESH` | 1.5 | 0.5-3.0 | Phase velocity threshold (rad/frame) |
|
||||
| `AMPLITUDE_CHANGE_THRESH` | 3.0 | 2.0-5.0 | Sigma multiplier for amplitude deviation |
|
||||
| `ARM_FRAMES` | 100 | 40-200 | Quiet frames required before arming (5s at 20 Hz) |
|
||||
| `DETECT_DEBOUNCE` | 3 | 2-10 | Consecutive disturbed frames before alert |
|
||||
| `ALERT_COOLDOWN` | 100 | 20-200 | Frames between re-alerts (5s at 20 Hz) |
|
||||
| `BASELINE_FRAMES` | 200 | 100-500 | Calibration frames (10s at 20 Hz) |
|
||||
|
||||
---
|
||||
|
||||
### Perimeter Breach Detection (`sec_perimeter_breach.rs`)
|
||||
|
||||
**What it does**: Divides the monitored area into 4 zones (mapped to subcarrier groups) and detects movement crossing zone boundaries. Classifies motion direction as approaching or departing using energy gradient trends.
|
||||
|
||||
**How it works**: Subcarriers are split into 4 equal groups, each representing a spatial zone. Per-zone metrics are computed every frame:
|
||||
1. **Phase gradient**: Mean absolute phase difference between current and previous frame within the zone's subcarrier range.
|
||||
2. **Variance ratio**: Current zone variance divided by calibrated baseline variance.
|
||||
|
||||
A breach is flagged when phase gradient exceeds 0.6 rad/subcarrier AND variance ratio exceeds 2.5x baseline. Direction is determined by linear regression slope over an 8-frame energy history buffer -- positive slope = approaching, negative = departing.
|
||||
|
||||
#### State Machine
|
||||
|
||||
There is no explicit state machine enum. Instead, per-zone counters track:
|
||||
- `disturb_run`: Consecutive breach frames (resets to 0 when zone is quiet).
|
||||
- `approach_run` / `departure_run`: Consecutive frames with positive/negative energy trend (debounced to 3 frames).
|
||||
- Four independent cooldown timers for breach, approach, departure, and transition events.
|
||||
|
||||
No stuck states possible: all counters either reset on quiet input or are bounded by `saturating_add`.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `PerimeterBreachDetector::new()` | `const fn` | Create uncalibrated detector |
|
||||
| `process_frame(phases, amplitudes, variance, motion_energy)` | `fn` | Process one frame, returns up to 4 events |
|
||||
| `is_calibrated()` | `fn -> bool` | Whether baseline calibration is complete |
|
||||
| `frame_count()` | `fn -> u32` | Total frames processed |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 210 | `EVENT_PERIMETER_BREACH` | Significant disturbance in any zone (value = energy score) |
|
||||
| 211 | `EVENT_APPROACH_DETECTED` | Energy trend rising in a breached zone (value = zone index) |
|
||||
| 212 | `EVENT_DEPARTURE_DETECTED` | Energy trend falling in a zone (value = zone index) |
|
||||
| 213 | `EVENT_ZONE_TRANSITION` | Movement shifted from one zone to another (value = `from*10 + to`) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `BASELINE_FRAMES` | 100 | 60-200 | Calibration frames (5s at 20 Hz) |
|
||||
| `BREACH_GRADIENT_THRESH` | 0.6 | 0.3-1.5 | Phase gradient for breach (rad/subcarrier) |
|
||||
| `VARIANCE_RATIO_THRESH` | 2.5 | 1.5-5.0 | Variance ratio above baseline for disturbance |
|
||||
| `DIRECTION_DEBOUNCE` | 3 | 2-8 | Consecutive trend frames for direction confirmation |
|
||||
| `COOLDOWN` | 40 | 20-100 | Frames between events of same type (2s at 20 Hz) |
|
||||
| `HISTORY_LEN` | 8 | 4-16 | Energy history buffer for trend estimation |
|
||||
| `MAX_ZONES` | 4 | 2-4 | Number of perimeter zones |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::sec_perimeter_breach::*;
|
||||
|
||||
let mut detector = PerimeterBreachDetector::new();
|
||||
|
||||
// Feed CSI frames (phases, amplitudes, variance arrays, motion energy scalar)
|
||||
let events = detector.process_frame(&phases, &litudes, &variance, motion_energy);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_PERIMETER_BREACH => {
|
||||
// value = energy score (higher = more severe)
|
||||
log!("Breach detected, energy={:.2}", value);
|
||||
}
|
||||
EVENT_APPROACH_DETECTED => {
|
||||
// value = zone index (0-3)
|
||||
log!("Approach in zone {}", value as u32);
|
||||
}
|
||||
EVENT_ZONE_TRANSITION => {
|
||||
// value encodes from*10 + to
|
||||
let from = (value as u32) / 10;
|
||||
let to = (value as u32) % 10;
|
||||
log!("Movement from zone {} to zone {}", from, to);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tutorial: Setting Up a 4-Zone Perimeter System
|
||||
|
||||
1. **Sensor placement**: Mount the ESP32-S3 at the center of the monitored boundary (e.g., warehouse entrance, property line). The WiFi AP should be on the opposite side so the sensing link crosses all 4 zones.
|
||||
|
||||
2. **Zone mapping**: Subcarriers are divided equally among 4 zones. With 32 subcarriers:
|
||||
- Zone 0: subcarriers 0-7 (nearest to the ESP32)
|
||||
- Zone 1: subcarriers 8-15
|
||||
- Zone 2: subcarriers 16-23
|
||||
- Zone 3: subcarriers 24-31 (nearest to the AP)
|
||||
|
||||
3. **Calibration**: Power on the system with no one in the monitored area. Wait 5 seconds (100 frames) for calibration to complete. `is_calibrated()` returns `true`.
|
||||
|
||||
4. **Alert integration**: Forward events to your security system:
|
||||
- `EVENT_PERIMETER_BREACH` (210) -> Trigger alarm siren / camera recording
|
||||
- `EVENT_APPROACH_DETECTED` (211) -> Pre-alert: someone approaching
|
||||
- `EVENT_ZONE_TRANSITION` (213) -> Track movement direction through zones
|
||||
|
||||
5. **Tuning**: If false alarms occur in windy or high-traffic environments, increase `BREACH_GRADIENT_THRESH` and `VARIANCE_RATIO_THRESH`. If detections are missed, decrease them.
|
||||
|
||||
---
|
||||
|
||||
### Concealed Metallic Object Detection (`sec_weapon_detect.rs`)
|
||||
|
||||
**What it does**: Detects concealed metallic objects (knives, firearms, tools) carried by a person walking through the sensing area. Metal has significantly higher RF reflectivity than human tissue, producing a characteristic amplitude-variance-to-phase-variance ratio.
|
||||
|
||||
**How it works**: During calibration (100 frames in an empty room), the detector computes baseline amplitude and phase variance per subcarrier using online variance accumulation. After calibration, running Welford statistics track amplitude and phase variance in real-time. The ratio of running amplitude variance to running phase variance is computed across all subcarriers. Metal produces a high ratio (amplitude swings wildly from specular reflection while phase varies less than diffuse tissue).
|
||||
|
||||
Two thresholds are applied:
|
||||
- **Metal anomaly** (ratio > 4.0, debounce 4 frames): General metallic object detection.
|
||||
- **Weapon alert** (ratio > 8.0, debounce 6 frames): High-reflectivity alert for larger metal masses.
|
||||
|
||||
Detection requires `presence >= 1` and `motion_energy >= 0.5` to avoid false positives on environmental noise.
|
||||
|
||||
**Important**: This module is research-grade and experimental. It requires per-environment calibration and should not be used as a sole security measure.
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `WeaponDetector::new()` | `const fn` | Create uncalibrated detector |
|
||||
| `process_frame(phases, amplitudes, variance, motion_energy, presence)` | `fn` | Process one frame, returns up to 3 events |
|
||||
| `is_calibrated()` | `fn -> bool` | Whether baseline calibration is complete |
|
||||
| `frame_count()` | `fn -> u32` | Total frames processed |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 220 | `EVENT_METAL_ANOMALY` | Metallic object signature detected (value = amp/phase ratio) |
|
||||
| 221 | `EVENT_WEAPON_ALERT` | High-reflectivity metal signature (value = amp/phase ratio) |
|
||||
| 222 | `EVENT_CALIBRATION_NEEDED` | Baseline drift exceeds threshold (value = max drift ratio) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `BASELINE_FRAMES` | 100 | 60-200 | Calibration frames (empty room, 5s at 20 Hz) |
|
||||
| `METAL_RATIO_THRESH` | 4.0 | 2.0-8.0 | Amp/phase variance ratio for metal detection |
|
||||
| `WEAPON_RATIO_THRESH` | 8.0 | 5.0-15.0 | Ratio for weapon-grade alert |
|
||||
| `MIN_MOTION_ENERGY` | 0.5 | 0.2-2.0 | Minimum motion to consider detection valid |
|
||||
| `METAL_DEBOUNCE` | 4 | 2-10 | Consecutive frames for metal anomaly |
|
||||
| `WEAPON_DEBOUNCE` | 6 | 3-12 | Consecutive frames for weapon alert |
|
||||
| `COOLDOWN` | 60 | 20-120 | Frames between events (3s at 20 Hz) |
|
||||
| `RECALIB_DRIFT_THRESH` | 3.0 | 2.0-5.0 | Drift ratio triggering recalibration alert |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::sec_weapon_detect::*;
|
||||
|
||||
let mut detector = WeaponDetector::new();
|
||||
|
||||
// Calibrate in empty room (100 frames)
|
||||
for _ in 0..100 {
|
||||
detector.process_frame(&phases, &litudes, &variance, 0.0, 0);
|
||||
}
|
||||
assert!(detector.is_calibrated());
|
||||
|
||||
// Normal operation: person walks through
|
||||
let events = detector.process_frame(&phases, &litudes, &variance, motion_energy, presence);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_METAL_ANOMALY => {
|
||||
log!("Metal detected, ratio={:.1}", value);
|
||||
}
|
||||
EVENT_WEAPON_ALERT => {
|
||||
log!("WEAPON ALERT, ratio={:.1}", value);
|
||||
// Trigger security response
|
||||
}
|
||||
EVENT_CALIBRATION_NEEDED => {
|
||||
log!("Environment changed, recalibration recommended");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Tailgating Detection (`sec_tailgating.rs`)
|
||||
|
||||
**What it does**: Detects tailgating at doorways -- two or more people passing through in rapid succession. A single authorized passage produces one smooth energy peak; a tailgater following closely produces a second peak within a configurable window (default 3 seconds).
|
||||
|
||||
**How it works**: The detector uses temporal clustering of motion energy peaks through a 3-state machine:
|
||||
|
||||
1. **Idle**: Waiting for motion energy to exceed the adaptive threshold.
|
||||
2. **InPeak**: Tracking an active peak. Records peak maximum energy and duration. Peak ends when energy drops below 30% of peak maximum. Noise spikes (peaks shorter than 3 frames) are discarded.
|
||||
3. **Watching**: Peak ended, monitoring for another peak within the tailgate window (60 frames = 3s). If another peak arrives, it transitions back to InPeak. When the window expires, it evaluates: 1 peak = single passage, 2+ peaks = tailgating.
|
||||
|
||||
The threshold adapts to ambient noise via exponential moving average of variance.
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
Idle ----[energy > threshold]----> InPeak
|
||||
|
|
||||
[energy < 30% of peak max]
|
||||
|
|
||||
[peak too short] v
|
||||
Idle <------------------------- InPeak end
|
||||
|
|
||||
[peak valid (>= 3 frames)]
|
||||
v
|
||||
Watching
|
||||
/ \
|
||||
[new peak starts] / \ [window expires]
|
||||
v v
|
||||
InPeak Evaluate
|
||||
/ \
|
||||
[1 peak] [2+ peaks]
|
||||
| |
|
||||
SINGLE_PASSAGE TAILGATE_DETECTED
|
||||
| + MULTI_PASSAGE
|
||||
v v
|
||||
Idle Idle
|
||||
```
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `TailgateDetector::new()` | `const fn` | Create detector |
|
||||
| `process_frame(motion_energy, presence, n_persons, variance)` | `fn` | Process one frame, returns up to 3 events |
|
||||
| `frame_count()` | `fn -> u32` | Total frames processed |
|
||||
| `tailgate_count()` | `fn -> u32` | Total tailgating events detected |
|
||||
| `single_passages()` | `fn -> u32` | Total single passages recorded |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 230 | `EVENT_TAILGATE_DETECTED` | Two or more peaks within window (value = peak count) |
|
||||
| 231 | `EVENT_SINGLE_PASSAGE` | Single peak followed by quiet window (value = peak energy) |
|
||||
| 232 | `EVENT_MULTI_PASSAGE` | Three or more peaks within window (value = peak count) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `ENERGY_PEAK_THRESH` | 2.0 | 1.0-5.0 | Motion energy threshold for peak start |
|
||||
| `ENERGY_VALLEY_FRAC` | 0.3 | 0.1-0.5 | Fraction of peak max to end peak |
|
||||
| `TAILGATE_WINDOW` | 60 | 20-120 | Max inter-peak gap for tailgating (3s at 20 Hz) |
|
||||
| `MIN_PEAK_ENERGY` | 1.5 | 0.5-3.0 | Minimum peak energy for valid passage |
|
||||
| `COOLDOWN` | 100 | 40-200 | Frames between events (5s at 20 Hz) |
|
||||
| `MIN_PEAK_FRAMES` | 3 | 2-10 | Minimum peak duration to filter noise spikes |
|
||||
| `MAX_PEAKS` | 8 | 4-16 | Maximum peaks tracked in one window |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::sec_tailgating::*;
|
||||
|
||||
let mut detector = TailgateDetector::new();
|
||||
|
||||
// Process frames from host
|
||||
let events = detector.process_frame(motion_energy, presence, n_persons, variance_mean);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_TAILGATE_DETECTED => {
|
||||
log!("TAILGATE: {} people in rapid succession", value as u32);
|
||||
// Lock door / alert security
|
||||
}
|
||||
EVENT_SINGLE_PASSAGE => {
|
||||
log!("Normal passage, energy={:.2}", value);
|
||||
}
|
||||
EVENT_MULTI_PASSAGE => {
|
||||
log!("Multi-passage: {} people", value as u32);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Loitering Detection (`sec_loitering.rs`)
|
||||
|
||||
**What it does**: Detects prolonged stationary presence in a monitored area. Distinguishes between a person passing through (normal) and someone standing still for an extended time (loitering). Default dwell threshold is 5 minutes.
|
||||
|
||||
**How it works**: Uses a 4-state machine that tracks presence duration and motion level. Only stationary frames (motion energy below 0.5) count toward the dwell threshold -- a person actively walking through does not accumulate loitering time. The exit cooldown (30 seconds) prevents false "loitering ended" events from brief signal dropouts or occlusions.
|
||||
|
||||
#### State Machine
|
||||
|
||||
```
|
||||
Absent --[presence + no post_end cooldown]--> Entering
|
||||
|
|
||||
[60 frames with presence]
|
||||
|
|
||||
[absence before 60] v
|
||||
Absent <------------------------------ Entering confirmed
|
||||
|
|
||||
v
|
||||
Present
|
||||
/ \
|
||||
[6000 stationary / \ [absent > 300
|
||||
frames] / \ frames]
|
||||
v v
|
||||
Loitering Absent
|
||||
/ \
|
||||
[presence continues] [absent >= 600 frames]
|
||||
| |
|
||||
LOITERING_ONGOING LOITERING_END
|
||||
(every 600 frames) |
|
||||
| v
|
||||
v Absent
|
||||
Loitering (post_end_cd = 200)
|
||||
```
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `LoiteringDetector::new()` | `const fn` | Create detector in Absent state |
|
||||
| `process_frame(presence, motion_energy)` | `fn` | Process one frame, returns up to 2 events |
|
||||
| `state()` | `fn -> LoiterState` | Current state (Absent/Entering/Present/Loitering) |
|
||||
| `frame_count()` | `fn -> u32` | Total frames processed |
|
||||
| `loiter_count()` | `fn -> u32` | Total loitering events |
|
||||
| `dwell_frames()` | `fn -> u32` | Current accumulated stationary dwell frames |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 240 | `EVENT_LOITERING_START` | Dwell threshold exceeded (value = dwell time in seconds) |
|
||||
| 241 | `EVENT_LOITERING_ONGOING` | Periodic report while loitering (value = total dwell seconds) |
|
||||
| 242 | `EVENT_LOITERING_END` | Loiterer departed after exit cooldown (value = total dwell seconds) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `ENTER_CONFIRM_FRAMES` | 60 | 20-120 | Presence confirmation (3s at 20 Hz) |
|
||||
| `DWELL_THRESHOLD` | 6000 | 1200-12000 | Stationary frames for loitering (5 min at 20 Hz) |
|
||||
| `EXIT_COOLDOWN` | 600 | 200-1200 | Absent frames before ending loitering (30s at 20 Hz) |
|
||||
| `STATIONARY_MOTION_THRESH` | 0.5 | 0.2-1.5 | Motion energy below which person is stationary |
|
||||
| `ONGOING_REPORT_INTERVAL` | 600 | 200-1200 | Frames between ongoing reports (30s at 20 Hz) |
|
||||
| `POST_END_COOLDOWN` | 200 | 100-600 | Cooldown after end before re-detection (10s at 20 Hz) |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::sec_loitering::*;
|
||||
|
||||
let mut detector = LoiteringDetector::new();
|
||||
|
||||
let events = detector.process_frame(presence, motion_energy);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_LOITERING_START => {
|
||||
log!("Loitering started after {:.0}s", value);
|
||||
// Alert security
|
||||
}
|
||||
EVENT_LOITERING_ONGOING => {
|
||||
log!("Still loitering, total {:.0}s", value);
|
||||
}
|
||||
EVENT_LOITERING_END => {
|
||||
log!("Loiterer departed after {:.0}s total", value);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Check state programmatically
|
||||
if detector.state() == LoiterState::Loitering {
|
||||
// Continuous monitoring actions
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Panic/Erratic Motion Detection (`sec_panic_motion.rs`)
|
||||
|
||||
**What it does**: Detects three categories of distress-related motion:
|
||||
1. **Panic**: Erratic, high-jerk motion with rapid random direction changes (e.g., someone flailing, being attacked).
|
||||
2. **Struggle**: Elevated jerk with moderate energy and some direction changes (e.g., physical altercation, trying to break free).
|
||||
3. **Fleeing**: Sustained high energy with low entropy -- running in one direction.
|
||||
|
||||
**How it works**: Maintains a 100-frame (5-second) circular buffer of motion energy and variance values. Computes window-level statistics each frame:
|
||||
|
||||
- **Mean jerk**: Average absolute rate-of-change of motion energy across the window. High jerk = erratic, unpredictable motion.
|
||||
- **Entropy proxy**: Fraction of frames with direction reversals (energy transitions from increasing to decreasing or vice versa). High entropy = chaotic motion.
|
||||
- **High jerk fraction**: Fraction of individual frame-to-frame jerks exceeding `JERK_THRESH`. Ensures the high mean is not from a single spike.
|
||||
|
||||
Detection logic:
|
||||
- **Panic** = `mean_jerk > 2.0` AND `entropy > 0.35` AND `high_jerk_frac > 0.3`
|
||||
- **Struggle** = `mean_jerk > 1.5` AND `energy in [1.0, 5.0)` AND `entropy > 0.175` AND not panic
|
||||
- **Fleeing** = `mean_energy > 5.0` AND `mean_jerk > 0.05` AND `entropy < 0.25` AND not panic
|
||||
|
||||
#### API
|
||||
|
||||
| Item | Type | Description |
|
||||
|------|------|-------------|
|
||||
| `PanicMotionDetector::new()` | `const fn` | Create detector |
|
||||
| `process_frame(motion_energy, variance_mean, phase_mean, presence)` | `fn` | Process one frame, returns up to 3 events |
|
||||
| `frame_count()` | `fn -> u32` | Total frames processed |
|
||||
| `panic_count()` | `fn -> u32` | Total panic events detected |
|
||||
|
||||
#### Events Emitted
|
||||
|
||||
| Event ID | Constant | When Emitted |
|
||||
|----------|----------|--------------|
|
||||
| 250 | `EVENT_PANIC_DETECTED` | Erratic high-jerk + high-entropy motion (value = severity 0-10) |
|
||||
| 251 | `EVENT_STRUGGLE_PATTERN` | Elevated jerk at moderate energy (value = mean jerk) |
|
||||
| 252 | `EVENT_FLEEING_DETECTED` | Sustained high-energy directional motion (value = mean energy) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Parameter | Default | Range | Description |
|
||||
|-----------|---------|-------|-------------|
|
||||
| `WINDOW` | 100 | 40-200 | Analysis window size (5s at 20 Hz) |
|
||||
| `JERK_THRESH` | 2.0 | 1.0-4.0 | Per-frame jerk threshold for panic |
|
||||
| `ENTROPY_THRESH` | 0.35 | 0.2-0.6 | Direction reversal rate threshold |
|
||||
| `MIN_MOTION` | 1.0 | 0.3-2.0 | Minimum motion energy (ignore idle) |
|
||||
| `TRIGGER_FRAC` | 0.3 | 0.2-0.5 | Fraction of window frames exceeding thresholds |
|
||||
| `COOLDOWN` | 100 | 40-200 | Frames between events (5s at 20 Hz) |
|
||||
| `FLEE_ENERGY_THRESH` | 5.0 | 3.0-10.0 | Minimum energy for fleeing detection |
|
||||
| `FLEE_JERK_THRESH` | 0.05 | 0.01-0.5 | Minimum jerk for fleeing (above noise floor) |
|
||||
| `FLEE_MAX_ENTROPY` | 0.25 | 0.1-0.4 | Maximum entropy for fleeing (directional motion) |
|
||||
| `STRUGGLE_JERK_THRESH` | 1.5 | 0.8-3.0 | Minimum mean jerk for struggle pattern |
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::sec_panic_motion::*;
|
||||
|
||||
let mut detector = PanicMotionDetector::new();
|
||||
|
||||
let events = detector.process_frame(motion_energy, variance_mean, phase_mean, presence);
|
||||
|
||||
for &(event_id, value) in events {
|
||||
match event_id {
|
||||
EVENT_PANIC_DETECTED => {
|
||||
log!("PANIC: severity={:.1}", value);
|
||||
// Immediate security dispatch
|
||||
}
|
||||
EVENT_STRUGGLE_PATTERN => {
|
||||
log!("Struggle detected, jerk={:.2}", value);
|
||||
// Investigate
|
||||
}
|
||||
EVENT_FLEEING_DETECTED => {
|
||||
log!("Person fleeing, energy={:.1}", value);
|
||||
// Track direction via perimeter module
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Event ID Registry (Security Range 200-299)
|
||||
|
||||
| Range | Module | Events |
|
||||
|-------|--------|--------|
|
||||
| 200-203 | `intrusion.rs` | INTRUSION_ALERT, INTRUSION_ZONE, INTRUSION_ARMED, INTRUSION_DISARMED |
|
||||
| 210-213 | `sec_perimeter_breach.rs` | PERIMETER_BREACH, APPROACH_DETECTED, DEPARTURE_DETECTED, ZONE_TRANSITION |
|
||||
| 220-222 | `sec_weapon_detect.rs` | METAL_ANOMALY, WEAPON_ALERT, CALIBRATION_NEEDED |
|
||||
| 230-232 | `sec_tailgating.rs` | TAILGATE_DETECTED, SINGLE_PASSAGE, MULTI_PASSAGE |
|
||||
| 240-242 | `sec_loitering.rs` | LOITERING_START, LOITERING_ONGOING, LOITERING_END |
|
||||
| 250-252 | `sec_panic_motion.rs` | PANIC_DETECTED, STRUGGLE_PATTERN, FLEEING_DETECTED |
|
||||
| 253-299 | | Reserved for future security modules |
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Run all security module tests (requires std feature)
|
||||
cd rust-port/wifi-densepose-rs/crates/wifi-densepose-wasm-edge
|
||||
cargo test --features std -- sec_ intrusion
|
||||
```
|
||||
|
||||
### Test Coverage Summary
|
||||
|
||||
| Module | Tests | Coverage Notes |
|
||||
|--------|-------|----------------|
|
||||
| `intrusion.rs` | 4 | Init, calibration, arming, intrusion detection |
|
||||
| `sec_perimeter_breach.rs` | 6 | Init, calibration, breach, zone transition, approach, quiet signal |
|
||||
| `sec_weapon_detect.rs` | 6 | Init, calibration, no presence, metal anomaly, normal person, drift recalib |
|
||||
| `sec_tailgating.rs` | 7 | Init, single passage, tailgate, wide spacing, noise spike, multi-passage, low energy |
|
||||
| `sec_loitering.rs` | 7 | Init, entering, cancel, loitering start/ongoing/end, brief absence, moving person |
|
||||
| `sec_panic_motion.rs` | 7 | Init, window fill, calm motion, panic, no presence, fleeing, struggle, low motion |
|
||||
|
||||
---
|
||||
|
||||
## Deployment Considerations
|
||||
|
||||
### Coverage Area per Sensor
|
||||
|
||||
Each ESP32-S3 with a WiFi AP link covers a single sensing path. The coverage area depends on:
|
||||
- **Distance**: 1-10 meters between ESP32 and AP (optimal: 3-5 meters for indoor).
|
||||
- **Width**: First Fresnel zone width -- approximately 0.5-1.5 meters at 5 GHz.
|
||||
- **Through-wall**: WiFi CSI penetrates drywall and wood but attenuates through concrete/metal. Signal quality degrades beyond one wall.
|
||||
|
||||
### Multi-Sensor Coordination
|
||||
|
||||
For larger areas, deploy multiple ESP32 sensors in a mesh:
|
||||
- Each sensor runs its own WASM module instance independently.
|
||||
- The aggregator server (`wifi-densepose-sensing-server`) collects events from all sensors.
|
||||
- Cross-sensor correlation (e.g., tracking a person across zones) is done server-side, not on-device.
|
||||
- Use `EVENT_ZONE_TRANSITION` (213) from perimeter breach to correlate movement across adjacent sensors.
|
||||
|
||||
### False Alarm Reduction
|
||||
|
||||
1. **Calibration**: Always calibrate in the intended operating conditions (time of day, HVAC state, door positions).
|
||||
2. **Threshold tuning**: Start with defaults, increase thresholds if false alarms occur, decrease if detections are missed.
|
||||
3. **Debounce tuning**: Increase debounce counters in high-noise environments (near HVAC vents, open windows).
|
||||
4. **Multi-module correlation**: Require 2+ modules to agree before triggering high-severity responses. For example: perimeter breach + panic motion = confirmed threat; perimeter breach alone = investigation.
|
||||
5. **Time-of-day filtering**: Server-side logic can suppress certain events during business hours (e.g., single passages are normal during the day).
|
||||
|
||||
### Integration with Existing Security Systems
|
||||
|
||||
- **Event forwarding**: Events are emitted via `csi_emit_event()` to the host firmware, which packs them into UDP packets sent to the aggregator.
|
||||
- **REST API**: The sensing server exposes events at `/api/v1/sensing/events` for integration with SIEM, VMS, or access control systems.
|
||||
- **Webhook support**: Configure the server to POST event payloads to external endpoints.
|
||||
- **MQTT**: For IoT integration, events can be published to MQTT topics (one per event type or per sensor).
|
||||
|
||||
### Resource Usage on ESP32-S3
|
||||
|
||||
| Resource | Budget | Notes |
|
||||
|----------|--------|-------|
|
||||
| RAM | ~2-4 KB per module | Static buffers, no heap allocation |
|
||||
| CPU | <5 ms per frame (S budget) | Well within 50 ms frame budget at 20 Hz |
|
||||
| Flash | ~3-8 KB WASM per module | Compiled with `opt-level = "s"` and LTO |
|
||||
| Total (6 modules) | ~15-25 KB RAM, ~30 KB Flash | Fits in 925 KB firmware with headroom |
|
||||
@@ -0,0 +1,444 @@
|
||||
# Signal Intelligence Modules -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Real-time WiFi signal analysis and enhancement running directly on the ESP32 chip. These modules clean, compress, and extract features from raw WiFi channel data so that higher-level modules (health, security, etc.) get better input.
|
||||
|
||||
## Overview
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|-------------|-----------|--------|
|
||||
| Flash Attention | `sig_flash_attention.rs` | Focuses processing on the most informative subcarrier groups | 700-702 | S (<5ms) |
|
||||
| Coherence Gate | `sig_coherence_gate.rs` | Filters out noisy/corrupted CSI frames using phase coherence | 710-712 | L (<2ms) |
|
||||
| Temporal Compress | `sig_temporal_compress.rs` | Stores CSI history in 3-tier compressed circular buffer | 705-707 | S (<5ms) |
|
||||
| Sparse Recovery | `sig_sparse_recovery.rs` | Recovers dropped subcarriers using ISTA sparse optimization | 715-717 | H (<10ms) |
|
||||
| Min-Cut Person Match | `sig_mincut_person_match.rs` | Maintains stable person IDs across frames using bipartite matching | 720-722 | H (<10ms) |
|
||||
| Optimal Transport | `sig_optimal_transport.rs` | Detects subtle motion via sliced Wasserstein distance | 725-727 | S (<5ms) |
|
||||
|
||||
## How Signal Processing Fits In
|
||||
|
||||
The signal intelligence modules form a processing pipeline between raw CSI data and application-level modules:
|
||||
|
||||
```
|
||||
Raw CSI from WiFi chipset (Tier 0-2 firmware DSP)
|
||||
|
|
||||
v
|
||||
+---------------------+ +---------------------+
|
||||
| Coherence Gate | --> | Sparse Recovery |
|
||||
| Reject noisy frames, | | Fill in dropped |
|
||||
| gate quality levels | | subcarriers via ISTA |
|
||||
+---------------------+ +---------------------+
|
||||
| |
|
||||
v v
|
||||
+---------------------+ +---------------------+
|
||||
| Flash Attention | | Temporal Compress |
|
||||
| Focus on informative | | Store CSI history |
|
||||
| subcarrier groups | | at 3 quality tiers |
|
||||
+---------------------+ +---------------------+
|
||||
| |
|
||||
v v
|
||||
+---------------------+ +---------------------+
|
||||
| Min-Cut Person Match | | Optimal Transport |
|
||||
| Track person IDs | | Detect subtle motion |
|
||||
| across frames | | via distribution |
|
||||
+---------------------+ +---------------------+
|
||||
| |
|
||||
v v
|
||||
Application modules: Health, Security, Smart Building, etc.
|
||||
```
|
||||
|
||||
The **Coherence Gate** acts as a quality filter at the top of the pipeline. Frames that pass the gate feed into the **Sparse Recovery** module (if subcarrier dropout is detected) and then into downstream analysis. **Flash Attention** identifies which spatial regions carry the most signal, while **Temporal Compress** maintains an efficient rolling history. **Min-Cut Person Match** and **Optimal Transport** extract higher-level features (person identity and motion) that application modules consume.
|
||||
|
||||
## Shared Utilities (`vendor_common.rs`)
|
||||
|
||||
All signal intelligence modules share these utilities from `vendor_common.rs`:
|
||||
|
||||
| Utility | Purpose |
|
||||
|---------|---------|
|
||||
| `CircularBuffer<N>` | Fixed-size ring buffer for phase history, stack-allocated |
|
||||
| `Ema` | Exponential moving average with configurable alpha |
|
||||
| `WelfordStats` | Online mean/variance/stddev in O(1) memory |
|
||||
| `dot_product`, `l2_norm`, `cosine_similarity` | Fixed-size vector math |
|
||||
| `dtw_distance`, `dtw_distance_banded` | Dynamic Time Warping for gesture/pattern matching |
|
||||
| `FixedPriorityQueue<CAP>` | Top-K selection without heap allocation |
|
||||
|
||||
---
|
||||
|
||||
## Modules
|
||||
|
||||
### Flash Attention (`sig_flash_attention.rs`)
|
||||
|
||||
**What it does**: Focuses processing on the WiFi channels that carry the most useful information -- ignores noise. Divides 32 subcarriers into 8 groups and computes attention weights showing where signal activity is concentrated.
|
||||
|
||||
**Algorithm**: Tiled attention (Q*K/sqrt(d)) over 8 subcarrier groups with softmax normalization and Shannon entropy tracking.
|
||||
|
||||
1. Compute group means: Q = current phase per group, K = previous phase per group, V = amplitude per group
|
||||
2. Score each group: `score[g] = Q[g] * K[g] / sqrt(8)`
|
||||
3. Softmax normalization (numerically stable: subtract max before exp)
|
||||
4. Track entropy H = -sum(p * ln(p)) via EMA smoothing
|
||||
|
||||
Low entropy means activity is focused in one spatial zone (a Fresnel region); high entropy means activity is spread uniformly.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct FlashAttention { /* ... */ }
|
||||
|
||||
impl FlashAttention {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, phases: &[f32], amplitudes: &[f32]) -> &[(i32, f32)];
|
||||
pub fn weights() -> &[f32; 8]; // Current attention weights per group
|
||||
pub fn entropy() -> f32; // EMA-smoothed entropy [0, ln(8)]
|
||||
pub fn peak_group() -> usize; // Group index with highest weight
|
||||
pub fn centroid() -> f32; // Weighted centroid position [0, 7]
|
||||
pub fn frame_count() -> u32;
|
||||
pub fn reset(&mut self);
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 700 | `ATTENTION_PEAK_SC` | Group index (0-7) | Which subcarrier group has the strongest attention weight |
|
||||
| 701 | `ATTENTION_SPREAD` | Entropy (0 to ~2.08) | How spread out the attention is (low = focused, high = uniform) |
|
||||
| 702 | `SPATIAL_FOCUS_ZONE` | Centroid (0.0-7.0) | Weighted center of attention across groups |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_GROUPS` | 8 | Number of subcarrier groups (tiles) |
|
||||
| `MAX_SC` | 32 | Maximum subcarriers processed |
|
||||
| `ENTROPY_ALPHA` | 0.15 | EMA smoothing factor for entropy |
|
||||
|
||||
#### Tutorial: Understanding Attention Weights
|
||||
|
||||
The 8 attention weights sum to 1.0. When a person stands in a particular area of the room, the WiFi signal changes most in the subcarrier group(s) whose Fresnel zones intersect that area.
|
||||
|
||||
- **All weights near 0.125 (= 1/8)**: Uniform attention. No localized activity -- either an empty room or whole-body motion affecting all subcarriers equally.
|
||||
- **One weight near 1.0, others near 0.0**: Highly focused. Activity concentrated in one spatial zone. The `peak_group` index tells you which zone.
|
||||
- **Two adjacent groups elevated**: Activity at the boundary between two spatial zones, or a person moving between them.
|
||||
- **Entropy below 1.0**: Strong spatial focus. Good for zone-level localization.
|
||||
- **Entropy above 1.8**: Nearly uniform. Hard to localize activity.
|
||||
|
||||
The `centroid` value (0.0 to 7.0) gives a weighted average position. Tracking centroid over time reveals motion direction across the room.
|
||||
|
||||
---
|
||||
|
||||
### Coherence Gate (`sig_coherence_gate.rs`)
|
||||
|
||||
**What it does**: Decides whether each incoming CSI frame is trustworthy enough to use for sensing, or should be discarded. Uses the statistical consistency of phase changes across subcarriers to measure signal quality.
|
||||
|
||||
**Algorithm**: Per-subcarrier phase deltas form unit phasors (cos + i*sin). The magnitude of the mean phasor is the coherence score [0,1]. Welford online statistics track mean/variance for Z-score computation. A hysteresis state machine prevents rapid oscillation between states.
|
||||
|
||||
State transitions:
|
||||
- Accept -> PredictOnly: 5 consecutive frames below LOW_THRESHOLD (0.40)
|
||||
- PredictOnly -> Reject: single frame below threshold
|
||||
- Reject/PredictOnly -> Accept: 10 consecutive frames above HIGH_THRESHOLD (0.75)
|
||||
- Any -> Recalibrate: running variance exceeds 4x the initial snapshot
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct CoherenceGate { /* ... */ }
|
||||
|
||||
impl CoherenceGate {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, phases: &[f32]) -> &[(i32, f32)];
|
||||
pub fn gate() -> GateDecision; // Accept/PredictOnly/Reject/Recalibrate
|
||||
pub fn coherence() -> f32; // Last coherence score [0, 1]
|
||||
pub fn zscore() -> f32; // Z-score of last coherence
|
||||
pub fn variance() -> f32; // Running variance of coherence
|
||||
pub fn frame_count() -> u32;
|
||||
pub fn reset(&mut self);
|
||||
}
|
||||
|
||||
pub enum GateDecision { Accept, PredictOnly, Reject, Recalibrate }
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 710 | `GATE_DECISION` | 2/1/0/-1 | Accept(2), PredictOnly(1), Reject(0), Recalibrate(-1) |
|
||||
| 711 | `COHERENCE_SCORE` | [0.0, 1.0] | Phase phasor coherence magnitude |
|
||||
| 712 | `RECALIBRATE_NEEDED` | Variance | Environment has changed significantly -- retrain baseline |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `HIGH_THRESHOLD` | 0.75 | Coherence above this = good quality |
|
||||
| `LOW_THRESHOLD` | 0.40 | Coherence below this = poor quality |
|
||||
| `DEGRADE_COUNT` | 5 | Consecutive bad frames before degrading |
|
||||
| `RECOVER_COUNT` | 10 | Consecutive good frames before recovering |
|
||||
| `VARIANCE_DRIFT_MULT` | 4.0 | Variance multiplier triggering recalibrate |
|
||||
|
||||
#### Tutorial: Using the Coherence Gate
|
||||
|
||||
The coherence gate protects downstream modules from processing garbage data. In practice:
|
||||
|
||||
1. **Accept** (value=2): Frame is clean. Use it for all sensing tasks (vitals, presence, gestures).
|
||||
2. **PredictOnly** (value=1): Frame quality is marginal. Use cached predictions from previous frames; do not update models.
|
||||
3. **Reject** (value=0): Frame is too noisy. Skip entirely. Do not feed to any learning module.
|
||||
4. **Recalibrate** (value=-1): The environment has changed fundamentally (furniture moved, new AP, door opened). Reset baselines and re-learn.
|
||||
|
||||
Common causes of low coherence:
|
||||
- Microwave oven running (2.4 GHz interference)
|
||||
- Multiple people walking in different directions (phase cancellation)
|
||||
- Hardware glitch (intermittent antenna contact)
|
||||
|
||||
---
|
||||
|
||||
### Temporal Compress (`sig_temporal_compress.rs`)
|
||||
|
||||
**What it does**: Maintains a rolling history of up to 512 CSI snapshots in compressed form. Recent data is stored at high precision; older data is progressively compressed to save memory while retaining long-term trends.
|
||||
|
||||
**Algorithm**: Three-tier quantization with automatic demotion at age boundaries.
|
||||
|
||||
| Tier | Age Range | Bits | Quantization Levels | Max Error |
|
||||
|------|-----------|------|---------------------|-----------|
|
||||
| Hot | 0-63 (newest) | 8-bit | 256 | <0.5% |
|
||||
| Warm | 64-255 | 5-bit | 32 | <3% |
|
||||
| Cold | 256-511 | 3-bit | 8 | <15% |
|
||||
|
||||
At 20 Hz, the buffer stores approximately:
|
||||
- Hot: 3.2 seconds of high-fidelity data
|
||||
- Warm: 9.6 seconds of medium-fidelity data
|
||||
- Cold: 12.8 seconds of low-fidelity data
|
||||
- Total: ~25.6 seconds, or longer at lower frame rates
|
||||
|
||||
Each snapshot stores 8 phase + 8 amplitude values (group means), plus a scale factor and tier tag.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct TemporalCompressor { /* ... */ }
|
||||
|
||||
impl TemporalCompressor {
|
||||
pub const fn new() -> Self;
|
||||
pub fn push_frame(&mut self, phases: &[f32], amps: &[f32], ts_ms: u32) -> &[(i32, f32)];
|
||||
pub fn on_timer() -> &[(i32, f32)];
|
||||
pub fn get_snapshot(age: usize) -> Option<[f32; 16]>; // Decompressed 8 phase + 8 amp
|
||||
pub fn compression_ratio() -> f32;
|
||||
pub fn frame_rate() -> f32;
|
||||
pub fn total_written() -> u32;
|
||||
pub fn occupied() -> usize;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 705 | `COMPRESSION_RATIO` | Ratio (>1.0) | Raw bytes / compressed bytes |
|
||||
| 706 | `TIER_TRANSITION` | Tier (1 or 2) | A snapshot was demoted to Warm(1) or Cold(2) |
|
||||
| 707 | `HISTORY_DEPTH_HOURS` | Hours | How much wall-clock time the buffer covers |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `CAP` | 512 | Total snapshot capacity |
|
||||
| `HOT_END` | 64 | First N snapshots at 8-bit precision |
|
||||
| `WARM_END` | 256 | Snapshots 64-255 at 5-bit precision |
|
||||
| `RATE_ALPHA` | 0.05 | EMA alpha for frame rate estimation |
|
||||
|
||||
---
|
||||
|
||||
### Sparse Recovery (`sig_sparse_recovery.rs`)
|
||||
|
||||
**What it does**: When WiFi hardware drops some subcarrier measurements (nulls/zeros due to deep fades, firmware glitches, or multipath nulls), this module reconstructs the missing values using mathematical optimization.
|
||||
|
||||
**Algorithm**: Iterative Shrinkage-Thresholding Algorithm (ISTA) -- an L1-minimizing sparse recovery method.
|
||||
|
||||
```
|
||||
x_{k+1} = soft_threshold(x_k + step * A^T * (b - A*x_k), lambda)
|
||||
```
|
||||
|
||||
where:
|
||||
- `A` is a tridiagonal correlation model (diagonal + immediate neighbors, 96 f32s instead of full 32x32=1024)
|
||||
- `b` is the observed (non-null) subcarrier values
|
||||
- `soft_threshold(x, t) = sign(x) * max(|x| - t, 0)` promotes sparsity
|
||||
- Maximum 10 iterations per frame
|
||||
|
||||
The correlation model is learned online from valid frames using EMA-blended products.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct SparseRecovery { /* ... */ }
|
||||
|
||||
impl SparseRecovery {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, amplitudes: &mut [f32]) -> &[(i32, f32)];
|
||||
pub fn dropout_rate() -> f32; // Fraction of null subcarriers
|
||||
pub fn last_residual_norm() -> f32; // L2 residual from last recovery
|
||||
pub fn last_recovered_count() -> u32; // How many subcarriers were recovered
|
||||
pub fn is_initialized() -> bool; // Whether correlation model is ready
|
||||
}
|
||||
```
|
||||
|
||||
Note: `process_frame` modifies `amplitudes` in place -- null subcarriers are overwritten with recovered values.
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 715 | `RECOVERY_COMPLETE` | Count | Number of subcarriers recovered |
|
||||
| 716 | `RECOVERY_ERROR` | L2 norm | Residual error of the recovery |
|
||||
| 717 | `DROPOUT_RATE` | Fraction [0,1] | Fraction of null subcarriers (emitted every 20 frames) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `NULL_THRESHOLD` | 0.001 | Amplitude below this = dropped out |
|
||||
| `MIN_DROPOUT_RATE` | 0.10 | Minimum dropout fraction to trigger recovery |
|
||||
| `MAX_ITERATIONS` | 10 | ISTA iteration cap per frame |
|
||||
| `STEP_SIZE` | 0.05 | Gradient descent learning rate |
|
||||
| `LAMBDA` | 0.01 | L1 sparsity penalty weight |
|
||||
| `CORR_ALPHA` | 0.05 | EMA alpha for correlation model updates |
|
||||
|
||||
#### Tutorial: When Recovery Kicks In
|
||||
|
||||
1. The module needs at least 10 fully valid frames to initialize the correlation model (`is_initialized() == true`).
|
||||
2. Recovery only triggers when dropout exceeds 10% (e.g., 4+ of 32 subcarriers are null).
|
||||
3. Below 10%, the nulls are too sparse to warrant recovery overhead.
|
||||
4. The tridiagonal correlation model exploits the fact that adjacent WiFi subcarriers are highly correlated. A null at subcarrier 15 can be estimated from subcarriers 14 and 16.
|
||||
5. Monitor `RECOVERY_ERROR` -- a rising residual suggests the correlation model is stale and the environment has changed.
|
||||
|
||||
---
|
||||
|
||||
### Min-Cut Person Match (`sig_mincut_person_match.rs`)
|
||||
|
||||
**What it does**: Maintains stable identity labels for up to 4 people in the sensing area. When people move around, their WiFi signatures change position -- this module tracks which signature belongs to which person across consecutive frames.
|
||||
|
||||
**Algorithm**: Inspired by `ruvector-mincut` (DynamicPersonMatcher). Each frame:
|
||||
|
||||
1. **Feature extraction**: For each detected person, extract the top-8 subcarrier variances (sorted descending) from their spatial region. This produces an 8D signature vector.
|
||||
2. **Cost matrix**: Compute L2 distances between all current features and all stored signatures.
|
||||
3. **Greedy assignment**: Pick the minimum-cost (detection, slot) pair, mark both as used, repeat. Like a simplified Hungarian algorithm, optimal for max 4 persons.
|
||||
4. **Signature update**: Blend new features into stored signatures via EMA (alpha=0.15).
|
||||
5. **Timeout**: Release slots after 100 frames of absence.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct PersonMatcher { /* ... */ }
|
||||
|
||||
impl PersonMatcher {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, amplitudes: &[f32], variances: &[f32], n_persons: usize) -> &[(i32, f32)];
|
||||
pub fn active_persons() -> u8;
|
||||
pub fn total_swaps() -> u32;
|
||||
pub fn is_person_stable(slot: usize) -> bool;
|
||||
pub fn person_signature(slot: usize) -> Option<&[f32; 8]>;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 720 | `PERSON_ID_ASSIGNED` | person_id + confidence*0.01 | Which slot was assigned (integer part) and match confidence (fractional part) |
|
||||
| 721 | `PERSON_ID_SWAP` | prev*16 + curr | An identity swap was detected (prev and curr slot indices encoded) |
|
||||
| 722 | `MATCH_CONFIDENCE` | [0.0, 1.0] | Average matching confidence across all detected persons (emitted every 10 frames) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_PERSONS` | 4 | Maximum simultaneous person tracks |
|
||||
| `FEAT_DIM` | 8 | Signature vector dimension |
|
||||
| `SIG_ALPHA` | 0.15 | EMA blending factor for signature updates |
|
||||
| `MAX_MATCH_DISTANCE` | 5.0 | L2 distance threshold for valid match |
|
||||
| `STABLE_FRAMES` | 10 | Frames before a track is considered stable |
|
||||
| `ABSENT_TIMEOUT` | 100 | Frames of absence before slot release (~5s at 20Hz) |
|
||||
|
||||
---
|
||||
|
||||
### Optimal Transport (`sig_optimal_transport.rs`)
|
||||
|
||||
**What it does**: Detects subtle motion that traditional variance-based detectors miss. Computes how much the overall shape of the WiFi signal distribution changes between frames, even when the total power stays constant.
|
||||
|
||||
**Algorithm**: Sliced Wasserstein distance -- a computationally efficient approximation to the full Wasserstein (earth mover's) distance.
|
||||
|
||||
1. Generate 4 fixed random projection directions (deterministic LCG PRNG, const-computed at compile time)
|
||||
2. Project both current and previous amplitude vectors onto each direction
|
||||
3. Sort the projected values (Shell sort with Ciura gaps, O(n^1.3))
|
||||
4. Compute 1D Wasserstein-1 distance between sorted projections (just mean absolute difference)
|
||||
5. Average across all 4 projections
|
||||
6. Smooth via EMA and compare against thresholds
|
||||
|
||||
**Subtle motion detection**: When the Wasserstein distance is elevated (distribution shape changed) but the variance is stable (total power unchanged), something moved without creating obvious disturbance -- e.g., slow hand motion, breathing, or a door slowly closing.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
pub struct OptimalTransportDetector { /* ... */ }
|
||||
|
||||
impl OptimalTransportDetector {
|
||||
pub const fn new() -> Self;
|
||||
pub fn process_frame(&mut self, amplitudes: &[f32]) -> &[(i32, f32)];
|
||||
pub fn distance() -> f32; // EMA-smoothed Wasserstein distance
|
||||
pub fn variance_smoothed() -> f32; // EMA-smoothed variance
|
||||
pub fn frame_count() -> u32;
|
||||
}
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| ID | Name | Value | Meaning |
|
||||
|----|------|-------|---------|
|
||||
| 725 | `WASSERSTEIN_DISTANCE` | Distance | Smoothed sliced Wasserstein distance (emitted every 5 frames) |
|
||||
| 726 | `DISTRIBUTION_SHIFT` | Distance | Large distribution change detected (debounced, 3 consecutive frames > 0.25) |
|
||||
| 727 | `SUBTLE_MOTION` | Distance | Motion detected despite stable variance (5 consecutive frames with distance > 0.10 and variance change < 15%) |
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_PROJ` | 4 | Number of random projection directions |
|
||||
| `ALPHA` | 0.15 | EMA alpha for distance smoothing |
|
||||
| `VAR_ALPHA` | 0.1 | EMA alpha for variance smoothing |
|
||||
| `WASS_SHIFT` | 0.25 | Wasserstein threshold for distribution shift event |
|
||||
| `WASS_SUBTLE` | 0.10 | Wasserstein threshold for subtle motion |
|
||||
| `VAR_STABLE` | 0.15 | Maximum relative variance change for "stable" classification |
|
||||
| `SHIFT_DEB` | 3 | Debounce count for distribution shift |
|
||||
| `SUBTLE_DEB` | 5 | Debounce count for subtle motion |
|
||||
|
||||
#### Tutorial: Interpreting Wasserstein Distance
|
||||
|
||||
The Wasserstein distance measures the "cost" of transforming one distribution into another. Unlike variance-based metrics that only measure spread, it captures changes in shape, location, and mode structure.
|
||||
|
||||
**Typical values:**
|
||||
- 0.00-0.05: No motion. Static environment.
|
||||
- 0.05-0.15: Breathing, subtle body sway, environmental drift.
|
||||
- 0.15-0.30: Walking, arm movement, normal activity.
|
||||
- 0.30+: Large motion, multiple people moving, or sudden environmental change.
|
||||
|
||||
**Why "subtle motion" matters**: A person sitting still and slowly raising their hand creates almost no change in total signal variance, but the Wasserstein distance increases because the spatial distribution of signal strength shifts. This is critical for:
|
||||
- Fall detection (pre-fall sway)
|
||||
- Gesture recognition (micro-movements)
|
||||
- Intruder detection (someone trying to move stealthily)
|
||||
|
||||
---
|
||||
|
||||
## Performance Budget
|
||||
|
||||
| Module | Budget Tier | Typical Latency | Stack Memory | Key Bottleneck |
|
||||
|--------|-------------|-----------------|--------------|----------------|
|
||||
| Flash Attention | S (<5ms) | ~0.5ms | ~512 bytes | Softmax exp() over 8 groups |
|
||||
| Coherence Gate | L (<2ms) | ~0.3ms | ~320 bytes | sin/cos per subcarrier |
|
||||
| Temporal Compress | S (<5ms) | ~0.8ms | ~12 KB | 512 snapshots * 24 bytes |
|
||||
| Sparse Recovery | H (<10ms) | ~3ms | ~768 bytes | 10 ISTA iterations * 32 subcarriers |
|
||||
| Min-Cut Person Match | H (<10ms) | ~1.5ms | ~640 bytes | 4x4 cost matrix + feature extraction |
|
||||
| Optimal Transport | S (<5ms) | ~1.5ms | ~1 KB | 8 Shell sorts (4 projections * 2 distributions) |
|
||||
|
||||
All latencies are estimated for ESP32-S3 running WASM3 interpreter at 240 MHz. Actual performance varies with subcarrier count and frame complexity.
|
||||
|
||||
## Memory Layout
|
||||
|
||||
All modules use fixed-size stack/static allocations. No heap, no `alloc`, no `Vec`. This is required for `no_std` WASM deployment on the ESP32-S3.
|
||||
|
||||
Total static memory for all 6 signal modules: approximately 15 KB, well within the ESP32-S3's available WASM linear memory.
|
||||
@@ -0,0 +1,448 @@
|
||||
# Spatial & Temporal Intelligence -- WiFi-DensePose Edge Intelligence
|
||||
|
||||
> Location awareness, activity patterns, and autonomous decision-making running on the ESP32 chip. These modules figure out where people are, learn daily routines, verify safety rules, and let the device plan its own actions.
|
||||
|
||||
## Spatial Reasoning
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| PageRank Influence | `spt_pagerank_influence.rs` | Finds the dominant person in multi-person scenes using cross-correlation PageRank | 760-762 | S (<5 ms) |
|
||||
| Micro-HNSW | `spt_micro_hnsw.rs` | On-device approximate nearest-neighbor search for CSI fingerprint matching | 765-768 | S (<5 ms) |
|
||||
| Spiking Tracker | `spt_spiking_tracker.rs` | Bio-inspired person tracking using LIF neurons with STDP learning | 770-773 | M (<8 ms) |
|
||||
|
||||
---
|
||||
|
||||
### PageRank Influence (`spt_pagerank_influence.rs`)
|
||||
|
||||
**What it does**: Figures out which person in a multi-person scene has the strongest WiFi signal influence, using the same math Google uses to rank web pages. Up to 4 persons are modelled as graph nodes; edge weights come from the normalized cross-correlation of their subcarrier phase groups (8 subcarriers per person).
|
||||
|
||||
**Algorithm**: 4x4 weighted adjacency graph built from abs(dot-product) / (norm_a * norm_b) cross-correlation. Standard PageRank power iteration with damping factor 0.85, 10 iterations, column-normalized transition matrix. Ranks are normalized to sum to 1.0 after each iteration.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::spt_pagerank_influence::PageRankInfluence;
|
||||
|
||||
let mut pr = PageRankInfluence::new(); // const fn, zero-alloc
|
||||
let events = pr.process_frame(&phases, 2); // phases: &[f32], n_persons: usize
|
||||
let score = pr.rank(0); // PageRank score for person 0
|
||||
let dom = pr.dominant_person(); // index of dominant person
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 760 | `EVENT_DOMINANT_PERSON` | Person index (0-3) | Every frame |
|
||||
| 761 | `EVENT_INFLUENCE_SCORE` | PageRank score of dominant person [0, 1] | Every frame |
|
||||
| 762 | `EVENT_INFLUENCE_CHANGE` | Encoded person_id + signed delta (fractional) | When rank shifts > 0.05 |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_PERSONS` | 4 | Maximum tracked persons |
|
||||
| `SC_PER_PERSON` | 8 | Subcarriers assigned per person group |
|
||||
| `DAMPING` | 0.85 | PageRank damping factor (standard) |
|
||||
| `PR_ITERS` | 10 | Power-iteration rounds |
|
||||
| `CHANGE_THRESHOLD` | 0.05 | Minimum rank change to emit change event |
|
||||
|
||||
#### Example: Detecting the Dominant Speaker in a Room
|
||||
|
||||
When multiple people are present, the person moving the most creates the strongest CSI disturbance. PageRank identifies which person's signal "influences" the others most strongly.
|
||||
|
||||
```
|
||||
Frame 1: Person 0 speaking (active), Person 1 seated
|
||||
-> EVENT_DOMINANT_PERSON = 0, EVENT_INFLUENCE_SCORE = 0.62
|
||||
|
||||
Frame 50: Person 1 stands and walks
|
||||
-> EVENT_DOMINANT_PERSON = 1, EVENT_INFLUENCE_SCORE = 0.58
|
||||
-> EVENT_INFLUENCE_CHANGE (person 1 rank increased by 0.08)
|
||||
```
|
||||
|
||||
#### How It Works (Step by Step)
|
||||
|
||||
1. Host reports `n_persons` and provides up to 32 subcarrier phases
|
||||
2. Module groups subcarriers: person 0 gets phases[0..8], person 1 gets phases[8..16], etc.
|
||||
3. Cross-correlation is computed between every pair of person groups (abs cosine similarity)
|
||||
4. A 4x4 adjacency matrix is built (no self-loops)
|
||||
5. PageRank power iteration runs 10 times with damping=0.85
|
||||
6. The person with the highest rank is reported as the dominant person
|
||||
7. If any person's rank changed by more than 0.05 since last frame, a change event fires
|
||||
|
||||
---
|
||||
|
||||
### Micro-HNSW (`spt_micro_hnsw.rs`)
|
||||
|
||||
**What it does**: Stores up to 64 reference CSI fingerprint vectors (8 dimensions each) in a single-layer navigable small-world graph, enabling fast approximate nearest-neighbor lookup. When the sensor sees a new CSI pattern, it finds the most similar stored reference and returns its classification label.
|
||||
|
||||
**Algorithm**: HNSW (Hierarchical Navigable Small World) simplified to a single layer for embedded use. 64 nodes, 4 neighbors per node, beam search width 4, maximum 8 hops. L2 (Euclidean) distance. Bidirectional edges with worst-neighbor replacement pruning when a node is full.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::spt_micro_hnsw::MicroHnsw;
|
||||
|
||||
let mut hnsw = MicroHnsw::new(); // const fn, zero-alloc
|
||||
let idx = hnsw.insert(&features_8d, label); // Option<usize>
|
||||
let (nearest_id, distance) = hnsw.search(&query_8d); // (usize, f32)
|
||||
let events = hnsw.process_frame(&features); // per-frame query
|
||||
let label = hnsw.last_label(); // u8 or 255=unknown
|
||||
let dist = hnsw.last_match_distance(); // f32
|
||||
let n = hnsw.size(); // number of stored vectors
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 765 | `EVENT_NEAREST_MATCH_ID` | Index of nearest stored vector | Every frame |
|
||||
| 766 | `EVENT_MATCH_DISTANCE` | L2 distance to nearest match | Every frame |
|
||||
| 767 | `EVENT_CLASSIFICATION` | Label of nearest match (255 if too far) | Every frame |
|
||||
| 768 | `EVENT_LIBRARY_SIZE` | Number of stored reference vectors | Every frame |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `MAX_VECTORS` | 64 | Maximum stored reference fingerprints |
|
||||
| `DIM` | 8 | Dimensions per feature vector |
|
||||
| `MAX_NEIGHBORS` | 4 | Edges per node in the graph |
|
||||
| `BEAM_WIDTH` | 4 | Search beam width (quality vs speed) |
|
||||
| `MAX_HOPS` | 8 | Maximum graph traversal depth |
|
||||
| `MATCH_THRESHOLD` | 2.0 | Distance above which classification returns "unknown" |
|
||||
|
||||
#### Example: Room Location Fingerprinting
|
||||
|
||||
Pre-load reference CSI fingerprints for known locations, then classify new readings in real-time.
|
||||
|
||||
```
|
||||
Setup:
|
||||
hnsw.insert(&kitchen_fingerprint, 1); // label 1 = kitchen
|
||||
hnsw.insert(&bedroom_fingerprint, 2); // label 2 = bedroom
|
||||
hnsw.insert(&bathroom_fingerprint, 3); // label 3 = bathroom
|
||||
|
||||
Runtime:
|
||||
Frame arrives with features = [0.32, 0.15, ...]
|
||||
-> EVENT_NEAREST_MATCH_ID = 1 (kitchen reference)
|
||||
-> EVENT_MATCH_DISTANCE = 0.45
|
||||
-> EVENT_CLASSIFICATION = 1 (kitchen)
|
||||
-> EVENT_LIBRARY_SIZE = 3
|
||||
```
|
||||
|
||||
#### How It Works (Step by Step)
|
||||
|
||||
1. **Insert**: New vector is added at position `n_vectors`. The module scans all existing nodes (N<=64, so linear scan is fine) to find the 4 nearest neighbors. Bidirectional edges are added; if a node already has 4 neighbors, the worst (farthest) is replaced if the new connection is shorter.
|
||||
2. **Search**: Starting from the entry point, a beam search (width 4) explores neighbor nodes for up to 8 hops. Each hop expands unvisited neighbors of the current beam and inserts closer ones. Search terminates when no hop improves the beam.
|
||||
3. **Classify**: If the nearest match distance is below `MATCH_THRESHOLD` (2.0), its label is returned. Otherwise, 255 (unknown).
|
||||
|
||||
---
|
||||
|
||||
### Spiking Tracker (`spt_spiking_tracker.rs`)
|
||||
|
||||
**What it does**: Tracks a person's location across 4 spatial zones using a biologically inspired spiking neural network. 32 Leaky Integrate-and-Fire (LIF) neurons (one per subcarrier) feed into 4 output neurons (one per zone). The zone with the highest spike rate indicates the person's location. Zone transitions measure velocity.
|
||||
|
||||
**Algorithm**: LIF neuron model with membrane leak factor 0.95, threshold 1.0, reset to 0.0. STDP (Spike-Timing-Dependent Plasticity) learning: potentiation LR=0.01 when pre+post fire within 1 frame, depression LR=0.005 when only pre fires. Weights clamped to [0, 2]. EMA smoothing on zone spike rates (alpha=0.1).
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::spt_spiking_tracker::SpikingTracker;
|
||||
|
||||
let mut st = SpikingTracker::new(); // const fn
|
||||
let events = st.process_frame(&phases, &prev_phases); // returns events
|
||||
let zone = st.current_zone(); // i8, -1 if lost
|
||||
let rate = st.zone_spike_rate(0); // f32 for zone 0
|
||||
let vel = st.velocity(); // EMA velocity
|
||||
let tracking = st.is_tracking(); // bool
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 770 | `EVENT_TRACK_UPDATE` | Zone ID (0-3) | When tracked |
|
||||
| 771 | `EVENT_TRACK_VELOCITY` | Zone transitions/frame (EMA) | When tracked |
|
||||
| 772 | `EVENT_SPIKE_RATE` | Mean spike rate across zones [0, 1] | Every frame |
|
||||
| 773 | `EVENT_TRACK_LOST` | Last known zone ID | When track lost |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `N_INPUT` | 32 | Input neurons (one per subcarrier) |
|
||||
| `N_OUTPUT` | 4 | Output neurons (one per zone) |
|
||||
| `THRESHOLD` | 1.0 | LIF firing threshold |
|
||||
| `LEAK` | 0.95 | Membrane decay per frame |
|
||||
| `STDP_LR_PLUS` | 0.01 | Potentiation learning rate |
|
||||
| `STDP_LR_MINUS` | 0.005 | Depression learning rate |
|
||||
| `W_MIN` / `W_MAX` | 0.0 / 2.0 | Weight bounds |
|
||||
| `MIN_SPIKE_RATE` | 0.05 | Minimum rate to consider zone active |
|
||||
|
||||
#### Example: Tracking Movement Between Zones
|
||||
|
||||
```
|
||||
Frames 1-30: Strong phase changes in subcarriers 0-7 (zone 0)
|
||||
-> EVENT_TRACK_UPDATE = 0, EVENT_SPIKE_RATE = 0.15
|
||||
|
||||
Frames 31-60: Activity shifts to subcarriers 16-23 (zone 2)
|
||||
-> EVENT_TRACK_UPDATE = 2, EVENT_TRACK_VELOCITY = 0.033
|
||||
STDP strengthens zone 2 connections, weakens zone 0
|
||||
|
||||
Frames 61-90: No activity
|
||||
-> Spike rates decay via EMA
|
||||
-> EVENT_TRACK_LOST = 2 (last known zone)
|
||||
```
|
||||
|
||||
#### How It Works (Step by Step)
|
||||
|
||||
1. Phase deltas (|current - previous|) inject current into LIF neurons
|
||||
2. Each neuron leaks (membrane *= 0.95), then adds current
|
||||
3. If membrane >= threshold (1.0), the neuron fires and resets to 0
|
||||
4. Input spikes propagate to output zones via weighted connections
|
||||
5. Output neurons fire when cumulative input exceeds threshold
|
||||
6. STDP adjusts weights: correlated pre+post firing strengthens connections, uncorrelated pre firing weakens them (sparse iteration skips silent neurons for 70-90% savings)
|
||||
7. Zone spike rates are EMA-smoothed; the zone with the highest rate above `MIN_SPIKE_RATE` is reported as the tracked location
|
||||
|
||||
---
|
||||
|
||||
## Temporal Analysis
|
||||
|
||||
| Module | File | What It Does | Event IDs | Budget |
|
||||
|--------|------|--------------|-----------|--------|
|
||||
| Pattern Sequence | `tmp_pattern_sequence.rs` | Learns daily activity routines and detects deviations | 790-793 | S (<5 ms) |
|
||||
| Temporal Logic Guard | `tmp_temporal_logic_guard.rs` | Verifies 8 LTL safety invariants on every frame | 795-797 | S (<5 ms) |
|
||||
| GOAP Autonomy | `tmp_goap_autonomy.rs` | Autonomous module management via A* goal-oriented planning | 800-803 | S (<5 ms) |
|
||||
|
||||
---
|
||||
|
||||
### Pattern Sequence (`tmp_pattern_sequence.rs`)
|
||||
|
||||
**What it does**: Learns daily activity routines and alerts when something changes. Each minute is discretized into a motion symbol (Empty, Still, LowMotion, HighMotion, MultiPerson), stored in a 24-hour circular buffer (1440 entries). An hourly LCS (Longest Common Subsequence) comparison between today and yesterday yields a routine confidence score. If grandma usually goes to the kitchen by 8am but has not moved, it notices.
|
||||
|
||||
**Algorithm**: Two-row dynamic programming LCS with O(n) memory (60-entry comparison window). Majority-vote symbol selection from per-frame accumulation. Two-day history buffer with day rollover.
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::tmp_pattern_sequence::PatternSequenceAnalyzer;
|
||||
|
||||
let mut psa = PatternSequenceAnalyzer::new(); // const fn
|
||||
psa.on_frame(presence, motion, n_persons); // called per CSI frame (~20 Hz)
|
||||
let events = psa.on_timer(); // called at ~1 Hz
|
||||
let conf = psa.routine_confidence(); // [0, 1]
|
||||
let n = psa.pattern_count(); // stored patterns
|
||||
let min = psa.current_minute(); // 0-1439
|
||||
let day = psa.day_offset(); // days since start
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 790 | `EVENT_PATTERN_DETECTED` | LCS length of detected pattern | Hourly |
|
||||
| 791 | `EVENT_PATTERN_CONFIDENCE` | Routine confidence [0, 1] | Hourly |
|
||||
| 792 | `EVENT_ROUTINE_DEVIATION` | Minute index where deviation occurred | Per minute (when deviating) |
|
||||
| 793 | `EVENT_PREDICTION_NEXT` | Predicted next-minute symbol (from yesterday) | Per minute |
|
||||
|
||||
#### Configuration Constants
|
||||
|
||||
| Constant | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `DAY_LEN` | 1440 | Minutes per day |
|
||||
| `MAX_PATTERNS` | 32 | Maximum stored pattern templates |
|
||||
| `PATTERN_LEN` | 16 | Maximum symbols per pattern |
|
||||
| `LCS_WINDOW` | 60 | Comparison window (1 hour) |
|
||||
| `THRESH_STILL` / `THRESH_LOW` / `THRESH_HIGH` | 0.05 / 0.3 / 0.7 | Motion discretization thresholds |
|
||||
|
||||
#### Symbols
|
||||
|
||||
| Symbol | Value | Condition |
|
||||
|--------|-------|-----------|
|
||||
| Empty | 0 | No presence |
|
||||
| Still | 1 | Present, motion < 0.05 |
|
||||
| LowMotion | 2 | Present, 0.3 < motion <= 0.7 |
|
||||
| HighMotion | 3 | Present, motion > 0.7 |
|
||||
| MultiPerson | 4 | More than 1 person present |
|
||||
|
||||
#### Example: Elderly Care Routine Monitoring
|
||||
|
||||
```
|
||||
Day 1: Learning phase
|
||||
07:00 - Still (person in bed)
|
||||
07:30 - HighMotion (getting ready)
|
||||
08:00 - LowMotion (breakfast)
|
||||
-> Patterns stored in history buffer
|
||||
|
||||
Day 2: Comparison active
|
||||
07:00 - Still (normal)
|
||||
07:30 - Still (DEVIATION! Expected HighMotion)
|
||||
-> EVENT_ROUTINE_DEVIATION = 450 (minute 7:30)
|
||||
-> EVENT_PREDICTION_NEXT = 3 (HighMotion expected)
|
||||
08:30 - Still (still no activity)
|
||||
-> Caregiver notified via DEVIATION events
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Temporal Logic Guard (`tmp_temporal_logic_guard.rs`)
|
||||
|
||||
**What it does**: Encodes 8 safety rules as Linear Temporal Logic (LTL) state machines. G-rules ("globally") are violated on any single frame. F-rules ("eventually") have deadlines. Every frame, the guard checks all rules and emits violations with counterexample frame indices.
|
||||
|
||||
**Algorithm**: State machine per rule (Satisfied/Pending/Violated). G-rules use immediate boolean checks. F-rules use deadline counters (frame-based). Counterexample tracking records the frame index when violation first occurs.
|
||||
|
||||
#### The 8 Safety Rules
|
||||
|
||||
| Rule | Type | Description | Violation Condition |
|
||||
|------|------|-------------|---------------------|
|
||||
| R0 | G | No fall alert when room is empty | `presence==0 AND fall_alert` |
|
||||
| R1 | G | No intrusion alert when nobody present | `intrusion_alert AND presence==0` |
|
||||
| R2 | G | No person ID active when nobody detected | `n_persons==0 AND person_id_active` |
|
||||
| R3 | G | No vital signs when coherence is too low | `coherence<0.3 AND vital_signs_active` |
|
||||
| R4 | F | Continuous motion must stop within 300s | Motion > 0.1 for 6000 consecutive frames |
|
||||
| R5 | F | Fast breathing must trigger alert within 5s | Breathing > 40 BPM for 100 consecutive frames |
|
||||
| R6 | G | Heart rate must not exceed 150 BPM | `heartrate_bpm > 150` |
|
||||
| R7 | G-F | After seizure, no normal gait within 60s | Normal gait reported < 1200 frames after seizure |
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::tmp_temporal_logic_guard::{TemporalLogicGuard, FrameInput};
|
||||
|
||||
let mut guard = TemporalLogicGuard::new(); // const fn
|
||||
let events = guard.on_frame(&input); // per-frame check
|
||||
let satisfied = guard.satisfied_count(); // how many rules OK
|
||||
let state = guard.rule_state(4); // Satisfied/Pending/Violated
|
||||
let vio = guard.violation_count(0); // total violations for rule 0
|
||||
let frame = guard.last_violation_frame(3); // frame index of last violation
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 795 | `EVENT_LTL_VIOLATION` | Rule index (0-7) | On violation |
|
||||
| 796 | `EVENT_LTL_SATISFACTION` | Count of currently satisfied rules | Every 200 frames |
|
||||
| 797 | `EVENT_COUNTEREXAMPLE` | Frame index when violation occurred | Paired with violation |
|
||||
|
||||
---
|
||||
|
||||
### GOAP Autonomy (`tmp_goap_autonomy.rs`)
|
||||
|
||||
**What it does**: Lets the ESP32 autonomously decide which sensing modules to activate or deactivate based on the current situation. Uses Goal-Oriented Action Planning (GOAP) with A* search over an 8-bit boolean world state to find the cheapest action sequence that achieves the highest-priority unsatisfied goal.
|
||||
|
||||
**Algorithm**: A* search over 8-bit world state. 6 prioritized goals, 8 actions with preconditions and effects encoded as bitmasks. Maximum plan depth 4, open set capacity 32. Replans every 60 seconds.
|
||||
|
||||
#### World State Properties
|
||||
|
||||
| Bit | Property | Meaning |
|
||||
|-----|----------|---------|
|
||||
| 0 | `has_presence` | Room occupancy detected |
|
||||
| 1 | `has_motion` | Motion energy above threshold |
|
||||
| 2 | `is_night` | Nighttime period |
|
||||
| 3 | `multi_person` | More than 1 person present |
|
||||
| 4 | `low_coherence` | Signal quality is degraded |
|
||||
| 5 | `high_threat` | Threat score above threshold |
|
||||
| 6 | `has_vitals` | Vital sign monitoring active |
|
||||
| 7 | `is_learning` | Pattern learning active |
|
||||
|
||||
#### Goals (Priority Order)
|
||||
|
||||
| # | Goal | Priority | Condition |
|
||||
|---|------|----------|-----------|
|
||||
| 0 | Monitor Health | 0.9 | Achieve `has_vitals = true` |
|
||||
| 1 | Secure Space | 0.8 | Achieve `has_presence = true` |
|
||||
| 2 | Count People | 0.7 | Achieve `multi_person = false` |
|
||||
| 3 | Learn Patterns | 0.5 | Achieve `is_learning = true` |
|
||||
| 4 | Save Energy | 0.3 | Achieve `is_learning = false` |
|
||||
| 5 | Self Test | 0.1 | Achieve `low_coherence = false` |
|
||||
|
||||
#### Actions
|
||||
|
||||
| # | Action | Precondition | Effect | Cost |
|
||||
|---|--------|-------------|--------|------|
|
||||
| 0 | Activate Vitals | Presence required | Sets `has_vitals` | 2 |
|
||||
| 1 | Activate Intrusion | None | Sets `has_presence` | 1 |
|
||||
| 2 | Activate Occupancy | Presence required | Clears `multi_person` | 2 |
|
||||
| 3 | Activate Gesture Learn | Low coherence must be false | Sets `is_learning` | 3 |
|
||||
| 4 | Deactivate Heavy | None | Clears `is_learning` + `has_vitals` | 1 |
|
||||
| 5 | Run Coherence Check | None | Clears `low_coherence` | 2 |
|
||||
| 6 | Enter Low Power | None | Clears `is_learning` + `has_motion` | 1 |
|
||||
| 7 | Run Self Test | None | Clears `low_coherence` + `high_threat` | 3 |
|
||||
|
||||
#### Public API
|
||||
|
||||
```rust
|
||||
use wifi_densepose_wasm_edge::tmp_goap_autonomy::GoapPlanner;
|
||||
|
||||
let mut planner = GoapPlanner::new(); // const fn
|
||||
planner.update_world(presence, motion, n_persons,
|
||||
coherence, threat, has_vitals, is_night);
|
||||
let events = planner.on_timer(); // called at ~1 Hz
|
||||
let ws = planner.world_state(); // u8 bitmask
|
||||
let goal = planner.current_goal(); // goal index or 0xFF
|
||||
let len = planner.plan_len(); // steps in current plan
|
||||
planner.set_goal_priority(0, 0.95); // dynamically adjust
|
||||
```
|
||||
|
||||
#### Events
|
||||
|
||||
| Event ID | Constant | Value | Frequency |
|
||||
|----------|----------|-------|-----------|
|
||||
| 800 | `EVENT_GOAL_SELECTED` | Goal index (0-5) | On replan |
|
||||
| 801 | `EVENT_MODULE_ACTIVATED` | Action index that activated a module | On plan step |
|
||||
| 802 | `EVENT_MODULE_DEACTIVATED` | Action index that deactivated a module | On plan step |
|
||||
| 803 | `EVENT_PLAN_COST` | Total cost of the planned action sequence | On replan |
|
||||
|
||||
#### Example: Autonomous Night-Mode Transition
|
||||
|
||||
```
|
||||
18:00 - World state: presence=1, motion=0, night=0, vitals=1
|
||||
Goal 0 (Monitor Health) satisfied, Goal 1 (Secure Space) satisfied
|
||||
-> Goal 2 selected (Count People, prio 0.7)
|
||||
|
||||
22:00 - World state: presence=0, motion=0, night=1
|
||||
-> Goal 1 selected (Secure Space, prio 0.8)
|
||||
-> Plan: [Action 1: Activate Intrusion] (cost=1)
|
||||
-> EVENT_GOAL_SELECTED = 1
|
||||
-> EVENT_MODULE_ACTIVATED = 1 (intrusion detection)
|
||||
-> EVENT_PLAN_COST = 1
|
||||
|
||||
03:00 - No presence, low coherence detected
|
||||
-> Goal 5 selected (Self Test, prio 0.1)
|
||||
-> Plan: [Action 5: Run Coherence Check] (cost=2)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Memory Layout Summary
|
||||
|
||||
All modules use fixed-size arrays and static event buffers. No heap allocation.
|
||||
|
||||
| Module | State Size (approx) | Static Event Buffer |
|
||||
|--------|---------------------|---------------------|
|
||||
| PageRank Influence | ~192 bytes (4x4 adj + 2x4 rank + meta) | 8 entries |
|
||||
| Micro-HNSW | ~3.5 KB (64 nodes x 48 bytes + meta) | 4 entries |
|
||||
| Spiking Tracker | ~1.1 KB (32x4 weights + membranes + rates) | 4 entries |
|
||||
| Pattern Sequence | ~3.2 KB (2x1440 history + 32 patterns + LCS rows) | 4 entries |
|
||||
| Temporal Logic Guard | ~120 bytes (8 rules + counters) | 12 entries |
|
||||
| GOAP Autonomy | ~1.6 KB (32 open-set nodes + goals + plan) | 4 entries |
|
||||
|
||||
## Integration with Host Firmware
|
||||
|
||||
These modules receive data from the ESP32 Tier 2 DSP pipeline via the WASM3 host API:
|
||||
|
||||
```
|
||||
ESP32 Firmware (C) WASM3 Runtime WASM Module (Rust)
|
||||
| | |
|
||||
CSI frame arrives | |
|
||||
Tier 2 DSP runs | |
|
||||
|--- csi_get_phase() ---->|--- host_get_phase() --->|
|
||||
|--- csi_get_presence() ->|--- host_get_presence()->|
|
||||
| | process_frame() |
|
||||
|<-- csi_emit_event() ----|<-- host_emit_event() ---|
|
||||
| | |
|
||||
Forward to aggregator | |
|
||||
```
|
||||
|
||||
Modules can be hot-loaded via OTA (ADR-040) without reflashing the firmware.
|
||||
@@ -0,0 +1,336 @@
|
||||
---
|
||||
license: mit
|
||||
tags:
|
||||
- wifi-sensing
|
||||
- pose-estimation
|
||||
- vital-signs
|
||||
- edge-ai
|
||||
- esp32
|
||||
- onnx
|
||||
- self-supervised
|
||||
- cognitum
|
||||
- csi
|
||||
- through-wall
|
||||
- privacy-preserving
|
||||
language:
|
||||
- en
|
||||
library_name: onnxruntime
|
||||
pipeline_tag: other
|
||||
---
|
||||
|
||||
# WiFi-DensePose: See Through Walls with WiFi + AI
|
||||
|
||||
**Detect people, track movement, and measure breathing -- through walls, without cameras, using a $27 sensor kit.**
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **License** | MIT |
|
||||
| **Framework** | ONNX Runtime |
|
||||
| **Hardware** | ESP32-S3 ($9) + optional Cognitum Seed ($15) |
|
||||
| **Training** | Self-supervised contrastive learning (no labels needed) |
|
||||
| **Privacy** | No cameras, no images, no personally identifiable data |
|
||||
|
||||
---
|
||||
|
||||
## What is this?
|
||||
|
||||
This model turns ordinary WiFi signals into a human sensing system. It can detect whether someone is in a room, count how many people are present, classify what they are doing, and even measure their breathing rate -- all without any cameras.
|
||||
|
||||
**How does it work?** Every WiFi router constantly sends signals that bounce off walls, furniture, and people. When a person moves -- or even just breathes -- those bouncing signals change in tiny but measurable ways. WiFi chips can capture these changes as numbers called *Channel State Information* (CSI). Think of it like ripples in a pond: drop a stone and the ripples tell you something happened, even if you cannot see the stone.
|
||||
|
||||
This model learned to read those "WiFi ripples" and figure out what is happening in the room. It was trained using a technique called *contrastive learning*, which means it taught itself by comparing thousands of WiFi signal snapshots -- no human had to manually label anything.
|
||||
|
||||
The result is a small, fast model that runs on a $9 microcontroller and preserves complete privacy because it never captures images or audio.
|
||||
|
||||
---
|
||||
|
||||
## What can it do?
|
||||
|
||||
| Capability | Accuracy | What you need | Notes |
|
||||
|---|---|---|---|
|
||||
| **Presence detection** | >95% | 1x ESP32-S3 ($9) | Is anyone in the room? |
|
||||
| **Motion classification** | >90% | 1x ESP32-S3 ($9) | Still, walking, exercising, fallen |
|
||||
| **Breathing rate** | +/- 2 BPM | 1x ESP32-S3 ($9) | Best when person is sitting or lying still |
|
||||
| **Heart rate estimate** | +/- 5 BPM | 1x ESP32-S3 ($9) | Experimental -- less accurate during movement |
|
||||
| **Person counting** | 1-4 people | 2x ESP32-S3 ($18) | Uses cross-node signal fusion |
|
||||
| **Pose estimation** | 17 COCO keypoints | 2x ESP32-S3 + Seed ($27) | Full skeleton: head, shoulders, elbows, etc. |
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Install
|
||||
|
||||
```bash
|
||||
pip install onnxruntime numpy
|
||||
```
|
||||
|
||||
### Run inference
|
||||
|
||||
```python
|
||||
import onnxruntime as ort
|
||||
import numpy as np
|
||||
|
||||
# Load the encoder model
|
||||
session = ort.InferenceSession("pretrained-encoder.onnx")
|
||||
|
||||
# Simulated 8-dim CSI feature vector from ESP32-S3
|
||||
# Dimensions: [amplitude_mean, amplitude_std, phase_slope, doppler_energy,
|
||||
# subcarrier_variance, temporal_stability, csi_ratio, spectral_entropy]
|
||||
features = np.array(
|
||||
[[0.45, 0.30, 0.69, 0.75, 0.50, 0.25, 0.00, 0.54]],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
# Encode into 128-dim embedding
|
||||
result = session.run(None, {"input": features})
|
||||
embedding = result[0] # shape: (1, 128)
|
||||
print(f"Embedding shape: {embedding.shape}")
|
||||
print(f"First 8 values: {embedding[0][:8]}")
|
||||
```
|
||||
|
||||
### Run task heads
|
||||
|
||||
```python
|
||||
# Load the task heads model
|
||||
heads = ort.InferenceSession("pretrained-heads.onnx")
|
||||
|
||||
# Feed the embedding from the encoder
|
||||
predictions = heads.run(None, {"embedding": embedding})
|
||||
|
||||
presence_score = predictions[0] # 0.0 = empty, 1.0 = occupied
|
||||
person_count = predictions[1] # estimated count (float, round to int)
|
||||
activity_class = predictions[2] # [still, walking, exercise, fallen]
|
||||
vitals = predictions[3] # [breathing_bpm, heart_bpm]
|
||||
|
||||
print(f"Presence: {presence_score[0]:.2f}")
|
||||
print(f"People: {int(round(person_count[0]))}")
|
||||
print(f"Activity: {['still', 'walking', 'exercise', 'fallen'][activity_class.argmax()]}")
|
||||
print(f"Breathing: {vitals[0][0]:.1f} BPM")
|
||||
print(f"Heart: {vitals[0][1]:.1f} BPM")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Model Architecture
|
||||
|
||||
```
|
||||
+-- Presence (binary)
|
||||
|
|
||||
WiFi signals --> ESP32-S3 --> 8-dim features --> Encoder (TCN) --> 128-dim embedding --> Task Heads --+-- Person Count
|
||||
(CSI) (on-device) (~2.5M params) (~100K) |
|
||||
+-- Activity (4 classes)
|
||||
|
|
||||
+-- Vitals (BR + HR)
|
||||
```
|
||||
|
||||
### Encoder
|
||||
|
||||
- **Type:** Temporal Convolutional Network (TCN)
|
||||
- **Input:** 8-dimensional feature vector extracted from raw CSI
|
||||
- **Output:** 128-dimensional embedding
|
||||
- **Parameters:** ~2.5M
|
||||
- **Format:** ONNX (runs on any platform with ONNX Runtime)
|
||||
|
||||
### Task Heads
|
||||
|
||||
- **Type:** Small MLPs (multi-layer perceptrons), one per task
|
||||
- **Input:** 128-dim embedding from the encoder
|
||||
- **Output:** Task-specific predictions (presence, count, activity, vitals)
|
||||
- **Parameters:** ~100K total across all heads
|
||||
- **Format:** ONNX
|
||||
|
||||
### Feature extraction (runs on ESP32-S3)
|
||||
|
||||
The ESP32-S3 captures raw CSI frames at ~100 Hz and computes 8 summary features per window:
|
||||
|
||||
| Feature | Description |
|
||||
|---|---|
|
||||
| `amplitude_mean` | Average signal strength across subcarriers |
|
||||
| `amplitude_std` | Variation in signal strength (movement indicator) |
|
||||
| `phase_slope` | Rate of phase change across subcarriers |
|
||||
| `doppler_energy` | Energy in the Doppler spectrum (velocity indicator) |
|
||||
| `subcarrier_variance` | How much individual subcarriers differ |
|
||||
| `temporal_stability` | Consistency of signal over time (stillness indicator) |
|
||||
| `csi_ratio` | Ratio between antenna pairs (direction indicator) |
|
||||
| `spectral_entropy` | Randomness of the frequency spectrum |
|
||||
|
||||
---
|
||||
|
||||
## Training Data
|
||||
|
||||
### How it was trained
|
||||
|
||||
This model was trained using **self-supervised contrastive learning**, which means it learned entirely from unlabeled WiFi signals. No cameras, no manual annotations, and no privacy-invasive data collection were needed.
|
||||
|
||||
The training process works like this:
|
||||
|
||||
1. **Collect** raw CSI frames from ESP32-S3 nodes placed in a room
|
||||
2. **Extract** 8-dimensional feature vectors from sliding windows of CSI data
|
||||
3. **Contrast** -- the model learns that features from nearby time windows should produce similar embeddings, while features from different scenarios should produce different embeddings
|
||||
4. **Fine-tune** task heads using weak labels from environmental sensors (PIR motion, temperature, pressure) on the Cognitum Seed companion device
|
||||
|
||||
### Data provenance
|
||||
|
||||
- **Source:** Live CSI from 2x ESP32-S3 nodes (802.11n, HT40, 114 subcarriers)
|
||||
- **Volume:** ~360,000 CSI frames (~3,600 feature vectors) per collection run
|
||||
- **Environment:** Residential room, ~4x5 meters
|
||||
- **Ground truth:** Environmental sensors on Cognitum Seed (PIR, BME280, light)
|
||||
- **Attestation:** Every collection run produces a cryptographic witness chain (`collection-witness.json`) that proves data provenance and integrity
|
||||
|
||||
### Witness chain
|
||||
|
||||
The `collection-witness.json` file contains a chain of SHA-256 hashes linking every step from raw CSI capture through feature extraction to model training. This allows anyone to verify that the published model was trained on data collected by specific hardware at a specific time.
|
||||
|
||||
---
|
||||
|
||||
## Hardware Requirements
|
||||
|
||||
### Minimum: single-node sensing ($9)
|
||||
|
||||
| Component | What it does | Cost | Where to get it |
|
||||
|---|---|---|---|
|
||||
| ESP32-S3 (8MB flash) | Captures WiFi CSI + runs feature extraction | ~$9 | Amazon, AliExpress, Adafruit |
|
||||
| USB-C cable | Power + data | ~$3 | Any electronics store |
|
||||
|
||||
This gets you: presence detection, motion classification, breathing rate.
|
||||
|
||||
### Recommended: dual-node sensing ($18)
|
||||
|
||||
Add a second ESP32-S3 to enable cross-node signal fusion for better accuracy and person counting.
|
||||
|
||||
### Full setup: sensing + ground truth ($27)
|
||||
|
||||
| Component | What it does | Cost |
|
||||
|---|---|---|
|
||||
| 2x ESP32-S3 (8MB) | WiFi CSI sensing nodes | ~$18 |
|
||||
| Cognitum Seed (Pi Zero 2W) | Runs inference + collects ground truth | ~$15 |
|
||||
| USB-C cables (x3) | Power + data | ~$9 |
|
||||
| **Total** | | **~$27** |
|
||||
|
||||
The Cognitum Seed runs the ONNX models on-device, orchestrates the ESP32 nodes over USB serial, and provides environmental ground truth via its onboard PIR and BME280 sensors.
|
||||
|
||||
---
|
||||
|
||||
## Files in this repo
|
||||
|
||||
| File | Size | Description |
|
||||
|---|---|---|
|
||||
| `pretrained-encoder.onnx` | ~2 MB | Contrastive encoder (TCN backbone, 8-dim input, 128-dim output) |
|
||||
| `pretrained-heads.onnx` | ~100 KB | Task heads (presence, count, activity, vitals) |
|
||||
| `pretrained.rvf` | ~500 KB | RuVector format embeddings for advanced fusion pipelines |
|
||||
| `room-profiles.json` | ~10 KB | Environment calibration profiles (room geometry, baseline noise) |
|
||||
| `collection-witness.json` | ~5 KB | Cryptographic witness chain proving data provenance |
|
||||
| `config.json` | ~2 KB | Training configuration (hyperparameters, feature schema, versions) |
|
||||
| `README.md` | -- | This file |
|
||||
|
||||
### RuVector format (.rvf)
|
||||
|
||||
The `.rvf` file contains pre-computed embeddings in RuVector format, used by the RuView application for advanced multi-node fusion and cross-viewpoint pose estimation. You only need this if you are using the full RuView pipeline. For basic inference, the ONNX files are sufficient.
|
||||
|
||||
---
|
||||
|
||||
## How to use with RuView
|
||||
|
||||
[RuView](https://github.com/ruvnet/RuView) is the open-source application that ties everything together: firmware flashing, real-time sensing, and a browser-based dashboard.
|
||||
|
||||
### 1. Flash firmware to ESP32-S3
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ruvnet/RuView.git
|
||||
cd RuView
|
||||
|
||||
# Flash firmware (requires ESP-IDF v5.4 or use pre-built binaries from Releases)
|
||||
# See the repo README for platform-specific instructions
|
||||
```
|
||||
|
||||
### 2. Download models
|
||||
|
||||
```bash
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained --local-dir models/
|
||||
```
|
||||
|
||||
### 3. Run inference
|
||||
|
||||
```bash
|
||||
# Start the CSI bridge (connects ESP32 serial output to the inference pipeline)
|
||||
python scripts/seed_csi_bridge.py --port COM7 --model models/pretrained-encoder.onnx
|
||||
|
||||
# Or run the full sensing server with web dashboard
|
||||
cargo run -p wifi-densepose-sensing-server
|
||||
```
|
||||
|
||||
### 4. Adapt to your room
|
||||
|
||||
The model works best after a brief calibration period (~60 seconds of no movement) to learn the baseline signal characteristics of your specific room. The `room-profiles.json` file contains example profiles; the system will create one for your environment automatically.
|
||||
|
||||
---
|
||||
|
||||
## Limitations
|
||||
|
||||
Be honest about what this technology can and cannot do:
|
||||
|
||||
- **Room-specific.** The model needs a short calibration period in each new environment. A model calibrated in a living room will not work as well in a warehouse without re-adaptation.
|
||||
- **Single room only.** There is no cross-room tracking. Each room needs its own sensing node(s).
|
||||
- **Person count accuracy degrades above 4.** Counting works well for 1-3 people, becomes unreliable above 4 in a single room.
|
||||
- **Vitals require stillness.** Breathing and heart rate estimation work best when the person is sitting or lying down. Accuracy drops significantly during walking or exercise.
|
||||
- **Heart rate is experimental.** The +/- 5 BPM accuracy is a best-case figure. In practice, cardiac sensing via WiFi is still a research-stage capability.
|
||||
- **Wall materials matter.** Metal walls, concrete reinforced with rebar, or foil-backed insulation will significantly attenuate the signal and reduce range.
|
||||
- **WiFi interference.** Heavy WiFi traffic from other devices can add noise. The system works best on a dedicated or lightly-used WiFi channel.
|
||||
- **Not a medical device.** Vital sign estimates are for informational and research purposes only. Do not use them for medical decisions.
|
||||
|
||||
---
|
||||
|
||||
## Use Cases
|
||||
|
||||
- **Elder care:** Non-invasive fall detection and activity monitoring without cameras
|
||||
- **Smart home:** Presence-based lighting and HVAC control
|
||||
- **Security:** Occupancy detection through walls
|
||||
- **Sleep monitoring:** Breathing rate tracking overnight
|
||||
- **Research:** Low-cost human sensing for academic experiments
|
||||
- **Disaster response:** The MAT (Mass Casualty Assessment Tool) uses this model to detect survivors through rubble via WiFi signal reflections
|
||||
|
||||
---
|
||||
|
||||
## Ethical Considerations
|
||||
|
||||
WiFi sensing is a privacy-preserving alternative to cameras, but it still detects human presence and activity. Consider these points:
|
||||
|
||||
- **Consent:** Always inform people that WiFi sensing is active in a space.
|
||||
- **No biometric identification:** This model cannot identify *who* someone is -- only that someone is present and what they are doing.
|
||||
- **Data minimization:** Raw CSI data is processed on-device and only summary features or embeddings leave the sensor. No images, audio, or video are ever captured.
|
||||
- **Dual use:** Like any sensing technology, this can be misused for surveillance. We encourage transparent deployment and clear signage.
|
||||
|
||||
---
|
||||
|
||||
## Citation
|
||||
|
||||
If you use this model in your research, please cite:
|
||||
|
||||
```bibtex
|
||||
@software{wifi_densepose_2026,
|
||||
title = {WiFi-DensePose: Human Pose Estimation from WiFi Channel State Information},
|
||||
author = {ruvnet},
|
||||
year = {2026},
|
||||
url = {https://github.com/ruvnet/RuView},
|
||||
license = {MIT},
|
||||
note = {Self-supervised contrastive learning on ESP32-S3 CSI data}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT License. See [LICENSE](https://github.com/ruvnet/RuView/blob/main/LICENSE) for details.
|
||||
|
||||
You are free to use, modify, and distribute this model for any purpose, including commercial applications.
|
||||
|
||||
---
|
||||
|
||||
## Links
|
||||
|
||||
- **GitHub:** [github.com/ruvnet/RuView](https://github.com/ruvnet/RuView)
|
||||
- **Hardware:** [ESP32-S3 DevKit](https://www.espressif.com/en/products/devkits) | [Cognitum Seed](https://cognitum.one)
|
||||
- **ONNX Runtime:** [onnxruntime.ai](https://onnxruntime.ai)
|
||||
@@ -0,0 +1,996 @@
|
||||
# GOAP Implementation Plan: ESP32-S3 + Pi Zero 2 W WiFi Pose Estimation
|
||||
|
||||
**Date:** 2026-04-02
|
||||
**Version:** 1.0
|
||||
**Status:** Proposed
|
||||
**Depends on:** ADR-029, ADR-068, SOTA survey (sota-wifi-sensing-2025.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal State Definition
|
||||
|
||||
### 1.1 Terminal Goal
|
||||
|
||||
A production-ready WiFi-based human pose estimation system where:
|
||||
- **ESP32-S3** nodes capture WiFi CSI at 100 Hz, perform temporal feature extraction, and transmit compressed features via UDP
|
||||
- **Raspberry Pi Zero 2 W** receives features from 1-4 ESP32 nodes, runs neural inference, and outputs 17-keypoint COCO poses at >= 10 Hz
|
||||
- **Single-person MPJPE** < 100mm in trained environments
|
||||
- **End-to-end latency** < 150ms (CSI capture to pose output)
|
||||
- **Total BOM cost** < $30 per sensing zone (1x Pi Zero + 2x ESP32)
|
||||
|
||||
### 1.2 World State Variables
|
||||
|
||||
```
|
||||
current_state:
|
||||
esp32_csi_capture: true # Already implemented
|
||||
multi_node_aggregation: true # ADR-018 UDP aggregator
|
||||
phase_alignment: true # ruvsense/phase_align.rs
|
||||
coherence_gating: true # ruvsense/coherence_gate.rs
|
||||
multistatic_fusion: true # ruvsense/multistatic.rs
|
||||
kalman_pose_tracking: true # ruvsense/pose_tracker.rs
|
||||
onnx_inference_engine: true # wifi-densepose-nn
|
||||
modality_translator: true # wifi-densepose-nn/translator.rs
|
||||
training_pipeline: true # wifi-densepose-train
|
||||
pi_zero_deployment: false # No Pi Zero target
|
||||
lightweight_model: false # No edge-optimized model
|
||||
temporal_conv_module: false # No TCN in inference path
|
||||
csi_compression: false # No ESP32-side compression
|
||||
int8_quantization: false # No quantization pipeline
|
||||
bone_constraint_loss: false # No skeleton physics in loss
|
||||
esp32_pi_protocol: false # No lightweight protocol
|
||||
edge_inference_engine: false # No ARM-optimized inference
|
||||
cross_env_adaptation: false # No domain adaptation
|
||||
multi_person_paf: false # No PAF-based multi-person
|
||||
3d_pose_lifting: false # No Z-axis estimation
|
||||
|
||||
goal_state:
|
||||
esp32_csi_capture: true
|
||||
multi_node_aggregation: true
|
||||
phase_alignment: true
|
||||
coherence_gating: true
|
||||
multistatic_fusion: true
|
||||
kalman_pose_tracking: true
|
||||
onnx_inference_engine: true
|
||||
modality_translator: true
|
||||
training_pipeline: true
|
||||
pi_zero_deployment: true # TARGET
|
||||
lightweight_model: true # TARGET
|
||||
temporal_conv_module: true # TARGET
|
||||
csi_compression: true # TARGET
|
||||
int8_quantization: true # TARGET
|
||||
bone_constraint_loss: true # TARGET
|
||||
esp32_pi_protocol: true # TARGET
|
||||
edge_inference_engine: true # TARGET
|
||||
cross_env_adaptation: true # TARGET (Phase 2)
|
||||
multi_person_paf: true # TARGET (Phase 2)
|
||||
3d_pose_lifting: true # TARGET (Phase 3)
|
||||
```
|
||||
|
||||
## 2. Action Definitions
|
||||
|
||||
Each action has preconditions, effects, estimated cost (developer-days), and priority.
|
||||
|
||||
### Action 1: Define ESP32-Pi Communication Protocol (ADR-069)
|
||||
|
||||
```
|
||||
name: define_esp32_pi_protocol
|
||||
cost: 3 days
|
||||
priority: CRITICAL (blocks all Pi Zero work)
|
||||
preconditions: [esp32_csi_capture]
|
||||
effects: [esp32_pi_protocol := true]
|
||||
```
|
||||
|
||||
**Description:** Design a lightweight binary protocol for ESP32 -> Pi Zero communication over UDP (WiFi) or UART (wired fallback).
|
||||
|
||||
**Protocol specification:**
|
||||
|
||||
```
|
||||
Frame Header (8 bytes):
|
||||
[0:1] magic: 0xCF01 (CSI Frame v1)
|
||||
[2] node_id: u8 (0-255, identifies ESP32 node)
|
||||
[3] frame_type: u8 (0=raw_csi, 1=compressed_features, 2=heartbeat)
|
||||
[4:5] sequence: u16 (monotonic frame counter, wraps at 65535)
|
||||
[6:7] payload_len: u16 (bytes following header)
|
||||
|
||||
Raw CSI Payload (frame_type=0):
|
||||
[0:3] timestamp_us: u32 (microseconds since boot, wraps at ~71 minutes)
|
||||
[4] channel: u8 (WiFi channel 1-13)
|
||||
[5] bandwidth: u8 (0=20MHz, 1=40MHz)
|
||||
[6] rssi: i8 (dBm)
|
||||
[7] noise_floor: i8 (dBm)
|
||||
[8:9] num_sc: u16 (number of subcarriers, typically 52 or 114)
|
||||
[10..] csi_data: [i16; num_sc * 2] (interleaved I/Q, little-endian)
|
||||
|
||||
Compressed Feature Payload (frame_type=1):
|
||||
[0:3] timestamp_us: u32
|
||||
[4] compression: u8 (0=none, 1=pca_16, 2=pca_32, 3=autoencoder)
|
||||
[5] num_features: u8 (number of feature dimensions)
|
||||
[6..] features: [f16; num_features] (half-precision floats)
|
||||
|
||||
Heartbeat Payload (frame_type=2):
|
||||
[0:3] uptime_s: u32
|
||||
[4:7] frames_sent: u32
|
||||
[8:9] free_heap: u16 (KB)
|
||||
[10] wifi_rssi: i8 (connection to AP)
|
||||
[11] battery_pct: u8 (0-100, 0xFF if wired)
|
||||
```
|
||||
|
||||
**Implementation locations:**
|
||||
- ESP32 firmware: `firmware/esp32-csi-node/main/protocol_v2.h`
|
||||
- Rust parser: `wifi-densepose-hardware/src/protocol_v2.rs`
|
||||
|
||||
**Design rationale:**
|
||||
- Fixed 8-byte header with magic number for frame synchronization
|
||||
- Half-precision (f16) for compressed features saves 50% bandwidth vs f32
|
||||
- Heartbeat enables Pi Zero to detect node failures and rebalance
|
||||
- Raw CSI mode for debugging; compressed mode for production
|
||||
|
||||
### Action 2: Implement Lightweight Model Architecture
|
||||
|
||||
```
|
||||
name: implement_lightweight_model
|
||||
cost: 10 days
|
||||
priority: CRITICAL (core inference capability)
|
||||
preconditions: [training_pipeline, onnx_inference_engine]
|
||||
effects: [lightweight_model := true, temporal_conv_module := true]
|
||||
```
|
||||
|
||||
**Architecture: WiFlowPose (hybrid WiFlow + MultiFormer)**
|
||||
|
||||
Based on SOTA analysis, we define a custom architecture combining the best elements:
|
||||
|
||||
```
|
||||
Input: CSI amplitude tensor [B, T, S]
|
||||
B = batch size
|
||||
T = temporal window (20 frames at 20 Hz = 1 second context)
|
||||
S = subcarriers (52 for ESP32-S3 20MHz, 114 for 40MHz)
|
||||
|
||||
Stage 1: Temporal Encoder (runs on ESP32 optionally, or Pi Zero)
|
||||
TCN with 4 layers, dilation [1, 2, 4, 8]
|
||||
Input: [B, T, S] = [B, 20, 52]
|
||||
Output: [B, T', C_t] = [B, 20, 64] (temporal features)
|
||||
|
||||
Stage 2: Spatial Encoder (runs on Pi Zero)
|
||||
Asymmetric convolution blocks (1xk kernels on subcarrier dimension)
|
||||
4 residual blocks: 64 -> 128 -> 128 -> 64 channels
|
||||
Subcarrier compression: 52 -> 26 -> 13 -> 7
|
||||
Output: [B, 64, 7]
|
||||
|
||||
Stage 3: Keypoint Decoder (runs on Pi Zero)
|
||||
Axial self-attention (2-stage, 4 heads)
|
||||
Reshape to [B, 17, 64] (17 keypoints x 64 features)
|
||||
Linear projection: 64 -> 2 (x, y coordinates)
|
||||
Output: [B, 17, 2] (17 COCO keypoints, normalized 0-1)
|
||||
|
||||
Optional Stage 4: Multi-person (Phase 2)
|
||||
PAF branch: predict 19 limb affinity fields
|
||||
Hungarian assignment for person grouping
|
||||
```
|
||||
|
||||
**Estimated model size:**
|
||||
- Temporal encoder: ~0.5M params
|
||||
- Spatial encoder: ~1.2M params
|
||||
- Keypoint decoder: ~0.8M params
|
||||
- Total: ~2.5M params
|
||||
- INT8 size: ~2.5 MB
|
||||
- FP16 size: ~5 MB
|
||||
- Estimated Pi Zero 2 W inference: 30-60ms per frame
|
||||
|
||||
**Rust implementation location:** New module in `wifi-densepose-nn/src/wiflow_pose.rs`
|
||||
|
||||
```rust
|
||||
/// WiFlowPose: Lightweight WiFi CSI to pose estimation model
|
||||
///
|
||||
/// Hybrid architecture combining WiFlow's TCN temporal encoder
|
||||
/// with MultiFormer's dual-token spatial processing and
|
||||
/// axial self-attention for keypoint decoding.
|
||||
pub struct WiFlowPoseConfig {
|
||||
/// Number of input subcarriers (52 for ESP32 20MHz, 114 for 40MHz)
|
||||
pub num_subcarriers: usize,
|
||||
/// Temporal window size in frames (default: 20)
|
||||
pub temporal_window: usize,
|
||||
/// TCN dilation factors (default: [1, 2, 4, 8])
|
||||
pub tcn_dilations: Vec<usize>,
|
||||
/// Number of output keypoints (default: 17, COCO format)
|
||||
pub num_keypoints: usize,
|
||||
/// Hidden dimension for spatial encoder (default: 64)
|
||||
pub hidden_dim: usize,
|
||||
/// Number of attention heads in axial attention (default: 4)
|
||||
pub num_attention_heads: usize,
|
||||
/// Enable multi-person PAF branch (default: false)
|
||||
pub multi_person: bool,
|
||||
}
|
||||
|
||||
impl Default for WiFlowPoseConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
num_subcarriers: 52,
|
||||
temporal_window: 20,
|
||||
tcn_dilations: vec![1, 2, 4, 8],
|
||||
num_keypoints: 17,
|
||||
hidden_dim: 64,
|
||||
num_attention_heads: 4,
|
||||
multi_person: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Action 3: Implement Bone Constraint Loss
|
||||
|
||||
```
|
||||
name: implement_bone_constraint_loss
|
||||
cost: 2 days
|
||||
priority: HIGH
|
||||
preconditions: [training_pipeline, lightweight_model]
|
||||
effects: [bone_constraint_loss := true]
|
||||
```
|
||||
|
||||
**Loss function following WiFlow:**
|
||||
|
||||
```
|
||||
L_total = L_keypoint + lambda_bone * L_bone + lambda_physics * L_physics
|
||||
|
||||
L_keypoint = SmoothL1(pred, gt, beta=0.1)
|
||||
|
||||
L_bone = (1/|B|) * sum_{(i,j) in bones} | ||pred_i - pred_j|| - bone_length_{ij} |
|
||||
|
||||
L_physics = (1/N) * sum_t max(0, ||pred_t - pred_{t-1}|| - v_max * dt)
|
||||
```
|
||||
|
||||
Where:
|
||||
- `bones` = 14 COCO bone connections (e.g., left_shoulder-left_elbow)
|
||||
- `bone_length_{ij}` = average human bone length ratios (normalized to torso length)
|
||||
- `v_max` = maximum physiologically plausible keypoint velocity (2 m/s for walking, 10 m/s for fast gestures)
|
||||
- `lambda_bone = 0.2`, `lambda_physics = 0.1`
|
||||
|
||||
**Bone length ratios (normalized to torso = shoulder_center to hip_center = 1.0):**
|
||||
|
||||
| Bone | Ratio |
|
||||
|------|-------|
|
||||
| shoulder-elbow | 0.55 |
|
||||
| elbow-wrist | 0.50 |
|
||||
| hip-knee | 0.85 |
|
||||
| knee-ankle | 0.80 |
|
||||
| shoulder-hip | 1.00 |
|
||||
| neck-nose | 0.30 |
|
||||
| nose-eye | 0.08 |
|
||||
| eye-ear | 0.12 |
|
||||
|
||||
**Implementation location:** `wifi-densepose-train/src/losses.rs` (add `BoneConstraintLoss`)
|
||||
|
||||
### Action 4: Implement INT8 Quantization Pipeline
|
||||
|
||||
```
|
||||
name: implement_int8_quantization
|
||||
cost: 5 days
|
||||
priority: HIGH
|
||||
preconditions: [lightweight_model, training_pipeline]
|
||||
effects: [int8_quantization := true]
|
||||
```
|
||||
|
||||
**Approach: Post-Training Quantization (PTQ) with calibration**
|
||||
|
||||
1. Train model in FP32 using standard pipeline
|
||||
2. Export to ONNX format
|
||||
3. Run ONNX Runtime quantization tool with calibration dataset:
|
||||
- Collect 1000 representative CSI frames across multiple environments
|
||||
- Run calibration to determine per-layer quantization ranges
|
||||
- Apply symmetric INT8 quantization for weights, asymmetric for activations
|
||||
4. Validate quantized model accuracy (target: <2% PCK@20 degradation)
|
||||
|
||||
**Quantization-aware considerations:**
|
||||
- TCN layers: quantize per-channel (dilated convolutions are sensitive to quantization)
|
||||
- Attention layers: keep attention logits in FP16 (softmax is numerically sensitive)
|
||||
- Output layer: keep in FP32 (final coordinate regression needs precision)
|
||||
|
||||
**Rust implementation:**
|
||||
```rust
|
||||
// In wifi-densepose-nn/src/quantize.rs
|
||||
pub struct QuantizationConfig {
|
||||
/// Quantization method
|
||||
pub method: QuantMethod, // PTQ, QAT, Dynamic
|
||||
/// Per-layer precision overrides
|
||||
pub layer_overrides: HashMap<String, Precision>,
|
||||
/// Calibration dataset path
|
||||
pub calibration_data: PathBuf,
|
||||
/// Number of calibration samples
|
||||
pub num_calibration_samples: usize,
|
||||
/// Target accuracy degradation threshold
|
||||
pub max_accuracy_loss: f32,
|
||||
}
|
||||
|
||||
pub enum Precision {
|
||||
INT8,
|
||||
FP16,
|
||||
FP32,
|
||||
}
|
||||
```
|
||||
|
||||
**ONNX quantization command (for build pipeline):**
|
||||
```bash
|
||||
python -m onnxruntime.quantization.quantize \
|
||||
--input model_fp32.onnx \
|
||||
--output model_int8.onnx \
|
||||
--calibrate \
|
||||
--calibration_data_reader CsiCalibrationReader \
|
||||
--quant_format QDQ \
|
||||
--activation_type QUInt8 \
|
||||
--weight_type QInt8
|
||||
```
|
||||
|
||||
### Action 5: Build Edge Inference Engine for Pi Zero
|
||||
|
||||
```
|
||||
name: build_edge_inference_engine
|
||||
cost: 8 days
|
||||
priority: CRITICAL
|
||||
preconditions: [lightweight_model, int8_quantization, esp32_pi_protocol]
|
||||
effects: [edge_inference_engine := true, pi_zero_deployment := true]
|
||||
```
|
||||
|
||||
**Architecture: Streaming inference with ring buffer**
|
||||
|
||||
```
|
||||
UDP/UART
|
||||
ESP32-S3 ---------> Pi Zero 2 W
|
||||
|
|
||||
v
|
||||
+-- RingBuffer<CsiFrame> --+
|
||||
| (capacity: 64 frames) |
|
||||
+------ | | -------------+
|
||||
v v
|
||||
+-- TemporalWindow --------+
|
||||
| (20 frames, sliding) |
|
||||
+------ | ----------------+
|
||||
v
|
||||
+-- WiFlowPose ONNX ------+
|
||||
| (INT8, XNNPACK accel) |
|
||||
+------ | ----------------+
|
||||
v
|
||||
+-- PoseTracker -----------+
|
||||
| (Kalman + skeleton) |
|
||||
+------ | ----------------+
|
||||
v
|
||||
PoseEstimate output
|
||||
(17 keypoints + confidence)
|
||||
```
|
||||
|
||||
**New Rust binary:** `wifi-densepose-cli/src/bin/edge_infer.rs`
|
||||
|
||||
```rust
|
||||
/// Edge inference daemon for Raspberry Pi Zero 2 W
|
||||
///
|
||||
/// Receives CSI frames from ESP32 nodes via UDP, maintains a temporal
|
||||
/// sliding window, runs INT8 ONNX inference, and outputs pose estimates.
|
||||
///
|
||||
/// Usage:
|
||||
/// wifi-densepose edge-infer \
|
||||
/// --model model_int8.onnx \
|
||||
/// --listen 0.0.0.0:5555 \
|
||||
/// --output-port 5556 \
|
||||
/// --window-size 20 \
|
||||
/// --max-nodes 4
|
||||
|
||||
struct EdgeInferConfig {
|
||||
/// Path to INT8 ONNX model
|
||||
model_path: PathBuf,
|
||||
/// UDP listen address for CSI frames
|
||||
listen_addr: SocketAddr,
|
||||
/// UDP output address for pose results
|
||||
output_addr: Option<SocketAddr>,
|
||||
/// Temporal window size
|
||||
window_size: usize,
|
||||
/// Maximum ESP32 nodes to accept
|
||||
max_nodes: usize,
|
||||
/// Inference thread count (1-4 on Pi Zero 2 W)
|
||||
num_threads: usize,
|
||||
/// Enable XNNPACK acceleration
|
||||
use_xnnpack: bool,
|
||||
}
|
||||
```
|
||||
|
||||
**Cross-compilation for Pi Zero 2 W:**
|
||||
|
||||
```bash
|
||||
# Install cross-compilation toolchain
|
||||
rustup target add aarch64-unknown-linux-gnu
|
||||
sudo apt install gcc-aarch64-linux-gnu
|
||||
|
||||
# Build for Pi Zero 2 W (64-bit Raspberry Pi OS)
|
||||
cross build --target aarch64-unknown-linux-gnu \
|
||||
--release \
|
||||
-p wifi-densepose-cli \
|
||||
--features edge-inference \
|
||||
--no-default-features
|
||||
|
||||
# Or for 32-bit Raspberry Pi OS:
|
||||
# rustup target add armv7-unknown-linux-gnueabihf
|
||||
# cross build --target armv7-unknown-linux-gnueabihf ...
|
||||
```
|
||||
|
||||
**ONNX Runtime linking for ARM:**
|
||||
- Use `ort` crate with `download-binaries` feature for automatic aarch64 binary download
|
||||
- Alternative: build OnnxStream from source for minimal binary size (~2 MB vs ~30 MB for full ONNX Runtime)
|
||||
|
||||
### Action 6: Implement CSI Compression on ESP32
|
||||
|
||||
```
|
||||
name: implement_csi_compression
|
||||
cost: 5 days
|
||||
priority: MEDIUM
|
||||
preconditions: [esp32_csi_capture, esp32_pi_protocol]
|
||||
effects: [csi_compression := true]
|
||||
```
|
||||
|
||||
**Three compression tiers:**
|
||||
|
||||
**Tier 0: No compression (raw CSI)**
|
||||
- Payload: 52 subcarriers x 2 (I/Q) x 2 bytes = 208 bytes per frame
|
||||
- Use case: debugging, maximum fidelity
|
||||
|
||||
**Tier 1: PCA-16 (run on ESP32)**
|
||||
- Pre-computed PCA projection matrix (52 -> 16 dimensions)
|
||||
- Stored in NVS flash during provisioning
|
||||
- Payload: 16 features x 2 bytes (f16) = 32 bytes per frame
|
||||
- Compression: 6.5x
|
||||
- Compute: ~0.1ms on ESP32-S3 (matrix-vector multiply, SIMD)
|
||||
|
||||
**Tier 2: PCA-32 (higher fidelity)**
|
||||
- 52 -> 32 dimensions
|
||||
- Payload: 32 x 2 = 64 bytes
|
||||
- Compression: 3.25x
|
||||
|
||||
**Tier 3: Learned autoencoder (future)**
|
||||
- ESP32-S3 has enough compute for a small encoder (~10K params)
|
||||
- Requires quantized encoder weights in flash
|
||||
- Most bandwidth-efficient but requires training
|
||||
|
||||
**PCA computation (offline, during provisioning):**
|
||||
|
||||
```rust
|
||||
// wifi-densepose-train/src/compression.rs
|
||||
|
||||
/// Compute PCA projection matrix from calibration CSI data
|
||||
pub fn compute_pca_projection(
|
||||
calibration_data: &[CsiFrame],
|
||||
target_dims: usize,
|
||||
) -> PcaProjection {
|
||||
// 1. Stack all CSI amplitude vectors into matrix [N, S]
|
||||
// 2. Center (subtract mean)
|
||||
// 3. Compute covariance matrix [S, S]
|
||||
// 4. Eigendecomposition, take top `target_dims` eigenvectors
|
||||
// 5. Return projection matrix [S, target_dims] and mean vector [S]
|
||||
// ...
|
||||
}
|
||||
|
||||
pub struct PcaProjection {
|
||||
/// Projection matrix [num_subcarriers, target_dims]
|
||||
pub matrix: Vec<f32>,
|
||||
/// Mean vector for centering [num_subcarriers]
|
||||
pub mean: Vec<f32>,
|
||||
/// Number of input subcarriers
|
||||
pub input_dims: usize,
|
||||
/// Number of output features
|
||||
pub output_dims: usize,
|
||||
}
|
||||
```
|
||||
|
||||
**ESP32 firmware integration:**
|
||||
- Store PCA matrix in NVS partition (32x52x4 = 6.5 KB for PCA-32)
|
||||
- Apply projection in CSI callback before UDP transmission
|
||||
- Selectable via provisioning command
|
||||
|
||||
### Action 7: Implement Cross-Environment Adaptation
|
||||
|
||||
```
|
||||
name: implement_cross_env_adaptation
|
||||
cost: 8 days
|
||||
priority: MEDIUM (Phase 2)
|
||||
preconditions: [lightweight_model, training_pipeline, pi_zero_deployment]
|
||||
effects: [cross_env_adaptation := true]
|
||||
```
|
||||
|
||||
**Approach: Rapid environment calibration with few-shot adaptation**
|
||||
|
||||
Inspired by Arena Physica's template-based design space and MERIDIAN (ADR-027):
|
||||
|
||||
1. **Environment fingerprinting (on Pi Zero, at deployment time):**
|
||||
- Collect 60 seconds of "empty room" CSI
|
||||
- Compute room signature: mean amplitude profile, delay spread, K-factor
|
||||
- Match to nearest room template (corridor, office, bedroom, etc.)
|
||||
- Load template-specific model weights
|
||||
|
||||
2. **Few-shot fine-tuning (optional, on workstation):**
|
||||
- Collect 5 minutes of calibration data with known poses
|
||||
- Fine-tune last 2 layers of the model (~50K params)
|
||||
- Transfer updated model back to Pi Zero
|
||||
|
||||
3. **Online adaptation (continuous, on Pi Zero):**
|
||||
- Track CSI statistics over time (sliding window mean/variance)
|
||||
- Detect distribution shift (KL divergence exceeds threshold)
|
||||
- Apply batch normalization statistics update (no gradient computation needed)
|
||||
|
||||
**Implementation location:** `wifi-densepose-train/src/rapid_adapt.rs` (extend existing module)
|
||||
|
||||
### Action 8: Implement Multi-Person PAF Decoding
|
||||
|
||||
```
|
||||
name: implement_multi_person_paf
|
||||
cost: 6 days
|
||||
priority: LOW (Phase 2)
|
||||
preconditions: [lightweight_model, bone_constraint_loss]
|
||||
effects: [multi_person_paf := true]
|
||||
```
|
||||
|
||||
**Architecture (following MultiFormer):**
|
||||
|
||||
Add a PAF branch to the WiFlowPose model:
|
||||
|
||||
```
|
||||
Stage 3 features [B, 64, 7]
|
||||
|
|
||||
+--> Keypoint head: [B, 17, 2] (single-person keypoints)
|
||||
|
|
||||
+--> PAF head: [B, 38, H, W] (19 limb affinity fields)
|
||||
|
|
||||
+--> Confidence head: [B, 19, H, W] (part confidence maps)
|
||||
```
|
||||
|
||||
**Multi-person assignment on Pi Zero:**
|
||||
1. Extract candidate keypoints from confidence maps via NMS
|
||||
2. Compute PAF integral scores between candidate pairs
|
||||
3. Solve bipartite matching with Hungarian algorithm
|
||||
4. Group keypoints into person instances
|
||||
|
||||
**Estimated additional cost:** ~1M parameters, ~10ms additional inference time
|
||||
|
||||
### Action 9: Implement 3D Pose Lifting
|
||||
|
||||
```
|
||||
name: implement_3d_pose_lifting
|
||||
cost: 5 days
|
||||
priority: LOW (Phase 3)
|
||||
preconditions: [lightweight_model, multi_person_paf, multistatic_fusion]
|
||||
effects: [3d_pose_lifting := true]
|
||||
```
|
||||
|
||||
**Approach: Multi-view triangulation + learned depth prior**
|
||||
|
||||
With 2+ ESP32 nodes at known positions, compute 3D pose via:
|
||||
|
||||
1. Each node pair provides a different viewing angle of the WiFi field
|
||||
2. 2D pose from each viewpoint is estimated independently
|
||||
3. Epipolar geometry constrains 3D position from 2D observations
|
||||
4. Learned depth prior resolves ambiguities (front/back confusion)
|
||||
|
||||
This leverages the existing `viewpoint/geometry.rs` module in wifi-densepose-ruvector which already computes GeometricDiversityIndex and Fisher Information for multi-node configurations.
|
||||
|
||||
## 3. Hardware Architecture
|
||||
|
||||
### 3.1 System Topology
|
||||
|
||||
```
|
||||
WiFi AP (existing home router)
|
||||
/ | \
|
||||
/ | \
|
||||
ESP32-S3 #1 ESP32-S3 #2 ESP32-S3 #3
|
||||
(CSI node) (CSI node) (CSI node, optional)
|
||||
| | |
|
||||
+------+------+------+-------+
|
||||
| UDP (WiFi) |
|
||||
v v
|
||||
Raspberry Pi Zero 2 W
|
||||
(edge inference node)
|
||||
|
|
||||
v
|
||||
Pose output (UDP/MQTT/WebSocket)
|
||||
to display / home automation / API
|
||||
```
|
||||
|
||||
### 3.2 Data Flow Timing
|
||||
|
||||
```
|
||||
T=0ms ESP32 #1 captures CSI frame (channel 1)
|
||||
T=2ms ESP32 #1 applies PCA compression (0.1ms compute)
|
||||
T=3ms ESP32 #1 sends UDP packet to Pi Zero (64 bytes)
|
||||
T=5ms ESP32 #2 captures CSI frame (channel 6, TDM slot)
|
||||
T=7ms ESP32 #2 sends UDP packet to Pi Zero
|
||||
T=10ms Pi Zero receives both frames, adds to ring buffer
|
||||
T=10ms Pi Zero checks temporal window (20 frames accumulated?)
|
||||
If yes: run inference
|
||||
T=15ms Temporal encoder processes 20-frame window (5ms)
|
||||
T=35ms Spatial encoder + attention (20ms)
|
||||
T=45ms Keypoint decoder (10ms)
|
||||
T=48ms Kalman filter update + skeleton constraints (3ms)
|
||||
T=50ms Pose estimate emitted (17 keypoints + confidence)
|
||||
```
|
||||
|
||||
**Total latency: ~50ms** (well under 150ms target)
|
||||
**Throughput: 20 Hz** (matching TDMA cycle)
|
||||
|
||||
### 3.3 Hardware Bill of Materials
|
||||
|
||||
| Component | Unit Cost | Quantity | Total |
|
||||
|-----------|----------|----------|-------|
|
||||
| ESP32-S3 DevKit (8MB) | $9 | 2 | $18 |
|
||||
| Raspberry Pi Zero 2 W | $15 | 1 | $15 |
|
||||
| MicroSD card (16GB) | $5 | 1 | $5 |
|
||||
| USB-C power supply | $5 | 1 | $5 |
|
||||
| **Total** | | | **$43** |
|
||||
|
||||
With ESP32-S3 SuperMini ($6 each), total drops to **$37**.
|
||||
|
||||
For minimum viable setup (1 ESP32 + 1 Pi Zero): **$24**.
|
||||
|
||||
### 3.4 Pi Zero 2 W Specifications
|
||||
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| SoC | BCM2710A1 (quad-core Cortex-A53 @ 1 GHz) |
|
||||
| RAM | 512 MB LPDDR2 |
|
||||
| WiFi | 802.11b/g/n (2.4 GHz only) |
|
||||
| Bluetooth | BLE 4.2 |
|
||||
| GPIO | 40-pin header (UART, SPI, I2C) |
|
||||
| Power | 5V/2A USB micro-B |
|
||||
| OS | Raspberry Pi OS Lite (64-bit, headless) |
|
||||
|
||||
**Memory budget for inference:**
|
||||
|
||||
| Component | Memory |
|
||||
|-----------|--------|
|
||||
| OS + services | ~100 MB |
|
||||
| WiFlowPose INT8 model | ~3 MB |
|
||||
| ONNX Runtime / OnnxStream | ~10-30 MB |
|
||||
| Ring buffer (64 frames x 4 nodes) | ~1 MB |
|
||||
| Inference workspace | ~20 MB |
|
||||
| **Total** | ~134-164 MB |
|
||||
| **Available** | ~348-378 MB headroom |
|
||||
|
||||
Comfortable fit within 512 MB RAM.
|
||||
|
||||
## 4. Rust Crate Modifications
|
||||
|
||||
### 4.1 Modified Crates
|
||||
|
||||
#### wifi-densepose-hardware
|
||||
|
||||
**New files:**
|
||||
- `src/protocol_v2.rs` -- Lightweight ESP32-Pi binary protocol parser/serializer
|
||||
- `src/pi_zero.rs` -- Pi Zero UDP receiver with ring buffer management
|
||||
|
||||
**Modified files:**
|
||||
- `src/lib.rs` -- Add `pub mod protocol_v2; pub mod pi_zero;`
|
||||
- `src/aggregator/mod.rs` -- Add support for protocol_v2 frame format
|
||||
|
||||
#### wifi-densepose-nn
|
||||
|
||||
**New files:**
|
||||
- `src/wiflow_pose.rs` -- WiFlowPose model definition (TCN + asymmetric conv + axial attention)
|
||||
- `src/edge_engine.rs` -- Edge-optimized inference engine (streaming, ARM NEON)
|
||||
- `src/quantize.rs` -- INT8 quantization configuration and validation
|
||||
|
||||
**Modified files:**
|
||||
- `src/lib.rs` -- Add new module exports
|
||||
- `src/onnx.rs` -- Add XNNPACK execution provider option, INT8 model loading
|
||||
- `src/translator.rs` -- Add WiFlowPose-compatible input format
|
||||
|
||||
#### wifi-densepose-train
|
||||
|
||||
**New files:**
|
||||
- `src/wiflow_pose_trainer.rs` -- Training loop for WiFlowPose architecture
|
||||
- `src/compression.rs` -- PCA computation for ESP32 CSI compression
|
||||
- `src/bone_loss.rs` -- Bone constraint and physics consistency losses
|
||||
|
||||
**Modified files:**
|
||||
- `src/losses.rs` -- Add `BoneConstraintLoss`, `PhysicsConsistencyLoss`
|
||||
- `src/config.rs` -- Add WiFlowPose training configuration options
|
||||
- `src/dataset.rs` -- Add ESP32-S3 CSI format support (52/114 subcarriers)
|
||||
- `src/rapid_adapt.rs` -- Add few-shot environment calibration
|
||||
|
||||
#### wifi-densepose-signal
|
||||
|
||||
**New files:**
|
||||
- `src/ruvsense/temporal_encoder.rs` -- TCN temporal feature extraction (shared code for ESP32 and Pi)
|
||||
|
||||
**Modified files:**
|
||||
- `src/ruvsense/mod.rs` -- Add `pub mod temporal_encoder;`
|
||||
|
||||
#### wifi-densepose-cli
|
||||
|
||||
**New files:**
|
||||
- `src/bin/edge_infer.rs` -- Pi Zero edge inference daemon
|
||||
- `src/bin/calibrate.rs` -- Environment calibration tool (PCA computation, room fingerprinting)
|
||||
|
||||
#### wifi-densepose-core
|
||||
|
||||
**Modified files:**
|
||||
- `src/types.rs` -- Add `CompressedCsiFrame`, `EdgePoseEstimate` types
|
||||
|
||||
### 4.2 New Feature Flags
|
||||
|
||||
```toml
|
||||
# wifi-densepose-nn/Cargo.toml
|
||||
[features]
|
||||
default = ["onnx"]
|
||||
onnx = ["ort"]
|
||||
edge-inference = ["onnx", "xnnpack"] # NEW: ARM NEON + XNNPACK
|
||||
candle = ["candle-core", "candle-nn"]
|
||||
tch-backend = ["tch"]
|
||||
|
||||
# wifi-densepose-cli/Cargo.toml
|
||||
[features]
|
||||
default = ["full"]
|
||||
full = ["wifi-densepose-nn/onnx", "wifi-densepose-train/tch-backend"]
|
||||
edge-inference = ["wifi-densepose-nn/edge-inference"] # NEW: minimal binary for Pi
|
||||
```
|
||||
|
||||
### 4.3 Cross-Compilation Configuration
|
||||
|
||||
```toml
|
||||
# .cargo/config.toml (add section)
|
||||
[target.aarch64-unknown-linux-gnu]
|
||||
linker = "aarch64-linux-gnu-gcc"
|
||||
rustflags = ["-C", "target-cpu=cortex-a53", "-C", "target-feature=+neon"]
|
||||
```
|
||||
|
||||
## 5. ESP32 Firmware Modifications
|
||||
|
||||
### 5.1 New Files
|
||||
|
||||
- `firmware/esp32-csi-node/main/protocol_v2.h` -- Protocol v2 frame packing
|
||||
- `firmware/esp32-csi-node/main/pca_compress.h` -- PCA compression for CSI
|
||||
- `firmware/esp32-csi-node/main/pca_compress.c` -- PCA implementation with ESP32 SIMD
|
||||
- `firmware/esp32-csi-node/main/pi_zero_mode.c` -- Pi Zero communication mode (lighter than full server mode)
|
||||
|
||||
### 5.2 Modified Files
|
||||
|
||||
- `firmware/esp32-csi-node/main/csi_handler.c` -- Add compression step in CSI callback
|
||||
- `firmware/esp32-csi-node/main/nvs_config.c` -- Store PCA matrix in NVS
|
||||
- `firmware/esp32-csi-node/main/Kconfig.projbuild` -- Add CONFIG_PI_ZERO_MODE, CONFIG_CSI_COMPRESSION options
|
||||
|
||||
### 5.3 Provisioning Updates
|
||||
|
||||
```bash
|
||||
# Provision for Pi Zero mode with PCA-16 compression
|
||||
python firmware/esp32-csi-node/provision.py \
|
||||
--port COM7 \
|
||||
--ssid "MyWiFi" \
|
||||
--password "secret" \
|
||||
--target-ip 192.168.1.50 \ # Pi Zero IP
|
||||
--target-port 5555 \
|
||||
--compression pca-16 \
|
||||
--pca-matrix pca_matrix_16.bin
|
||||
```
|
||||
|
||||
## 6. Training Pipeline
|
||||
|
||||
### 6.1 Training Workflow
|
||||
|
||||
```
|
||||
Phase 1: Pre-train on public datasets (GPU workstation)
|
||||
Dataset: MM-Fi + Wi-Pose (Intel 5300 format, 30 subcarriers)
|
||||
Model: WiFlowPose with 30 subcarriers
|
||||
Loss: L_keypoint + 0.2 * L_bone + 0.1 * L_physics
|
||||
Duration: ~20 hours on single A100
|
||||
|
||||
Phase 2: Domain adaptation for ESP32 CSI (GPU workstation)
|
||||
Dataset: Self-collected ESP32-S3 data (52 subcarriers)
|
||||
Method: Fine-tune all layers with lower learning rate (1e-4)
|
||||
Subcarrier interpolation: 30 -> 52 using existing interpolate_subcarriers()
|
||||
Duration: ~4 hours
|
||||
|
||||
Phase 3: Quantization (CPU workstation)
|
||||
Method: Post-training quantization with 1000 calibration samples
|
||||
Format: ONNX INT8 (QDQ format)
|
||||
Validation: PCK@20 degradation < 2%
|
||||
|
||||
Phase 4: Environment calibration (on Pi Zero)
|
||||
Method: 60-second empty-room CSI collection
|
||||
Output: Room fingerprint + PCA matrix
|
||||
Duration: ~2 minutes total
|
||||
```
|
||||
|
||||
### 6.2 Dataset Collection Protocol
|
||||
|
||||
For self-collected ESP32 training data:
|
||||
|
||||
1. **Setup:** 2 ESP32-S3 nodes at opposite corners of 4x4m room, Pi Zero receiving
|
||||
2. **Ground truth:** Smartphone camera running MediaPipe Pose (30 FPS), synchronized via NTP
|
||||
3. **Activities:** Standing, walking, sitting, waving, falling, idle (2 minutes each)
|
||||
4. **Subjects:** 5+ volunteers with varying body types
|
||||
5. **Environments:** 3+ rooms (bedroom, office, corridor) for generalization
|
||||
6. **Total target:** ~100K synchronized CSI-pose frame pairs
|
||||
|
||||
**Synchronization approach:**
|
||||
- ESP32 and Pi Zero synchronized via NTP (< 10ms accuracy on LAN)
|
||||
- Camera frames timestamped with system clock
|
||||
- Offline alignment via cross-correlation of movement signals
|
||||
|
||||
### 6.3 Transfer Learning Strategy
|
||||
|
||||
Following DensePose-WiFi's proven approach:
|
||||
|
||||
```
|
||||
L_total = lambda_pose * L_pose
|
||||
+ lambda_bone * L_bone
|
||||
+ lambda_transfer * L_transfer
|
||||
+ lambda_physics * L_physics
|
||||
|
||||
L_transfer = MSE(features_student, features_teacher)
|
||||
```
|
||||
|
||||
Where `features_teacher` come from a pre-trained image-based pose model (HRNet or ViTPose) and `features_student` come from the WiFi CSI model at corresponding intermediate layers.
|
||||
|
||||
**Lambda schedule:**
|
||||
- Epochs 1-20: lambda_transfer = 0.5 (heavy transfer guidance)
|
||||
- Epochs 20-50: lambda_transfer = 0.2 (moderate guidance)
|
||||
- Epochs 50-100: lambda_transfer = 0.05 (fine-tuning freedom)
|
||||
|
||||
## 7. Timeline and Milestones
|
||||
|
||||
### Phase 1: Foundation (Weeks 1-4)
|
||||
|
||||
| Week | Actions | Deliverable |
|
||||
|------|---------|-------------|
|
||||
| 1 | Action 1 (protocol), ADR-069 draft | Protocol spec + parser tests |
|
||||
| 2 | Action 2 (model architecture, begin) | WiFlowPose model definition in Rust |
|
||||
| 2 | Action 3 (bone loss) | Loss functions implemented and tested |
|
||||
| 3 | Action 2 (model architecture, complete) | Full model with ONNX export |
|
||||
| 4 | Action 4 (quantization) | INT8 model, accuracy validated |
|
||||
|
||||
**Milestone M1:** WiFlowPose model trained on MM-Fi, exported to INT8 ONNX, PCK@20 > 85% on validation set.
|
||||
|
||||
### Phase 2: Edge Deployment (Weeks 5-8)
|
||||
|
||||
| Week | Actions | Deliverable |
|
||||
|------|---------|-------------|
|
||||
| 5 | Action 5 (edge engine, begin) | Cross-compilation working, model loads on Pi |
|
||||
| 6 | Action 5 (edge engine, complete) | Streaming inference at >= 10 Hz on Pi Zero |
|
||||
| 6 | Action 6 (CSI compression) | PCA compression on ESP32, verified bandwidth reduction |
|
||||
| 7 | Integration testing | ESP32 -> Pi Zero full pipeline working |
|
||||
| 8 | Performance optimization | Latency < 100ms, memory < 200 MB |
|
||||
|
||||
**Milestone M2:** End-to-end demo: ESP32 captures CSI, Pi Zero outputs pose at 10+ Hz.
|
||||
|
||||
### Phase 3: Accuracy and Adaptation (Weeks 9-12)
|
||||
|
||||
| Week | Actions | Deliverable |
|
||||
|------|---------|-------------|
|
||||
| 9 | Data collection (ESP32-S3 training data) | 50K+ synchronized CSI-pose frames |
|
||||
| 10 | Domain adaptation training | ESP32-specific model, MPJPE < 120mm |
|
||||
| 11 | Action 7 (cross-env adaptation) | Room calibration working |
|
||||
| 12 | Validation and documentation | ADR-069 finalized, witness bundle |
|
||||
|
||||
**Milestone M3:** Single-person MPJPE < 100mm in calibrated environment, cross-environment deployment working with 60-second calibration.
|
||||
|
||||
### Phase 4: Multi-Person and 3D (Weeks 13-20)
|
||||
|
||||
| Week | Actions | Deliverable |
|
||||
|------|---------|-------------|
|
||||
| 13-14 | Action 8 (multi-person PAF) | 2-person pose separation working |
|
||||
| 15-16 | Action 9 (3D lifting) | Z-axis estimation from multi-node |
|
||||
| 17-18 | Advanced optimization | Model distillation, QAT |
|
||||
| 19-20 | Production hardening | OTA updates, monitoring, alerting |
|
||||
|
||||
**Milestone M4:** Multi-person 3D pose at 10 Hz on Pi Zero 2 W.
|
||||
|
||||
## 8. Risk Analysis
|
||||
|
||||
### 8.1 Technical Risks
|
||||
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| Pi Zero 2 W inference too slow (> 100ms) | Medium | High | Fall back to activity recognition (smaller model); use Pi 4 instead |
|
||||
| ESP32-S3 CSI quality insufficient for pose | Low | Critical | Already validated in ADR-028; add directional antennas if needed |
|
||||
| INT8 quantization degrades accuracy > 5% | Medium | Medium | Use FP16 instead (2x size, ~1.5x slower); apply QAT |
|
||||
| Cross-environment generalization poor | High | High | Room calibration (Action 7); template-based models; continuous adaptation |
|
||||
| WiFi interference degrades CSI | Medium | Medium | Coherence gating (already implemented); channel hopping; 5 GHz fallback |
|
||||
| ONNX Runtime binary too large for Pi Zero | Low | Medium | Use OnnxStream (2 MB) instead of full ONNX Runtime (30 MB) |
|
||||
| Multi-person association errors | High | Medium | Limit to 2 persons initially; use PAF + Hungarian; AETHER re-ID |
|
||||
|
||||
### 8.2 Hardware Risks
|
||||
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| Pi Zero 2 W supply shortage | Medium | Medium | Design also works with Pi 3A+ or Pi 4 |
|
||||
| ESP32-S3 firmware instability | Low | Medium | Existing firmware battle-tested; OTA rollback |
|
||||
| WiFi AP interference with CSI | Low | Low | Dedicated 2.4 GHz channel; ESP32 channel hopping |
|
||||
| Power supply issues (brownout) | Low | Medium | Proper power supply; ESP32 brownout detection |
|
||||
|
||||
### 8.3 Research Risks
|
||||
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| WiFlow results don't reproduce | Medium | High | Fall back to CSI-Former or MultiFormer architecture |
|
||||
| ESP32 CSI fundamentally different from Intel 5300 | Medium | High | Collect ESP32-specific training data; subcarrier interpolation |
|
||||
| Bone constraint loss doesn't improve edge accuracy | Low | Low | Remove if no benefit; constraint is simple and cheap |
|
||||
| PCA compression loses critical CSI information | Low | Medium | Validate with ablation study; fall back to raw CSI if needed |
|
||||
|
||||
## 9. Dependency Graph (Action Ordering)
|
||||
|
||||
```
|
||||
[esp32_csi_capture] (DONE)
|
||||
/ \
|
||||
v v
|
||||
[Action 1: Protocol] [training_pipeline] (DONE)
|
||||
| / | \
|
||||
v v v v
|
||||
[Action 6: Compression] [Action 2: Model] [Action 3: Bone Loss]
|
||||
| | |
|
||||
| +------+-------+
|
||||
| v
|
||||
| [Action 4: Quantization]
|
||||
| |
|
||||
+---------------+------------+
|
||||
v
|
||||
[Action 5: Edge Engine]
|
||||
|
|
||||
v
|
||||
[Action 7: Cross-Env] (Phase 2)
|
||||
|
|
||||
v
|
||||
[Action 8: Multi-Person] (Phase 2)
|
||||
|
|
||||
v
|
||||
[Action 9: 3D Lifting] (Phase 3)
|
||||
```
|
||||
|
||||
**Critical path:** Action 1 -> Action 2 -> Action 4 -> Action 5
|
||||
**Parallel path:** Action 3 can proceed concurrently with Action 2
|
||||
**Parallel path:** Action 6 can proceed concurrently with Actions 2-4
|
||||
|
||||
## 10. Success Criteria
|
||||
|
||||
### Phase 1 Exit Criteria
|
||||
|
||||
- [ ] WiFlowPose model trains to convergence on MM-Fi dataset
|
||||
- [ ] PCK@20 >= 85% on MM-Fi validation set
|
||||
- [ ] INT8 ONNX model size < 5 MB
|
||||
- [ ] Bone constraint loss reduces physically implausible predictions by > 50%
|
||||
|
||||
### Phase 2 Exit Criteria
|
||||
|
||||
- [ ] edge_infer binary cross-compiles for aarch64 and runs on Pi Zero 2 W
|
||||
- [ ] End-to-end latency < 150ms (CSI capture to pose output)
|
||||
- [ ] Inference rate >= 10 Hz sustained
|
||||
- [ ] PCA compression reduces bandwidth by >= 3x without > 5% accuracy loss
|
||||
- [ ] Multi-node support (2 ESP32 nodes + 1 Pi Zero) working
|
||||
|
||||
### Phase 3 Exit Criteria
|
||||
|
||||
- [ ] Single-person MPJPE < 100mm in calibrated environment
|
||||
- [ ] Cross-environment deployment works with 60-second calibration
|
||||
- [ ] System runs continuously for 24 hours without crashes
|
||||
- [ ] ESP32 OTA firmware update working for CSI compression parameters
|
||||
|
||||
### Phase 4 Exit Criteria
|
||||
|
||||
- [ ] 2-person pose separation working (MPJPE < 150mm per person)
|
||||
- [ ] 3D pose estimation from 2+ nodes (Z-axis error < 200mm)
|
||||
- [ ] Production monitoring and alerting operational
|
||||
|
||||
## 11. Relationship to Existing ADRs
|
||||
|
||||
| ADR | Relationship |
|
||||
|-----|-------------|
|
||||
| ADR-018 | Protocol v2 (Action 1) extends ADR-018 binary frame format |
|
||||
| ADR-024 | AETHER re-ID embeddings used in multi-person tracking (Action 8) |
|
||||
| ADR-027 | MERIDIAN cross-env generalization informs Action 7 |
|
||||
| ADR-028 | ESP32 capability audit validates CSI quality assumptions |
|
||||
| ADR-029 | RuvSense pipeline stages feed into edge inference (Action 5) |
|
||||
| ADR-068 | Per-node state pipeline directly used by multi-node inference |
|
||||
|
||||
## 12. New ADR Required
|
||||
|
||||
**ADR-069: Edge Inference on Raspberry Pi Zero 2 W**
|
||||
|
||||
This implementation plan should be formalized as ADR-069 covering:
|
||||
- Protocol v2 specification
|
||||
- WiFlowPose architecture selection rationale
|
||||
- Pi Zero deployment constraints and optimizations
|
||||
- INT8 quantization strategy
|
||||
- Cross-compilation approach
|
||||
- Environment calibration protocol
|
||||
|
||||
Status: Proposed, pending this plan's approval.
|
||||
@@ -0,0 +1,142 @@
|
||||
# Analysis: Arena Physica and Atlas RF Studio
|
||||
|
||||
## Company Overview
|
||||
|
||||
Arena Physica positions itself as building "Electromagnetic Superintelligence" -- a foundation model trained directly on electromagnetic fields, one of the four fundamental forces of physics.
|
||||
|
||||
**Website:** https://www.arenaphysica.com/
|
||||
**Key Product:** Atlas RF Studio (Beta)
|
||||
**Core Models:** Heaviside-0 (forward prediction), Marconi-0 (inverse design)
|
||||
|
||||
## Technical Architecture
|
||||
|
||||
### Heaviside-0: Forward Electromagnetic Model
|
||||
|
||||
A transformer-based neural network that predicts S-parameters (scattering parameters) from circuit geometry.
|
||||
|
||||
**Performance claims:**
|
||||
- Weighted MAE: < 1 dB
|
||||
- Speed: 13ms per design vs 4 minutes for traditional EM solvers
|
||||
- Speedup: 18,000x to 800,000x over commercial solvers (HFSS, CST)
|
||||
|
||||
**Architecture insights:**
|
||||
- Transformer backbone (specific architecture undisclosed)
|
||||
- Trained on electromagnetic field data, not just input-output mappings
|
||||
- Field augmentation acts as a regularizer -- even 0.3% field coverage during training reduced OOD loss
|
||||
|
||||
### Marconi-0: Inverse Design Model
|
||||
|
||||
A diffusion-based generative model that produces physical RF geometries matching target S-parameter specifications.
|
||||
|
||||
**Approach:**
|
||||
- Iterative refinement (diffusion process)
|
||||
- Generates "alien structures" -- non-intuitive geometries that meet specs
|
||||
- Trades compute time for quality (more diffusion steps = better designs)
|
||||
|
||||
### Training Data
|
||||
|
||||
**Simulated data:** 3 million designs across 25 expert templates with procedural variations, plus random organic structures to force learning in unexplored design space regions.
|
||||
|
||||
**Measured data:** Fabricated designs tested with vector network analyzers to capture manufacturing tolerances, material variations, connector parasitics.
|
||||
|
||||
**Total claimed:** 20M+ simulated designs in the broader training set.
|
||||
|
||||
### Current Design Space
|
||||
|
||||
- 2-layer PCB designs (8mm x 8mm)
|
||||
- 3 dielectric material choices
|
||||
- Ground vias
|
||||
- Filters and antennas
|
||||
|
||||
## Key Technical Insight: Fields as Fundamental Quantities
|
||||
|
||||
Arena Physica's central thesis is that Maxwell's equations govern electromagnetic fields, and models trained on field distributions learn the underlying physics rather than surface-level correlations between geometry and S-parameters.
|
||||
|
||||
This is directly relevant to WiFi sensing because:
|
||||
|
||||
1. **CSI IS an electromagnetic field measurement.** WiFi Channel State Information captures the complex transfer function H(f) between transmitter and receiver antennas across frequency subcarriers. This is a discrete sampling of the electromagnetic field in the propagation environment.
|
||||
|
||||
2. **Human bodies perturb the electromagnetic field.** Pose estimation from WiFi works because the human body (70% water, high permittivity) creates measurable perturbations in the ambient electromagnetic field.
|
||||
|
||||
3. **Foundation model approach could apply to sensing.** A model trained on electromagnetic field distributions in rooms with human bodies could potentially generalize across environments better than models trained on CSI-to-pose mappings directly.
|
||||
|
||||
## Relevance to WiFi-DensePose Project
|
||||
|
||||
### Direct Applicability: Moderate
|
||||
|
||||
Arena Physica's current focus is RF component design (filters, antennas), not sensing. However, several concepts transfer directly:
|
||||
|
||||
### 1. Physics-Informed Neural Architecture
|
||||
|
||||
Arena Physica trains on the electromagnetic field itself, not just input-output pairs. We should adopt this principle:
|
||||
|
||||
**Current approach in wifi-densepose:**
|
||||
```
|
||||
CSI amplitude/phase -> CNN/Transformer -> Keypoint coordinates
|
||||
```
|
||||
|
||||
**Physics-informed approach inspired by Arena Physica:**
|
||||
```
|
||||
CSI amplitude/phase -> Field reconstruction -> Body perturbation extraction -> Pose estimation
|
||||
```
|
||||
|
||||
Concretely, this means adding an intermediate field reconstruction stage that produces a spatial electromagnetic field map (similar to our existing `tomography.rs` module in RuvSense) and then extracting body perturbation from the field rather than going directly from CSI to pose.
|
||||
|
||||
### 2. Forward Model for Data Augmentation
|
||||
|
||||
Heaviside-0 predicts S-parameters from geometry. An analogous forward model for WiFi sensing would predict CSI from (room geometry + human pose). This enables:
|
||||
|
||||
- **Synthetic training data generation:** Generate CSI samples for arbitrary room layouts and poses
|
||||
- **Domain adaptation:** Bridge the sim-to-real gap by training the forward model on measured data
|
||||
- **Physics-based data augmentation:** Perturb room geometry parameters to generate diverse training environments
|
||||
|
||||
This directly addresses our MERIDIAN cross-environment generalization challenge (ADR-027).
|
||||
|
||||
### 3. Diffusion-Based Inverse Models
|
||||
|
||||
Marconi-0 uses diffusion to solve the inverse problem (S-parameters -> geometry). The analogous inverse problem for WiFi sensing is (CSI -> pose). Recent work on diffusion-based pose estimation could be adapted:
|
||||
|
||||
- Generate multiple pose hypotheses from a single CSI observation
|
||||
- Score hypotheses by physical plausibility (bone length constraints, joint angle limits)
|
||||
- Select the highest-scoring hypothesis
|
||||
|
||||
This is more robust than single-shot regression for ambiguous CSI measurements.
|
||||
|
||||
### 4. Multi-Resolution Field Representation
|
||||
|
||||
Arena Physica operates on 2-layer PCB designs at the mm scale. WiFi sensing operates at the wavelength scale (12.5 cm at 2.4 GHz). However, the principle of multi-resolution field representation applies:
|
||||
|
||||
- **Coarse grid:** Room-level field structure (presence detection, zone occupancy)
|
||||
- **Medium grid:** Body-level perturbation (bounding box, silhouette)
|
||||
- **Fine grid:** Limb-level detail (keypoint localization)
|
||||
|
||||
This maps to our existing RuvSense tomography module which implements RF tomography on a voxel grid, but suggests a multi-resolution approach would be more efficient.
|
||||
|
||||
## Adaptation Strategy for ESP32 + Pi Zero Deployment
|
||||
|
||||
### What to borrow from Arena Physica:
|
||||
|
||||
1. **Field-augmented training:** During training (on GPU workstation), include an auxiliary loss that encourages the model to predict the electromagnetic field distribution, not just keypoints. This regularizes the model and improves OOD generalization. At inference time on Pi Zero, the field prediction head is pruned.
|
||||
|
||||
2. **Lightweight forward model:** Train a small forward model (CSI predictor given room parameters) on the ESP32 side. This enables on-device anomaly detection: if observed CSI deviates significantly from the forward model prediction, flag the observation as potentially adversarial or corrupted.
|
||||
|
||||
3. **Template-based design space:** Arena Physica uses 25 expert templates with procedural variations. We should define "room templates" (corridor, open office, bedroom, living room) and train specialized lightweight models per template, selected at deployment time.
|
||||
|
||||
### What does NOT transfer:
|
||||
|
||||
1. **Scale of training data:** 20M+ designs is infeasible for WiFi sensing. Real CSI data collection is expensive. Synthetic data (ray tracing simulation) partially addresses this but lacks the fidelity of Arena Physica's EM simulations.
|
||||
|
||||
2. **Diffusion models on edge:** Marconi-0's diffusion approach is too computationally expensive for Pi Zero inference. We need single-shot architectures for real-time operation.
|
||||
|
||||
3. **2D geometry inputs:** Arena Physica processes 2D PCB layouts. WiFi sensing requires processing time-series data with complex spatial structure. The input representations are fundamentally different.
|
||||
|
||||
## Conclusions
|
||||
|
||||
Arena Physica demonstrates that foundation models trained on electromagnetic field data achieve superior generalization compared to models trained on input-output mappings alone. The key transferable insights for WiFi-DensePose are:
|
||||
|
||||
1. **Train on fields, not just observations** -- include field reconstruction as an auxiliary task
|
||||
2. **Use forward models for augmentation** -- predict CSI from room+pose for synthetic data
|
||||
3. **Multi-resolution representations** -- coarse-to-fine field reconstruction improves efficiency
|
||||
4. **Template-based specialization** -- room-type-specific models improve accuracy with lower compute
|
||||
|
||||
These insights inform the implementation plan, particularly the training pipeline design and the novel "field-augmented" training approach proposed in the implementation plan.
|
||||
@@ -0,0 +1,444 @@
|
||||
# Arena Physica Studio Analysis
|
||||
|
||||
Research document for wifi-densepose project.
|
||||
Date: 2026-04-02
|
||||
|
||||
---
|
||||
|
||||
## 1. What is Arena Physica?
|
||||
|
||||
Arena Physica (trading as Arena, arena-ai.com / arenaphysica.com) is a startup pursuing "Electromagnetic Superintelligence" -- building AI foundation models that develop superhuman intuition for how geometry shapes electromagnetic fields.
|
||||
|
||||
- **Founded**: 2019
|
||||
- **Founders**: Pratap Ranade (CEO), Arya Hezarkhani, Claire Pan, Michael Frei, Harish Krishnaswamy
|
||||
- **Funding**: $30M Series B (April 2025)
|
||||
- **Offices**: NYC (HQ), SF, LA
|
||||
- **Customers**: AMD, Anduril Industries, Sivers Semiconductors, Bausch & Lomb
|
||||
- **Impact claimed**: 35% reduction in engineering man-hours, multi-month acceleration in time-to-market, >3% improvement in product quality
|
||||
|
||||
Arena does NOT do WiFi sensing. They build AI-driven tools for RF/electromagnetic hardware design -- antennas, PCBs, filters, RF components. Their relevance to our project is methodological: they demonstrate how to build neural surrogates for Maxwell's equations that run 18,000x to 800,000x faster than traditional solvers.
|
||||
|
||||
|
||||
## 2. Atlas Platform and RF Studio
|
||||
|
||||
### 2.1 Atlas (Main Platform)
|
||||
|
||||
Atlas is Arena's "agentic platform" for hardware design workflows. It is deployed in production with Fortune 500 companies. Atlas encompasses:
|
||||
|
||||
- AI-driven electromagnetic simulation
|
||||
- Design generation and optimization
|
||||
- Hardware verification workflows
|
||||
- Integration with existing engineering tools
|
||||
|
||||
### 2.2 Atlas RF Studio (Public Beta)
|
||||
|
||||
Atlas RF Studio (https://studio.arenaphysica.com/) is a lightweight public instance of the Atlas platform, released as an "interactive sandbox for AI-driven inverse RF design." It serves as a research preview of their electromagnetic foundation model.
|
||||
|
||||
**Current capabilities (Beta):**
|
||||
- Two-layer RF structures
|
||||
- 8mm x 8mm maximum dimensions
|
||||
- Ground vias support
|
||||
- 3 dielectric material choices
|
||||
- AI-driven design generation from specifications
|
||||
- Real-time S-parameter prediction
|
||||
|
||||
**Workflow:**
|
||||
1. User inputs electromagnetic specifications (target S-parameters)
|
||||
2. Marconi-0 (inverse model) generates candidate geometries via conditional diffusion
|
||||
3. Heaviside-0 (forward model) evaluates each candidate in 13ms
|
||||
4. System iterates: generate -> simulate -> refine
|
||||
5. User receives optimized RF component design
|
||||
|
||||
### 2.3 Foundation Models
|
||||
|
||||
**Heaviside-0 (Forward Model)**:
|
||||
- Named after Oliver Heaviside (reformulated Maxwell's equations into modern vector form)
|
||||
- Predicts: S-parameters (magnitude + phase) and electromagnetic field distributions
|
||||
- Speed: 13ms single design, 0.3ms batched
|
||||
- Traditional solver comparison: ~4 minutes (HFSS/FDTD)
|
||||
- Speedup: 18,000x - 800,000x
|
||||
- Trained on 3 million designs across 25 expert templates + random structures
|
||||
- Training data represents 20+ years of combined simulation time
|
||||
- Accuracy: < 1 dB magnitude-weighted MAE
|
||||
|
||||
**Marconi-0 (Inverse Model)**:
|
||||
- Named after Guglielmo Marconi (radio pioneer)
|
||||
- Generates physical geometries from target S-parameter specifications
|
||||
- Uses conditional diffusion process (similar to Stable Diffusion / DALL-E architecture)
|
||||
- Can produce unconventional geometries that outperform human-designed solutions
|
||||
|
||||
### 2.4 Roadmap
|
||||
|
||||
Planned extensions include:
|
||||
- Multi-layer structures
|
||||
- Silicon integration (tapeout planned by end 2026)
|
||||
- Multiphysics integration (thermal, mechanical beyond EM)
|
||||
- Broader frequency ranges and design spaces
|
||||
|
||||
|
||||
## 3. Studio Technical Architecture
|
||||
|
||||
### 3.1 Frontend Stack
|
||||
|
||||
Based on runtime analysis of https://studio.arenaphysica.com/:
|
||||
|
||||
| Component | Technology | Evidence |
|
||||
|---|---|---|
|
||||
| Framework | Next.js (App Router, server-side streaming) | `__next_f`, `__next_s` arrays, static chunk loading |
|
||||
| UI Library | Mantine | Responsive breakpoint utilities (xs, sm, md, lg, xl) |
|
||||
| Rendering | React (server components + client hydration) | React streaming, component loading |
|
||||
| Fonts | Custom: Rules (Regular/Medium/Bold), EditionNumericalXXIX, Geist Mono (Google Fonts) | Font declarations in page source |
|
||||
| Theme | Dark mode default for "rf" domain | `ATLAS_DOMAIN: "rf"` config triggers dark theme |
|
||||
|
||||
### 3.2 Backend / API Infrastructure
|
||||
|
||||
| Service | Detail |
|
||||
|---|---|
|
||||
| API Domain | `https://api.emfm.atlas.arena-ai.com` (Auth0 audience) |
|
||||
| Organization | `emfmprod` |
|
||||
| Authentication | Auth0 with custom organization ID |
|
||||
| Feature Flags | DevCycle SDK (A/B testing) |
|
||||
| Monitoring | Datadog RUM (Real User Monitoring) |
|
||||
| 3D Rendering | Unreal Engine server at `https://52.61.97.121` (AWS IP) |
|
||||
| Terms of Service | Required (`ATLAS_REQUIRE_TOS: true`) |
|
||||
|
||||
### 3.3 Configuration Flags (from runtime config)
|
||||
|
||||
```json
|
||||
{
|
||||
"AUTH0_AUDIENCE": "https://api.emfm.atlas.arena-ai.com",
|
||||
"ATLAS_DOMAIN": "rf",
|
||||
"ATLAS_REQUIRE_TOS": true,
|
||||
"POLL_FOR_MESSAGES": false,
|
||||
"ENABLE_HOTJAR": false,
|
||||
"SHOW_DEBUG_LOGS": false
|
||||
}
|
||||
```
|
||||
|
||||
Key observations:
|
||||
- `POLL_FOR_MESSAGES: false` -- Messages likely use WebSocket/SSE push rather than polling
|
||||
- `ENABLE_HOTJAR: false` -- Session replay disabled in production
|
||||
- `SHOW_DEBUG_LOGS: false` -- Debug mode off
|
||||
- The `emfm` in the API domain likely stands for "ElectroMagnetic Field Model"
|
||||
|
||||
### 3.4 3D Visualization via Unreal Engine
|
||||
|
||||
The most technically interesting finding: Studio connects to an Unreal Engine server (IP: 52.61.97.121, AWS us-west region) for 3D electromagnetic field visualization.
|
||||
|
||||
**Likely architecture:**
|
||||
1. User submits design geometry in the Next.js frontend
|
||||
2. Backend runs Heaviside-0/Marconi-0 inference
|
||||
3. S-parameter results and field distribution data sent to Unreal Engine instance
|
||||
4. Unreal Engine renders 3D field visualization (E-field, H-field, current distributions)
|
||||
5. Pixel streaming sends rendered frames back to browser via WebRTC/WebSocket
|
||||
6. Interactive controls (rotate, zoom, slice planes) forwarded to Unreal Engine
|
||||
|
||||
This is consistent with Unreal Engine's Pixel Streaming technology, which renders on a remote GPU and streams video to a web browser. The `52.61.97.121` IP being hardcoded suggests a dedicated rendering server or fleet.
|
||||
|
||||
**Unreal Engine WebSocket Protocol** (standard):
|
||||
- Signaling server negotiates WebRTC connection
|
||||
- Control messages: `{ type: "input", data: { ... } }` for mouse/keyboard
|
||||
- Video stream: H.264/VP8 encoded, streamed via WebRTC data channel
|
||||
- Bidirectional: user input -> Unreal, rendered frames -> browser
|
||||
|
||||
### 3.5 Data Formats (Inferred)
|
||||
|
||||
Based on the S-parameter focus:
|
||||
|
||||
**Input (Design Specification):**
|
||||
- Target S-parameters: S11, S21, S12, S22 (magnitude + phase vs frequency)
|
||||
- Frequency range (likely GHz, given RF focus)
|
||||
- Material properties (dielectric constant, loss tangent)
|
||||
- Geometric constraints (layer count, max dimensions)
|
||||
|
||||
**Output (Design Result):**
|
||||
- Geometry: likely a discretized grid (64x64 binary material map based on Not Boring article)
|
||||
- S-parameters: complex-valued frequency response curves
|
||||
- Field distributions: 2D/3D electromagnetic field maps
|
||||
- Performance metrics: return loss, insertion loss, bandwidth
|
||||
|
||||
**Probable API format** (speculative, based on EM conventions):
|
||||
```json
|
||||
{
|
||||
"design": {
|
||||
"layers": [
|
||||
{
|
||||
"geometry": [[0,1,1,0,...], ...], // Binary material grid
|
||||
"material": "FR4",
|
||||
"thickness_mm": 0.2
|
||||
}
|
||||
],
|
||||
"vias": [{"x": 3, "y": 5, "radius_mm": 0.15}],
|
||||
"dielectric": "rogers_4003c"
|
||||
},
|
||||
"simulation": {
|
||||
"s_parameters": {
|
||||
"frequencies_ghz": [1.0, 1.1, ..., 40.0],
|
||||
"s11_mag_db": [-5.2, -5.4, ...],
|
||||
"s11_phase_deg": [45.2, 44.8, ...],
|
||||
"s21_mag_db": [-0.3, -0.3, ...]
|
||||
},
|
||||
"field_data": {
|
||||
"type": "near_field",
|
||||
"grid_size": [64, 64],
|
||||
"e_field_magnitude": [[...], ...]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## 4. UI Components and Features
|
||||
|
||||
### 4.1 Observed UI Elements
|
||||
|
||||
Based on page source analysis:
|
||||
|
||||
- **Dark theme** with custom fonts (Rules family -- geometric sans-serif)
|
||||
- **Icon system** ("IconMark" component -- likely a custom RF/EM icon set)
|
||||
- **Responsive design** via Mantine breakpoints
|
||||
- **ToS gate** requiring acceptance before use
|
||||
- **Organization-scoped access** (Auth0 org-based multi-tenancy)
|
||||
|
||||
### 4.2 Likely Feature Set (inferred from product description and tech stack)
|
||||
|
||||
| Feature | Description | UI Component |
|
||||
|---|---|---|
|
||||
| Specification Input | Enter target S-parameters, frequency range, constraints | Form with frequency sweep chart |
|
||||
| Design Canvas | View/edit 2D geometry layers | Interactive grid editor |
|
||||
| S-parameter Viewer | Plot S11/S21/S12/S22 vs frequency | Interactive chart (likely Recharts or D3) |
|
||||
| 3D Field Viewer | Visualize E/H field distributions | Unreal Engine pixel-streamed viewport |
|
||||
| Design History | Browse previous designs and iterations | List/card view with thumbnails |
|
||||
| Compare View | Side-by-side design comparison | Split-pane layout |
|
||||
| Export | Download design files (Gerber, GDSII, S-parameter Touchstone) | Download buttons |
|
||||
|
||||
### 4.3 Agentic Workflow UI
|
||||
|
||||
Atlas RF Studio describes "agentic workflows" that:
|
||||
1. Accept natural-language or parametric specifications
|
||||
2. Generate multiple candidate designs
|
||||
3. Simulate each candidate
|
||||
4. Present ranked results
|
||||
5. Allow iterative refinement
|
||||
|
||||
This suggests an LLM chat interface (translating intent to specs) alongside the technical EM visualization. The pairing of LLM + LFM (Large Field Model) is explicitly described in their architecture.
|
||||
|
||||
|
||||
## 5. Lessons for Our Sensing Server UI
|
||||
|
||||
### 5.1 Architecture Patterns to Adopt
|
||||
|
||||
| Arena Physica Pattern | Application to wifi-densepose sensing-server |
|
||||
|---|---|
|
||||
| Dark theme default | Already appropriate for a sensing/monitoring dashboard |
|
||||
| Next.js + Mantine | Consider for our sensing-server UI (currently Axum + vanilla) |
|
||||
| Auth0 multi-tenancy | Overkill for local deployment; useful for cloud/multi-site |
|
||||
| Unreal Engine 3D | Too heavy; use Three.js/WebGL for 3D pose visualization |
|
||||
| WebSocket push (not polling) | Match our real-time CSI streaming needs |
|
||||
| Feature flags (DevCycle) | Useful for gradual feature rollout |
|
||||
| Datadog RUM | Consider lightweight alternative (e.g., self-hosted analytics) |
|
||||
|
||||
### 5.2 Visualization Approaches
|
||||
|
||||
**What Arena visualizes:**
|
||||
- S-parameters (frequency-domain complex response) -- charts
|
||||
- Electromagnetic field distributions -- 3D heatmaps
|
||||
- Design geometry -- 2D grid with material layers
|
||||
|
||||
**What we need to visualize:**
|
||||
- CSI amplitude/phase across subcarriers -- frequency-domain charts (similar to S-parameters)
|
||||
- Person occupancy heatmap -- 2D/3D voxel grid (similar to field visualization)
|
||||
- Pose skeleton overlay -- 2D/3D joint rendering
|
||||
- Vital signs (HR, BR) -- time-series charts
|
||||
- Node mesh topology -- graph visualization
|
||||
- Signal quality metrics -- dashboard gauges
|
||||
|
||||
**Shared patterns:**
|
||||
- Both need real-time frequency-domain data visualization
|
||||
- Both show spatial field/occupancy distributions
|
||||
- Both benefit from interactive 3D (but at different scales)
|
||||
- Both require low-latency streaming from computation backend
|
||||
|
||||
### 5.3 Data Flow Architecture Comparison
|
||||
|
||||
**Arena Physica:**
|
||||
```
|
||||
Browser (Next.js) -> API (inference) -> Heaviside-0/Marconi-0 -> Unreal Engine -> Pixel Stream -> Browser
|
||||
```
|
||||
|
||||
**wifi-densepose (recommended):**
|
||||
```
|
||||
ESP32 nodes -> sensing-server (Axum) -> WebSocket -> Browser (React/Mantine)
|
||||
|
|
||||
v
|
||||
RuvSense pipeline -> pose/vitals -> WebSocket -> Browser
|
||||
```
|
||||
|
||||
Key difference: Arena renders 3D on the server (Unreal Engine) and streams pixels. We should render 3D on the client (Three.js/WebGL) and stream data, because:
|
||||
- Our 3D scenes are simpler (skeleton + voxels vs. full EM field)
|
||||
- Client-side rendering avoids GPU server costs
|
||||
- Lower latency for real-time sensing feedback
|
||||
- Works offline / on local network
|
||||
|
||||
### 5.4 API Design Lessons
|
||||
|
||||
**Arena's API pattern** (REST + WebSocket):
|
||||
- REST for design submission and retrieval
|
||||
- WebSocket/SSE for live simulation progress and results
|
||||
- Auth0 JWT for authentication
|
||||
- Organization-scoped resources
|
||||
|
||||
**Recommended for sensing-server:**
|
||||
- REST endpoints for configuration, history, calibration
|
||||
- WebSocket for real-time CSI, pose, and vitals streaming
|
||||
- Optional: SSE as fallback for environments where WebSocket is blocked
|
||||
- API key or local-only access (no OAuth needed for embedded deployment)
|
||||
|
||||
**Proposed WebSocket protocol for sensing-server:**
|
||||
```json
|
||||
// Server -> Client: CSI frame
|
||||
{
|
||||
"type": "csi_frame",
|
||||
"timestamp_us": 1712000000000,
|
||||
"node_id": "esp32-node-1",
|
||||
"subcarriers": 56,
|
||||
"amplitude": [0.45, 0.52, ...],
|
||||
"phase": [-1.23, 0.87, ...]
|
||||
}
|
||||
|
||||
// Server -> Client: Pose update
|
||||
{
|
||||
"type": "pose",
|
||||
"timestamp_us": 1712000000000,
|
||||
"persons": [
|
||||
{
|
||||
"id": 0,
|
||||
"keypoints": [
|
||||
{"name": "nose", "x": 2.3, "y": 1.5, "z": 1.7, "confidence": 0.92},
|
||||
...
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
// Server -> Client: Vitals update
|
||||
{
|
||||
"type": "vitals",
|
||||
"timestamp_us": 1712000000000,
|
||||
"person_id": 0,
|
||||
"heart_rate_bpm": 72.5,
|
||||
"breathing_rate_rpm": 16.2,
|
||||
"presence_score": 0.98
|
||||
}
|
||||
|
||||
// Server -> Client: Occupancy grid
|
||||
{
|
||||
"type": "occupancy",
|
||||
"timestamp_us": 1712000000000,
|
||||
"nx": 8, "ny": 8, "nz": 4,
|
||||
"bounds": [0.0, 0.0, 0.0, 6.0, 6.0, 3.0],
|
||||
"densities": [0.0, 0.0, 0.12, ...]
|
||||
}
|
||||
|
||||
// Client -> Server: Configuration
|
||||
{
|
||||
"type": "config",
|
||||
"action": "set",
|
||||
"key": "tomography.lambda",
|
||||
"value": 0.15
|
||||
}
|
||||
```
|
||||
|
||||
### 5.5 Specific UI Components to Build
|
||||
|
||||
Based on Arena Physica's approach and our sensing needs:
|
||||
|
||||
**Priority 1 (Core Dashboard):**
|
||||
1. **Real-time CSI waterfall** -- Subcarrier amplitude over time, color-mapped (similar to spectrogram)
|
||||
2. **Pose skeleton view** -- 2D/3D rendering of detected keypoints with skeleton connections
|
||||
3. **Node topology map** -- Show ESP32 mesh with RSSI-colored edges
|
||||
4. **Vitals panel** -- Heart rate and breathing rate with time-series charts
|
||||
|
||||
**Priority 2 (Advanced Visualization):**
|
||||
5. **Occupancy heatmap** -- 2D top-down view of tomographic voxel grid
|
||||
6. **Phase coherence indicator** -- Per-link coherence scores (green/yellow/red)
|
||||
7. **Fresnel zone overlay** -- Show first Fresnel zone on room floor plan per link
|
||||
|
||||
**Priority 3 (Configuration/Debug):**
|
||||
8. **Calibration wizard** -- Guide through empty-room calibration for field_model
|
||||
9. **Link quality matrix** -- NxN grid showing per-link signal metrics
|
||||
10. **Raw CSI inspector** -- Select individual link, view amplitude + phase per subcarrier
|
||||
|
||||
|
||||
## 6. Public API Endpoints and Protocols
|
||||
|
||||
### 6.1 Confirmed Endpoints
|
||||
|
||||
| Endpoint | Protocol | Purpose |
|
||||
|---|---|---|
|
||||
| `https://studio.arenaphysica.com` | HTTPS | Main web application (Next.js SSR) |
|
||||
| `https://api.emfm.atlas.arena-ai.com` | HTTPS | Backend API (Auth0 audience) |
|
||||
| `https://52.61.97.121` | HTTPS/WSS | Unreal Engine rendering server |
|
||||
|
||||
### 6.2 Authentication
|
||||
|
||||
- Auth0-based with organization scoping
|
||||
- Custom audience: `https://api.emfm.atlas.arena-ai.com`
|
||||
- Organization: `emfmprod`
|
||||
- Terms of Service required before access
|
||||
|
||||
### 6.3 Feature Flags
|
||||
|
||||
DevCycle SDK integrated for A/B testing and feature gating. This suggests gradual rollout of new capabilities.
|
||||
|
||||
### 6.4 Monitoring
|
||||
|
||||
Datadog RUM (Real User Monitoring) for performance tracking. Session replay (Hotjar) is available but disabled in production.
|
||||
|
||||
### 6.5 What is NOT Publicly Documented
|
||||
|
||||
- REST API endpoints (no public API docs found)
|
||||
- WebSocket message schemas
|
||||
- S-parameter data format
|
||||
- Geometry encoding format
|
||||
- Rate limits or usage quotas
|
||||
- Pricing model
|
||||
|
||||
Arena Physica appears to operate as a closed platform without public API access. The Studio beta is a controlled preview, not an open API.
|
||||
|
||||
|
||||
## 7. Summary of Findings
|
||||
|
||||
### What Arena Physica Is
|
||||
A $30M-funded startup building neural surrogates for electromagnetic simulation. Their AI predicts S-parameters and field distributions 18,000-800,000x faster than traditional solvers. They serve Fortune 500 hardware companies (AMD, Anduril) for RF component design.
|
||||
|
||||
### What Arena Physica Is NOT
|
||||
They are not a WiFi sensing company. They do not do human pose estimation, CSI analysis, or IoT sensing. The relevance to our project is purely methodological.
|
||||
|
||||
### Key Technical Takeaways for wifi-densepose
|
||||
|
||||
1. **Neural surrogates for Maxwell's equations work** -- Arena proves that training on millions of simulation examples produces models accurate to < 1 dB MAE running in milliseconds. We could apply the same approach to CSI prediction.
|
||||
|
||||
2. **Inverse design via conditional diffusion** -- Marconi-0's approach (generating geometry from target specs) parallels our inverse problem (generating pose from CSI). Conditional diffusion is a viable architecture.
|
||||
|
||||
3. **Bidirectional search** -- The generate-evaluate-refine loop is more effective than direct inversion. For real-time sensing, the evaluator (forward model) must be fast.
|
||||
|
||||
4. **Domain-specific models beat general LLMs** -- For electromagnetic tasks, specialized architectures substantially outperform GPT-4 / Claude. This validates our approach of building specialized CSI processing rather than relying on general-purpose models.
|
||||
|
||||
5. **Studio UI is Next.js + Mantine + Unreal Engine** -- A modern stack, but the Unreal Engine component is overkill for our visualization needs. Three.js/WebGL on the client is more appropriate for our real-time sensing dashboard.
|
||||
|
||||
6. **WebSocket push over polling** -- Confirmed by their `POLL_FOR_MESSAGES: false` configuration. Our sensing-server should use WebSocket push for real-time data streaming.
|
||||
|
||||
|
||||
## References
|
||||
|
||||
- Arena Physica Homepage: https://www.arenaphysica.com/
|
||||
- Atlas RF Studio Beta: https://studio.arenaphysica.com/
|
||||
- Introducing Atlas RF Studio (publication): https://www.arenaphysica.com/publications/rf-studio
|
||||
- Electromagnetism Secretly Runs the World (Not Boring essay): https://www.notboring.co/p/electromagnetism-secretly-runs-the
|
||||
- Arena Launches Atlas (press release): https://www.prnewswire.com/news-releases/arena-launches-atlas-to-accelerate-humanitys-rate-of-hardware-innovation-302423412.html
|
||||
- Arena AI raises $30M (SiliconANGLE): https://siliconangle.com/2025/04/08/arena-ai-raises-30m-accelerate-innovation-hardware-testing-atlas/
|
||||
- Artificial Intuition (CDFAM presentation): https://www.designforam.com/p/artificial-intuition-building-an
|
||||
- Pratap Ranade LinkedIn announcement: https://www.linkedin.com/posts/pratap-ranade-7272829_today-im-excited-to-introduce-arena-physica-activity-7442204772725723137-RRtE
|
||||
- Mantine UI: https://mantine.dev/
|
||||
- Unreal Engine Pixel Streaming: https://dev.epicgames.com/documentation/en-us/unreal-engine/remote-control-api-websocket-reference-for-unreal-engine
|
||||
@@ -0,0 +1,141 @@
|
||||
# Deep Analysis: arXiv 2505.15472 -- PhysicsArena
|
||||
|
||||
**Date:** 2026-04-02
|
||||
**Analyst:** GOAP Planning Agent
|
||||
**Relevance to wifi-densepose:** Indirect (physics reasoning benchmark, not WiFi sensing)
|
||||
|
||||
---
|
||||
|
||||
## 1. Paper Identity
|
||||
|
||||
- **Title:** PhysicsArena: The First Multimodal Physics Reasoning Benchmark Exploring Variable, Process, and Solution Dimensions
|
||||
- **Authors:** Song Dai, Yibo Yan, Jiamin Su, Dongfang Zihao, Yubo Gao, Yonghua Hei, Jungang Li, Junyan Zhang, Sicheng Tao, Zhuoran Gao, Xuming Hu
|
||||
- **Submitted:** 2025-05-21, revised 2025-05-22
|
||||
- **Category:** cs.CL (Computation and Language)
|
||||
- **arXiv ID:** 2505.15472v2
|
||||
|
||||
## 2. Core Contribution
|
||||
|
||||
PhysicsArena introduces a multimodal benchmark for evaluating how Large Language Models (MLLMs) reason about physics problems. The benchmark assesses three dimensions:
|
||||
|
||||
1. **Variable Identification** -- Can the model correctly identify physical variables from multimodal inputs (diagrams, text, equations)?
|
||||
2. **Physical Process Formulation** -- Can the model select and chain the correct physical laws and processes?
|
||||
3. **Solution Derivation** -- Can the model produce correct numerical/symbolic solutions?
|
||||
|
||||
This is the first benchmark to decompose physics reasoning into these three granular dimensions rather than only evaluating final answers.
|
||||
|
||||
## 3. Technical Approach
|
||||
|
||||
### 3.1 Benchmark Structure
|
||||
|
||||
The benchmark presents physics problems with multimodal inputs (text descriptions accompanied by diagrams, graphs, and physical setups). Problems span classical mechanics, electromagnetism, thermodynamics, optics, and modern physics.
|
||||
|
||||
### 3.2 Evaluation Protocol
|
||||
|
||||
Unlike prior benchmarks that score only final answers, PhysicsArena evaluates intermediate reasoning:
|
||||
|
||||
- **Variable extraction accuracy:** Does the model identify all relevant physical quantities (mass, velocity, charge, field strength, etc.)?
|
||||
- **Process correctness:** Does the model apply the right sequence of physical laws (Newton's laws, Maxwell's equations, conservation laws)?
|
||||
- **Solution accuracy:** Does the final numerical answer match the ground truth within tolerance?
|
||||
|
||||
### 3.3 Key Finding
|
||||
|
||||
Current MLLMs (GPT-4V, Claude, Gemini) perform significantly worse on variable identification and process formulation than on final solution derivation when provided with correct intermediate steps. This reveals that models often arrive at correct answers through pattern matching rather than genuine physics reasoning.
|
||||
|
||||
## 4. Relevance to WiFi-DensePose
|
||||
|
||||
### 4.1 Direct Relevance: Low
|
||||
|
||||
This paper is not about WiFi sensing, CSI processing, pose estimation, or edge deployment. It benchmarks LLM reasoning about physics problems.
|
||||
|
||||
### 4.2 Indirect Relevance: Moderate
|
||||
|
||||
Several concepts transfer to our domain:
|
||||
|
||||
#### 4.2.1 Physics-Informed Reasoning for Signal Processing
|
||||
|
||||
The paper's decomposition of physics reasoning into (variables, process, solution) maps onto WiFi sensing:
|
||||
|
||||
| PhysicsArena Dimension | WiFi-DensePose Analog |
|
||||
|------------------------|----------------------|
|
||||
| Variable identification | CSI feature extraction (amplitude, phase, subcarrier indices, antenna config) |
|
||||
| Process formulation | Signal processing pipeline selection (phase alignment, coherence gating, multiband fusion) |
|
||||
| Solution derivation | Pose/activity estimation output |
|
||||
|
||||
This suggests a potential architecture where intermediate representations are explicitly supervised -- not just end-to-end loss on final pose, but also losses on intermediate physical quantities (estimated path lengths, Doppler shifts, angle-of-arrival).
|
||||
|
||||
#### 4.2.2 Multimodal Grounding
|
||||
|
||||
PhysicsArena's core challenge is grounding abstract reasoning in physical reality from multimodal inputs. WiFi-DensePose faces the same challenge: grounding neural network predictions in the actual physics of electromagnetic wave propagation through space containing human bodies.
|
||||
|
||||
#### 4.2.3 Decomposed Evaluation
|
||||
|
||||
The three-dimension evaluation framework suggests we should evaluate our pipeline at multiple stages:
|
||||
|
||||
1. **CSI quality metrics** (SNR, coherence, phase stability) -- analogous to variable identification
|
||||
2. **Feature extraction quality** (does the modality translator preserve physically meaningful information?) -- analogous to process formulation
|
||||
3. **Pose accuracy** (PCK@50, MPJPE) -- analogous to solution derivation
|
||||
|
||||
This would help diagnose whether failures in pose estimation originate from poor CSI capture, lossy feature translation, or incorrect pose regression.
|
||||
|
||||
### 4.3 Transferable Insight: Intermediate Supervision
|
||||
|
||||
The paper's key insight -- that evaluating only final outputs masks fundamental reasoning failures -- argues for adding intermediate supervision signals to the wifi-densepose training pipeline:
|
||||
|
||||
```
|
||||
L_total = lambda_pose * L_pose
|
||||
+ lambda_physics * L_physics_consistency
|
||||
+ lambda_intermediate * L_intermediate_features
|
||||
```
|
||||
|
||||
Where `L_physics_consistency` penalizes predictions that violate known electromagnetic propagation physics (e.g., predicted person positions that are inconsistent with observed CSI phase relationships).
|
||||
|
||||
## 5. Applicable Techniques for Implementation Plan
|
||||
|
||||
### 5.1 Physics-Constrained Loss Functions
|
||||
|
||||
Add a physics consistency loss that enforces:
|
||||
|
||||
- **Fresnel zone consistency:** Predicted body positions must be consistent with the Fresnel zones that would produce the observed CSI perturbations
|
||||
- **Multipath geometry:** The number of strong multipath components should be consistent with the predicted scene geometry
|
||||
- **Doppler-velocity consistency:** If temporal CSI changes indicate Doppler shift, the predicted keypoint velocities must match
|
||||
|
||||
### 5.2 Hierarchical Evaluation Pipeline
|
||||
|
||||
Implement three-stage evaluation matching PhysicsArena's decomposition:
|
||||
|
||||
```rust
|
||||
pub struct HierarchicalEvaluation {
|
||||
/// Stage 1: CSI quality assessment
|
||||
pub csi_quality: CsiQualityMetrics,
|
||||
/// Stage 2: Feature translation fidelity
|
||||
pub translation_fidelity: TranslationMetrics,
|
||||
/// Stage 3: Pose estimation accuracy
|
||||
pub pose_accuracy: PoseMetrics,
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 Structured Intermediate Representations
|
||||
|
||||
Rather than a single encoder-decoder, structure the network to produce interpretable intermediate outputs:
|
||||
|
||||
```
|
||||
CSI input -> [Physics Encoder] -> physical_features (AoA, ToF, Doppler)
|
||||
-> [Geometry Decoder] -> spatial_occupancy_map
|
||||
-> [Pose Regressor] -> keypoint_coordinates
|
||||
```
|
||||
|
||||
Each intermediate output can be supervised independently where ground truth is available.
|
||||
|
||||
## 6. Conclusion
|
||||
|
||||
While arXiv 2505.15472 is not directly about WiFi sensing, its framework for decomposing physics reasoning into interpretable stages provides a valuable architectural pattern. The key takeaway for wifi-densepose is: **do not rely solely on end-to-end training; add intermediate physics-grounded supervision signals to improve robustness and interpretability.**
|
||||
|
||||
This aligns with the existing RuvSense architecture which already has explicit stages (multiband fusion, phase alignment, coherence scoring, coherence gating, pose tracking) -- the paper's framework validates this design choice and argues for adding supervision at each stage boundary.
|
||||
|
||||
## 7. Cross-References
|
||||
|
||||
- **Arena Physica (arena-physica-analysis.md):** Their thesis that "fields are the fundamental quantities" reinforces the physics-first approach recommended here. Training on electromagnetic field distributions rather than end-to-end CSI-to-pose would constitute the WiFi sensing analog of PhysicsArena's decomposed evaluation.
|
||||
- **WiFlow (sota-wifi-sensing-2025.md, Section 1.1):** WiFlow's bone constraint loss is a concrete implementation of physics-informed intermediate supervision -- the skeleton must obey anatomical constraints at every prediction step.
|
||||
- **MultiFormer (sota-wifi-sensing-2025.md, Section 1.2):** MultiFormer's dual-token (time + frequency) tokenization is analogous to PhysicsArena's variable identification -- it explicitly separates the physical dimensions of the CSI measurement before reasoning about them.
|
||||
- **Implementation plan (implementation-plan.md):** The hierarchical evaluation pipeline in Section 5.2 directly implements the three-stage evaluation framework recommended here.
|
||||
@@ -0,0 +1,615 @@
|
||||
# Maxwell's Equations in WiFi/RF Sensing
|
||||
|
||||
Research document for wifi-densepose project.
|
||||
Date: 2026-04-02
|
||||
|
||||
---
|
||||
|
||||
## 1. Maxwell's Equations and CSI Extraction
|
||||
|
||||
### 1.1 Foundational Electromagnetic Theory
|
||||
|
||||
All WiFi-based sensing ultimately derives from Maxwell's four partial differential equations governing electromagnetic field behavior:
|
||||
|
||||
```
|
||||
(1) Gauss's Law (Electric): nabla . E = rho / epsilon_0
|
||||
(2) Gauss's Law (Magnetic): nabla . B = 0
|
||||
(3) Faraday's Law: nabla x E = -dB/dt
|
||||
(4) Ampere-Maxwell Law: nabla x B = mu_0 * J + mu_0 * epsilon_0 * dE/dt
|
||||
```
|
||||
|
||||
In free space with no charges or currents (the indoor propagation case), these simplify to the wave equation:
|
||||
|
||||
```
|
||||
nabla^2 E - mu_0 * epsilon_0 * d^2 E / dt^2 = 0
|
||||
```
|
||||
|
||||
yielding plane wave solutions `E(r, t) = E_0 * exp(j(k . r - omega * t))` where `k = 2*pi / lambda` is the wavenumber. At 2.4 GHz WiFi, `lambda ~ 12.5 cm`; at 5 GHz, `lambda ~ 6 cm`.
|
||||
|
||||
### 1.2 From Maxwell to Channel State Information
|
||||
|
||||
Channel State Information (CSI) is the frequency-domain representation of the wireless channel's impulse response. The derivation from Maxwell's equations proceeds through several simplification layers:
|
||||
|
||||
**Layer 1: Full Maxwell's equations** -- Exact but computationally intractable for room-scale environments at GHz frequencies.
|
||||
|
||||
**Layer 2: High-frequency ray optics (Geometrical Optics / Uniform Theory of Diffraction)** -- When object dimensions >> lambda (walls, furniture), Maxwell's equations reduce to ray tracing. Each ray follows Snell's law at interfaces, with Fresnel reflection/transmission coefficients computed from the dielectric contrast.
|
||||
|
||||
**Layer 3: Multipath channel model** -- The channel impulse response aggregates all propagation paths:
|
||||
|
||||
```
|
||||
h(t) = sum_{n=1}^{N} alpha_n * exp(-j * phi_n) * delta(t - tau_n)
|
||||
```
|
||||
|
||||
where for each path n:
|
||||
- `alpha_n` = complex attenuation (from free-space path loss, reflection, diffraction)
|
||||
- `phi_n = 2*pi*f*tau_n` = phase shift
|
||||
- `tau_n = d_n / c` = propagation delay (distance / speed of light)
|
||||
|
||||
**Layer 4: Channel Frequency Response (CFR) = CSI** -- The Fourier transform of h(t):
|
||||
|
||||
```
|
||||
H(f_k) = sum_{n=1}^{N} alpha_n * exp(-j * 2*pi * f_k * tau_n)
|
||||
```
|
||||
|
||||
Each OFDM subcarrier k at frequency f_k provides one complex CSI measurement:
|
||||
|
||||
```
|
||||
H(f_k) = |H(f_k)| * exp(j * angle(H(f_k)))
|
||||
```
|
||||
|
||||
With 802.11n/ac providing 56-256 subcarriers and 802.11ax up to 512 subcarriers across 160 MHz bandwidth, CSI captures a frequency-sampled version of the channel's multipath structure.
|
||||
|
||||
**Key insight for sensing**: When a human moves in the environment, paths reflecting off the body change their `alpha_n`, `tau_n`, and `phi_n`, modulating the CSI. The sensing problem is to invert this relationship -- recover body state from CSI changes.
|
||||
|
||||
### 1.3 The Two CSI Models
|
||||
|
||||
The Tsinghua WiFi Sensing Tutorial (tns.thss.tsinghua.edu.cn) identifies two mainstream models:
|
||||
|
||||
**Ray-Tracing Model**: Establishes explicit geometric relationships between signal paths and CSI. The received signal is:
|
||||
|
||||
```
|
||||
V = sum_{n=1}^{N} |V_n| * exp(-j * phi_n)
|
||||
```
|
||||
|
||||
This model enables extraction of geometric parameters (distances, reflection points, angles of arrival) from CSI data. It underpins localization and tracking applications.
|
||||
|
||||
**Scattering Model**: Decomposes CSI into static and dynamic contributions:
|
||||
|
||||
```
|
||||
H(f,t) = sum_{o in Omega_s} H_o(f,t) + sum_{p in Omega_d} H_p(f,t)
|
||||
```
|
||||
|
||||
Dynamic scatterers (moving bodies) contribute through angular integration:
|
||||
|
||||
```
|
||||
H_p(f,t) = integral_0^{2pi} integral_0^{pi} h_p(alpha, beta, f, t) * exp(-j*k*v_p*cos(alpha)*t) d_alpha d_beta
|
||||
```
|
||||
|
||||
The scattering model yields the CSI autocorrelation:
|
||||
|
||||
```
|
||||
rho_H(f, tau) ~ sinc(k * v * tau)
|
||||
```
|
||||
|
||||
enabling speed extraction from autocorrelation peak analysis:
|
||||
|
||||
```
|
||||
v = x_0 * lambda / (2 * pi * tau_0)
|
||||
```
|
||||
|
||||
where `x_0` is the first sinc extremum location and `tau_0` is the corresponding time lag.
|
||||
|
||||
### 1.4 Practical Simplifications Used in WiFi Sensing
|
||||
|
||||
| Approximation | Physical Basis | Used When | Accuracy |
|
||||
|---|---|---|---|
|
||||
| Ray tracing (GO/UTD) | High-frequency limit of Maxwell | Objects >> lambda | Good for LOS + major reflections |
|
||||
| Fresnel zone model | Wave diffraction | Target near TX-RX line | Excellent for presence/respiration |
|
||||
| Born approximation | Weak scattering (small perturbation) | Low-contrast objects | Breaks down for human body |
|
||||
| Rytov approximation | Phase perturbation expansion | Moderate scattering | Better for lossy media |
|
||||
| Free-space path loss | 1/r^2 power decay | Coarse attenuation models | Adequate for RSSI-based sensing |
|
||||
|
||||
**Relevance to wifi-densepose**: Our `field_model.rs` implements the eigenstructure approach (Layer 2.5 -- between full ray tracing and statistical models), decomposing the channel covariance via SVD to separate environmental modes from body perturbation. Our `tomography.rs` implements the voxel-based inverse at Layer 3 using L1-regularized least squares.
|
||||
|
||||
|
||||
## 2. Physics-Informed Neural Networks (PINNs) for RF Sensing
|
||||
|
||||
### 2.1 PINN Architecture for Wireless Channels
|
||||
|
||||
Physics-Informed Neural Networks embed physical laws as constraints in the loss function or network architecture. For RF sensing, PINNs encode electromagnetic propagation principles:
|
||||
|
||||
**Standard PINN loss for RF propagation:**
|
||||
|
||||
```
|
||||
L_total = L_data + lambda_physics * L_physics + lambda_boundary * L_boundary
|
||||
|
||||
where:
|
||||
L_data = (1/N) * sum |H_pred(f_k) - H_meas(f_k)|^2 (CSI measurement fit)
|
||||
L_physics = (1/M) * sum |nabla^2 E + k^2 * E|^2 (Helmholtz equation residual)
|
||||
L_boundary = (1/B) * sum |E_pred - E_bc|^2 (boundary conditions)
|
||||
```
|
||||
|
||||
The Helmholtz equation `nabla^2 E + k^2 * n^2(r) * E = 0` (time-harmonic Maxwell) constrains the solution space, where `n(r)` is the spatially varying refractive index.
|
||||
|
||||
### 2.2 Key Papers and Approaches
|
||||
|
||||
**PINN + GNN for RF Map Construction** (arXiv 2507.22513):
|
||||
- Combines Physics-Informed Neural Networks with Graph Neural Networks
|
||||
- Physical constraints from EM propagation laws guide learning
|
||||
- Parameterizes multipath signals into received power, delay, and angle of arrival
|
||||
- Integrates spatial dependencies for accurate prediction
|
||||
|
||||
**PINN for Wireless Channel Estimation** (NeurIPS 2025, OpenReview r3plaU6DvW):
|
||||
- Synergistically combines model-based channel estimation with deep network
|
||||
- Exploits prior information about environmental propagation
|
||||
- Critical for next-gen wireless systems: precoding, interference reduction, sensing
|
||||
|
||||
**ReVeal: High-Fidelity Radio Propagation** (DySPAN 2025):
|
||||
- Physics-informed approach for radio environment mapping
|
||||
- Achieves high fidelity with limited measurement data
|
||||
|
||||
**Physics-Informed Generative Model for Passive RF Sensing** (arXiv 2310.04173, Savazzi et al.):
|
||||
- Variational Auto-Encoder integrating EM body diffraction
|
||||
- Forward model: predicts CSI perturbation from body position/pose
|
||||
- Validated against classical diffraction-based EM tools AND real RF measurements
|
||||
- Enables real-time processing where traditional EM is too slow
|
||||
|
||||
**Multi-Modal Foundational Model** (arXiv 2602.04016, February 2026):
|
||||
- Foundation model for AI-driven physical-layer wireless systems
|
||||
- Physics-guided pretraining grounded in EM propagation principles
|
||||
- Treats wireless as inherently multimodal physical system
|
||||
|
||||
**Generative AI for Wireless Sensing** (arXiv 2509.15258, September 2025):
|
||||
- Physics-informed diffusion models for data augmentation
|
||||
- Channel prediction and environment modeling
|
||||
- Conditional mechanisms constrained by EM laws
|
||||
|
||||
### 2.3 PINN Architecture for CSI-Based Sensing
|
||||
|
||||
```
|
||||
Algorithm: Physics-Informed CSI Sensing Network
|
||||
|
||||
Input: CSI tensor H[time, subcarrier, antenna] of shape (T, K, M)
|
||||
Output: Body state estimate (pose, position, or occupancy)
|
||||
|
||||
1. PREPROCESSING (physics-guided):
|
||||
a. Remove carrier frequency offset (CFO): H_clean = H * exp(-j*2*pi*delta_f*t)
|
||||
b. Conjugate multiply across antenna pairs to cancel common phase noise
|
||||
c. Compute CSI-ratio: H_ratio(f,t) = H_dynamic(f,t) / H_static(f,t)
|
||||
|
||||
2. PHYSICS ENCODER:
|
||||
a. Embed Fresnel zone geometry as positional encoding
|
||||
b. Apply multi-head attention with frequency-aware kernels
|
||||
c. Enforce causality: attention mask respects propagation delay ordering
|
||||
|
||||
3. PHYSICS-CONSTRAINED DECODER:
|
||||
a. Predict body state x_hat
|
||||
b. Forward-simulate expected CSI from x_hat using ray-tracing differentiable renderer
|
||||
c. Compute physics loss: L_phys = ||H_simulated(x_hat) - H_measured||^2
|
||||
|
||||
4. TRAINING LOSS:
|
||||
L = L_pose_supervision + alpha * L_phys + beta * L_temporal_smoothness
|
||||
```
|
||||
|
||||
### 2.4 Relevance to wifi-densepose
|
||||
|
||||
Our RuvSense pipeline already implements physics-guided preprocessing (phase alignment, coherence gating, Fresnel zone awareness). The next step would be to:
|
||||
|
||||
1. Add a differentiable ray-tracing forward model as a physics constraint during NN training
|
||||
2. Use the field model eigenstructure (from `field_model.rs`) as an informed prior
|
||||
3. Embed Fresnel zone geometry from link topology as architectural bias
|
||||
|
||||
|
||||
## 3. Inverse Electromagnetic Scattering for Body Reconstruction
|
||||
|
||||
### 3.1 The Inverse Problem
|
||||
|
||||
The forward problem: given a known body position/shape and room geometry, predict the CSI.
|
||||
|
||||
```
|
||||
Forward: body_state -> Maxwell/ray-tracing -> H(f,t) [well-posed]
|
||||
Inverse: H(f,t) -> ??? -> body_state [ill-posed]
|
||||
```
|
||||
|
||||
WiFi sensing is fundamentally an inverse scattering problem. A WiFi antenna receives signal as 1D amplitude/phase -- the spatial information of the 3D scene is collapsed to a single CSI complex number per subcarrier per antenna pair. Reconstructing fine-grained spatial information from this compressed observation is severely ill-posed.
|
||||
|
||||
### 3.2 Linearized Inverse Scattering: Born and Rytov Approximations
|
||||
|
||||
**Helmholtz equation with scatterer:**
|
||||
|
||||
```
|
||||
nabla^2 E(r) + k^2 * (1 + O(r)) * E(r) = 0
|
||||
```
|
||||
|
||||
where `O(r) = epsilon_r(r) - 1` is the object function (dielectric contrast of the body relative to free space).
|
||||
|
||||
**Born approximation** (first-order): Assumes the field inside the scatterer equals the incident field:
|
||||
|
||||
```
|
||||
E_scattered(r) ~ k^2 * integral O(r') * E_incident(r') * G(r, r') dr'
|
||||
```
|
||||
|
||||
where `G(r, r')` is the free-space Green's function. This is valid when `O(r)` is small and the object is electrically small. For the human body at 2.4 GHz (`epsilon_r ~ 40-60` for muscle tissue), the Born approximation is grossly violated.
|
||||
|
||||
**Rytov approximation**: Expands the complex phase rather than the field:
|
||||
|
||||
```
|
||||
E_total(r) = E_incident(r) * exp(psi(r))
|
||||
|
||||
psi(r) ~ (k^2 / E_incident(r)) * integral O(r') * E_incident(r') * G(r, r') dr'
|
||||
```
|
||||
|
||||
The Rytov approximation handles larger phase accumulation than Born but still assumes weak scattering. It works better for lossy media where absorption limits multiple scattering.
|
||||
|
||||
**Extended Phaseless Rytov Approximation (xPRA-LM)** (Dubey et al., arXiv 2110.03211):
|
||||
- First linear phaseless inverse scattering approximation with large validity range
|
||||
- Demonstrated with 2.4 GHz WiFi nodes for indoor imaging
|
||||
- Handles objects with `epsilon_r` up to 15+j1.5 (20x wavelength size)
|
||||
- At `epsilon_r = 77+j7` (water/tissue), shape reconstruction still accurate
|
||||
|
||||
### 3.3 Iterative Nonlinear Methods
|
||||
|
||||
For high-contrast scatterers like the human body, iterative methods are required:
|
||||
|
||||
**Distorted Born Iterative Method (DBIM):**
|
||||
|
||||
```
|
||||
Algorithm: DBIM for WiFi Body Imaging
|
||||
|
||||
Input: Measured scattered field E_s at receiver locations
|
||||
Output: Object function O(r) (dielectric map of scene)
|
||||
|
||||
1. Initialize: O_0(r) = 0 (empty room)
|
||||
2. For iteration i = 0, 1, 2, ...:
|
||||
a. Solve forward problem: compute total field E_i(r) in medium with O_i(r)
|
||||
b. Compute Green's function G_i(r, r') for medium O_i(r)
|
||||
c. Linearize: delta_E_s = K_i * delta_O (Frechet derivative)
|
||||
d. Solve: delta_O = K_i^+ * (E_s_measured - E_s_computed(O_i))
|
||||
e. Update: O_{i+1} = O_i + delta_O
|
||||
f. Check convergence: ||E_s_measured - E_s_computed(O_{i+1})|| < epsilon
|
||||
```
|
||||
|
||||
**Challenges for WiFi sensing:**
|
||||
- WiFi provides sparse spatial sampling (few antenna pairs vs. full aperture)
|
||||
- Phase is often unavailable (RSSI-only) or corrupted by hardware imperfections
|
||||
- Real-time requirement conflicts with iterative forward solves
|
||||
- Human body is a strong, moving scatterer
|
||||
|
||||
### 3.4 Radio Tomographic Imaging (RTI)
|
||||
|
||||
RTI (Wilson & Patwari, 2010) simplifies the inverse scattering problem by:
|
||||
1. Using only RSS (received signal strength) -- phaseless
|
||||
2. Assuming a voxelized scene with additive attenuation model
|
||||
3. Linearizing: measured attenuation = sum of voxel attenuations along path
|
||||
|
||||
**Forward model:**
|
||||
|
||||
```
|
||||
y = W * x + n
|
||||
|
||||
where:
|
||||
y = [y_1, ..., y_L]^T attenuation measurements (L links)
|
||||
x = [x_1, ..., x_V]^T voxel occupancy values (V voxels)
|
||||
W = [w_{l,v}] weight matrix (link-voxel intersection)
|
||||
n = measurement noise
|
||||
```
|
||||
|
||||
**Weight model (elliptical):**
|
||||
|
||||
```
|
||||
w_{l,v} = { 1 / sqrt(d_l) if d_{l,v}^tx + d_{l,v}^rx < d_l + lambda_w
|
||||
{ 0 otherwise
|
||||
|
||||
where:
|
||||
d_l = distance between TX_l and RX_l
|
||||
d_{l,v}^tx = distance from TX_l to voxel v center
|
||||
d_{l,v}^rx = distance from RX_l to voxel v center
|
||||
lambda_w = excess path length parameter (typically ~lambda/4)
|
||||
```
|
||||
|
||||
**Inverse solution (Tikhonov-regularized):**
|
||||
|
||||
```
|
||||
x_hat = (W^T W + alpha * C^{-1})^{-1} * W^T * y
|
||||
```
|
||||
|
||||
where `C` is the spatial covariance matrix and `alpha` controls regularization.
|
||||
|
||||
**Our implementation** (`tomography.rs`) uses ISTA (Iterative Shrinkage-Thresholding Algorithm) with L1 regularization for sparsity:
|
||||
|
||||
```
|
||||
Algorithm: ISTA for RF Tomography (as in tomography.rs)
|
||||
|
||||
Input: Weight matrix W, observations y, lambda (L1 weight)
|
||||
Output: Sparse voxel densities x
|
||||
|
||||
1. Initialize x = 0
|
||||
2. step_size = 1 / ||W^T * W||_spectral
|
||||
3. For iter = 1 to max_iterations:
|
||||
a. gradient = W^T * (W * x - y)
|
||||
b. x_candidate = x - step_size * gradient
|
||||
c. x = soft_threshold(x_candidate, lambda * step_size)
|
||||
where soft_threshold(z, t) = sign(z) * max(|z| - t, 0)
|
||||
d. residual = ||W * x - y||
|
||||
e. if residual < tolerance: break
|
||||
```
|
||||
|
||||
### 3.5 Reconciling RTI with Inverse Scattering
|
||||
|
||||
Dubey, Li & Murch (arXiv 2311.09633) reconciled empirical RTI with formal inverse scattering theory:
|
||||
- RTI's additive attenuation model corresponds to a first-order Born approximation of the scattered field amplitude
|
||||
- Their enhanced method reconstructs both shape AND material properties
|
||||
- Validated at 2.4 GHz with WiFi transceivers indoors
|
||||
|
||||
### 3.6 State-of-the-Art: Deep Learning Approaches
|
||||
|
||||
**DensePose From WiFi** (Geng, Huang, De la Torre, arXiv 2301.00250, CMU):
|
||||
- Maps WiFi CSI amplitude+phase to UV coordinates across 24 body regions
|
||||
- Uses 3 TX + 3 RX antennas, 56 subcarriers per link
|
||||
- Teacher-student training: camera-based DensePose provides labels
|
||||
- Performance comparable to image-based approaches
|
||||
- Works through walls and in darkness
|
||||
|
||||
**RF-Pose** (Zhao et al., CVPR 2018, MIT CSAIL):
|
||||
- Through-wall human pose estimation using radio signals
|
||||
- Cross-modal supervision: vision model trains RF model
|
||||
- Generalizes to through-wall scenarios with no through-wall training data
|
||||
|
||||
**Person-in-WiFi** (Wang et al., ICCV 2019, CMU):
|
||||
- End-to-end body segmentation and pose from WiFi
|
||||
- Standard 802.11n signals, off-the-shelf hardware
|
||||
|
||||
**3D WiFi Pose Estimation** (arXiv 2204.07878):
|
||||
- Free-form and moving activities
|
||||
- 3D joint position estimation from CSI
|
||||
|
||||
**HoloCSI** (2025-2026):
|
||||
- Holographic tomography pipeline coupling physics-guided projection with adaptive top-k sparse transformer
|
||||
- Preprocesses: CFO rectification, Doppler compensation, antenna-pair normalization
|
||||
- Sparse multi-head attention prunes low-magnitude query-key pairs (quadratic -> near-linear complexity)
|
||||
- Results: +2.9 dB PSNR, +3.6% SSIM, +12.4% mesh IoU vs baselines
|
||||
- 25 fps on RTX-4070-mobile at 5% sparsity; 7 fps on Raspberry Pi 5 with attention-GRU variant
|
||||
|
||||
|
||||
## 4. Computational Electromagnetics for WiFi Sensing
|
||||
|
||||
### 4.1 FDTD (Finite-Difference Time-Domain)
|
||||
|
||||
FDTD discretizes Maxwell's curl equations on a Yee grid and marches forward in time:
|
||||
|
||||
```
|
||||
Algorithm: FDTD Update (2D TM mode, simplified)
|
||||
|
||||
Grid: dx = dy = lambda/20 (minimum 10 cells per wavelength)
|
||||
Time step: dt = dx / (c * sqrt(2)) [Courant condition]
|
||||
|
||||
For each time step n:
|
||||
1. Update H fields:
|
||||
H_z^{n+1/2}(i,j) = H_z^{n-1/2}(i,j) + (dt/mu_0) * [
|
||||
(E_x^n(i,j+1) - E_x^n(i,j)) / dy -
|
||||
(E_y^n(i+1,j) - E_y^n(i,j)) / dx
|
||||
]
|
||||
|
||||
2. Update E fields:
|
||||
E_x^{n+1}(i,j) = E_x^n(i,j) + (dt / epsilon(i,j)) * [
|
||||
(H_z^{n+1/2}(i,j) - H_z^{n+1/2}(i,j-1)) / dy
|
||||
]
|
||||
```
|
||||
|
||||
**For WiFi at 2.4 GHz:**
|
||||
- Wavelength: 12.5 cm
|
||||
- Grid cell: ~6 mm (20 cells/lambda)
|
||||
- Room 6m x 6m x 3m: 1000 x 1000 x 500 = 500M cells
|
||||
- Memory: ~24 GB (6 field components * 4 bytes * 500M)
|
||||
- Time steps: ~10,000 for steady state
|
||||
|
||||
**Key references for WiFi FDTD:**
|
||||
- Lauer & Ertel (2003), "Using Large-Scale FDTD for Indoor WLAN" -- Full FDTD at 2.45 GHz in office environments
|
||||
- Lui et al. (2018), "Human Body Shadowing" -- FDTD human body model for ray-tracing calibration (Hindawi IJAP 9084830)
|
||||
- Martinez-Gonzalez et al. (2008), "FDTD Assessment Human Exposure WiFi/Bluetooth" -- SAR computation with anatomical body models
|
||||
|
||||
**Practical limitations**: FDTD is too slow for real-time sensing but valuable for:
|
||||
- Generating training data for neural networks
|
||||
- Validating approximate models
|
||||
- Understanding near-field body-wave interaction
|
||||
|
||||
### 4.2 Method of Moments (MoM)
|
||||
|
||||
MoM converts Maxwell's integral equations into matrix equations by expanding fields in basis functions:
|
||||
|
||||
```
|
||||
[Z] * [I] = [V]
|
||||
|
||||
where:
|
||||
Z_{mn} = integral integral G(r_m, r_n) * f_m(r) * f_n(r') dS dS'
|
||||
I_n = unknown current coefficients
|
||||
V_m = incident field excitation
|
||||
```
|
||||
|
||||
**Application**: MoM excels for antenna analysis and is used to model WiFi antenna patterns. Less practical for full room simulation due to O(N^2) memory and O(N^3) solve time.
|
||||
|
||||
### 4.3 FEM (Finite Element Method)
|
||||
|
||||
FEM handles complex geometries and material interfaces more naturally than FDTD:
|
||||
|
||||
```
|
||||
Weak form of Helmholtz equation:
|
||||
integral nabla x E_test . (1/mu_r * nabla x E) dV - k_0^2 * integral E_test . epsilon_r * E dV
|
||||
= -j * omega * integral E_test . J_s dV
|
||||
```
|
||||
|
||||
**Application**: HFSS (Ansys) and COMSOL use FEM for electromagnetic simulation. Arena Physica's Heaviside-0 model was trained against such commercial FEM solvers.
|
||||
|
||||
### 4.4 Comparison for WiFi Sensing Applications
|
||||
|
||||
| Method | Speed | Accuracy | Body Modeling | Room Scale | Real-Time |
|
||||
|---|---|---|---|---|---|
|
||||
| FDTD | Hours | Full-wave exact | Excellent | Feasible (GPU) | No |
|
||||
| MoM | Hours | Exact for surfaces | Good (surface) | Impractical | No |
|
||||
| FEM | Hours | Exact | Excellent | Feasible | No |
|
||||
| Ray tracing | Seconds | GO/UTD approximation | Coarse | Easy | Near real-time |
|
||||
| RTI (ISTA) | Milliseconds | Linear approximation | Voxelized | Easy | Yes |
|
||||
| Neural surrogate | Milliseconds | Trained accuracy | Implicit | Trained domain | Yes |
|
||||
|
||||
### 4.5 Hybrid Approaches: Neural Surrogates Trained on CEM
|
||||
|
||||
The most promising direction combines full-wave accuracy with real-time speed:
|
||||
|
||||
1. **Offline**: Run thousands of FDTD/FEM simulations with different body positions
|
||||
2. **Train**: Neural network learns the mapping from body state to CSI
|
||||
3. **Deploy**: Neural surrogate runs in milliseconds for real-time inference
|
||||
|
||||
This is exactly Arena Physica's approach (Section 5), applied to RF component design rather than sensing. The same methodology applies to WiFi sensing: train a neural forward model on FDTD data, then use it as a differentiable physics constraint during inverse model training.
|
||||
|
||||
|
||||
## 5. Arena Physica's Approach
|
||||
|
||||
### 5.1 Company Overview
|
||||
|
||||
Arena Physica (arena-ai.com / arenaphysica.com) pursues "Electromagnetic Superintelligence" -- building foundation models that develop superhuman intuition for how geometry shapes electromagnetic fields. Founded by Pratap Ranade (CEO), Arya Hezarkhani, Claire Pan, Michael Frei, and Harish Krishnaswamy. Offices in NYC (HQ), SF, LA.
|
||||
|
||||
Raised $30M Series B (April 2025). Deployed with AMD, Anduril Industries, Sivers Semiconductors, Bausch & Lomb. Claims 35% reduction in engineering man-hours and multi-month acceleration in time-to-market.
|
||||
|
||||
### 5.2 Technical Architecture
|
||||
|
||||
Arena's Atlas platform uses two foundation models:
|
||||
|
||||
**Heaviside-0 (Forward Model)**:
|
||||
- Input: PCB/RF geometry (discretized as grid)
|
||||
- Output: S-parameters (magnitude + phase) and field distributions
|
||||
- Speed: 13ms per design (single), 0.3ms batched
|
||||
- Comparison: Traditional solver (HFSS/FDTD) takes ~4 minutes
|
||||
- Speedup: 18,000x to 800,000x
|
||||
|
||||
**Marconi-0 (Inverse Model)**:
|
||||
- Input: Target S-parameter specification
|
||||
- Output: Physical geometry that achieves the specification
|
||||
- Method: Conditional diffusion process (similar to image generation)
|
||||
- Generates unconventional geometries no human designer would conceive
|
||||
|
||||
**Training data**: 3 million simulated designs across 25 expert templates + random structures, totaling 20+ years of combined simulation time. Incorporates both S-parameter data and electromagnetic field distributions.
|
||||
|
||||
**Validation**: Predictions validated against commercial numerical field solvers (likely HFSS). Internal testing shows < 1 dB magnitude-weighted MAE (RF engineers operate in 20-30 dB ranges).
|
||||
|
||||
### 5.3 Relationship to Maxwell's Equations
|
||||
|
||||
Arena does NOT solve Maxwell's equations directly. Instead:
|
||||
|
||||
1. **Training phase**: Maxwell's equations are solved by conventional solvers (FDTD/FEM/MoM) millions of times to generate training data
|
||||
2. **Inference phase**: Neural surrogate approximates Maxwell's solutions in milliseconds
|
||||
3. **Design loop**: Generator proposes geometry -> Evaluator predicts EM behavior -> Iterate
|
||||
|
||||
As Pratap Ranade states: the model "learns the syntax of physics" inductively from examples, rather than deductively from equations. This trades precision for speed -- acceptable when searching design space where "speed and direction matter more than precision."
|
||||
|
||||
### 5.4 The "Large Field Model" (LFM) Concept
|
||||
|
||||
Arena's LFM is distinct from Large Language Models:
|
||||
- LLMs learn linguistic patterns from text
|
||||
- LFMs learn electromagnetic field patterns from simulation data
|
||||
- The input is geometry (not text); the output is field distributions (not tokens)
|
||||
- Domain-specific architecture substantially outperforms general LLMs on EM tasks
|
||||
|
||||
### 5.5 Relevance to WiFi Sensing
|
||||
|
||||
Arena Physica focuses on RF component design (antennas, PCBs, filters), not WiFi sensing. However, their approach is directly transferable:
|
||||
|
||||
| Arena Physica (Design) | WiFi Sensing (Our Case) |
|
||||
|---|---|
|
||||
| Forward: geometry -> S-parameters | Forward: body pose -> CSI |
|
||||
| Inverse: S-parameters -> geometry | Inverse: CSI -> body pose |
|
||||
| Train on FDTD/FEM simulations | Train on ray-tracing / FDTD simulations |
|
||||
| 13ms inference | Real-time CSI inference |
|
||||
| Conditional diffusion for generation | Conditional generation for pose prediction |
|
||||
|
||||
**Key lesson for wifi-densepose**: Building a neural forward model (body_pose -> expected_CSI) trained on electromagnetic simulation data, then using it as a differentiable physics constraint during inverse model training, could significantly improve our pose estimation accuracy and generalization. This is the "physics-informed" approach with the computational burden shifted to offline training.
|
||||
|
||||
|
||||
## 6. Connections to wifi-densepose Codebase
|
||||
|
||||
### 6.1 Existing Physics-Based Modules
|
||||
|
||||
| Module | Physical Model | Maxwell Connection |
|
||||
|---|---|---|
|
||||
| `field_model.rs` | SVD eigenstructure decomposition | Eigenmode basis of room's EM field |
|
||||
| `tomography.rs` | L1-regularized RTI (ISTA solver) | Linearized inverse scattering |
|
||||
| `multistatic.rs` | Attention-weighted cross-node fusion | Exploits geometric diversity of multiple TX/RX |
|
||||
| `phase_align.rs` | LO phase offset estimation | Corrects hardware-induced phase corruption |
|
||||
| `coherence.rs` | Z-score coherence scoring | Statistical test on EM field stability |
|
||||
| `coherence_gate.rs` | Accept/Reject decisions | Quality control on EM measurements |
|
||||
| `adversarial.rs` | Physical impossibility detection | Enforces EM consistency constraints |
|
||||
|
||||
### 6.2 Potential Enhancements Based on This Research
|
||||
|
||||
1. **Differentiable ray-tracing forward model**: Train a neural surrogate on ray-tracing simulations of CSI for various body poses in the deployment room. Use as physics constraint in pose estimation.
|
||||
|
||||
2. **Fresnel zone integration**: Augment the attention mechanism in `multistatic.rs` with Fresnel zone geometry -- links where the body falls within the first Fresnel zone should receive higher attention weight.
|
||||
|
||||
3. **xPRA-LM inverse scattering**: For higher-resolution body imaging than RTI, implement the Extended Phaseless Rytov Approximation. Our tomography module currently uses the simpler additive attenuation model.
|
||||
|
||||
4. **HoloCSI-style sparse transformer**: Replace the dense attention in cross-viewpoint fusion with top-k sparse attention for efficiency on ESP32-constrained deployments.
|
||||
|
||||
5. **Physics-informed training loss**: When training the DensePose model, add a loss term penalizing physically impossible CSI patterns (e.g., signals that would require faster-than-light propagation or negative attenuation).
|
||||
|
||||
|
||||
## 7. References
|
||||
|
||||
### Core WiFi Sensing Surveys
|
||||
- WiFi Sensing with Channel State Information: A Survey. ACM Computing Surveys, 2019. https://dl.acm.org/doi/fullHtml/10.1145/3310194
|
||||
- Cross-Domain WiFi Sensing with Channel State Information: A Survey. ACM Computing Surveys, 2022. https://dl.acm.org/doi/10.1145/3570325
|
||||
- Wireless sensing applications with Wi-Fi CSI, preprocessing techniques, and detection algorithms: A survey. Computer Communications, 2024. https://www.sciencedirect.com/science/article/abs/pii/S0140366424002214
|
||||
- Understanding CSI (Tsinghua Tutorial). https://tns.thss.tsinghua.edu.cn/wst/docs/pre/
|
||||
|
||||
### Physics-Informed Neural Networks for RF
|
||||
- PINN and GNN-based RF Map Construction. arXiv 2507.22513
|
||||
- Physics-Informed Neural Networks for Wireless Channel Estimation. NeurIPS 2025, OpenReview r3plaU6DvW
|
||||
- ReVeal: High-Fidelity Radio Propagation. DySPAN 2025. https://wici.iastate.edu/wp-content/uploads/2025/03/ReVeal-DySPAN25.pdf
|
||||
- Physics-informed generative model for passive RF sensing. Savazzi et al., arXiv 2310.04173
|
||||
- Multi-Modal Foundational Model for Wireless Communication and Sensing. arXiv 2602.04016
|
||||
- Generative AI Meets Wireless Sensing: Towards Wireless Foundation Model. arXiv 2509.15258
|
||||
- Physics-Informed Neural Networks for Sensing Radio Spectrum. IJRTE v14i3, 2025
|
||||
|
||||
### Inverse Scattering and Body Reconstruction
|
||||
- DensePose From WiFi. Geng, Huang, De la Torre. arXiv 2301.00250
|
||||
- Through-Wall Human Pose Estimation Using Radio Signals. Zhao et al., CVPR 2018. https://rfpose.csail.mit.edu/
|
||||
- Person-in-WiFi: Fine-grained Person Perception. Wang et al., ICCV 2019
|
||||
- 3D Human Pose Estimation for Free-from Activities Using WiFi. arXiv 2204.07878
|
||||
- EM-POSE: 3D Human Pose from Sparse Electromagnetic Trackers. ICCV 2021
|
||||
- Reconciling Radio Tomographic Imaging with Phaseless Inverse Scattering. Dubey, Li, Murch. arXiv 2311.09633
|
||||
- Accurate Indoor RF Imaging using Extended Rytov Approximation. Dubey et al., arXiv 2110.03211
|
||||
- Phaseless Extended Rytov Approximation for Strongly Scattering Low-Loss Media. IEEE, 2022. https://ieeexplore.ieee.org/document/9766313/
|
||||
- Distorted Wave Extended Phaseless Rytov Iterative Method. arXiv 2205.12578
|
||||
- 3D Full Convolution Electromagnetic Reconstruction Neural Network (3D-FCERNN). PMC 9689780
|
||||
|
||||
### Radio Tomographic Imaging
|
||||
- Radio Tomographic Imaging with Wireless Networks. Wilson & Patwari, 2010. https://span.ece.utah.edu/uploads/RTI_version_3.pdf
|
||||
- Compressive Sensing Based Radio Tomographic Imaging with Spatial Diversity. PMC 6386865
|
||||
- Passive Localization Based on Radio Tomography Images with CNN. Nature Scientific Reports, 2025
|
||||
- Enhancing Accuracy of WiFi Tomographic Imaging Using Human-Interference Model. 2018
|
||||
|
||||
### Fresnel Zone Models
|
||||
- WiFi CSI-based device-free sensing: from Fresnel zone model to CSI-ratio model. CCF Trans. Pervasive Computing, 2021. https://link.springer.com/article/10.1007/s42486-021-00077-z
|
||||
- Towards a Dynamic Fresnel Zone Model for WiFi-based Human Activity Recognition. ACM IMWUT, 2023. https://dl.acm.org/doi/10.1145/3596270
|
||||
- CSI-based human sensing using model-based approaches: a survey. JCDE, 2021. https://academic.oup.com/jcde/article/8/2/510/6137731
|
||||
|
||||
### Computational Electromagnetics
|
||||
- Using Large-Scale FDTD for Indoor WLAN. ResearchGate. https://www.researchgate.net/publication/42637096
|
||||
- Human Body Shadowing -- FDTD and UTD. Hindawi IJAP, 2018. https://www.hindawi.com/journals/ijap/2018/9084830/
|
||||
- FDTD Assessment Human Exposure WiFi/Bluetooth. ResearchGate. https://www.researchgate.net/publication/23400115
|
||||
- Simulation of Wireless LAN Indoor Propagation Using FDTD. IEEE, 2007. https://ieeexplore.ieee.org/document/4396450
|
||||
- Waveguide Models of Indoor Channels: FDTD Insights. ResearchGate. https://www.researchgate.net/publication/4368711
|
||||
- XFdtd 3D EM Simulation Software. Remcom. https://www.remcom.com/xfdtd-3d-em-simulation-software
|
||||
- Wireless InSite Ray Tracing. Remcom. https://www.remcom.com/wireless-insite-em-propagation-software/
|
||||
|
||||
### Arena Physica
|
||||
- Introducing Atlas RF Studio. https://www.arenaphysica.com/publications/rf-studio
|
||||
- Electromagnetism Secretly Runs the World. Not Boring (Packy McCormick). https://www.notboring.co/p/electromagnetism-secretly-runs-the
|
||||
- Arena Launches Atlas (Press Release). https://www.prnewswire.com/news-releases/arena-launches-atlas-to-accelerate-humanitys-rate-of-hardware-innovation-302423412.html
|
||||
- Arena AI raises $30M. SiliconANGLE. https://siliconangle.com/2025/04/08/arena-ai-raises-30m-accelerate-innovation-hardware-testing-atlas/
|
||||
- Artificial Intuition: Building an AI Mind for EM Design. CDFAM NYC 2025. https://www.designforam.com/p/artificial-intuition-building-an
|
||||
|
||||
### Holographic / Advanced
|
||||
- HoloCSI: Holographic tomography pipeline with physics-guided projection and sparse transformer. 2025-2026
|
||||
- CSI-Bench: Large-Scale In-the-Wild Dataset for Multi-task WiFi Sensing. arXiv 2505.21866
|
||||
- RFBoost: Understanding and Boosting Deep WiFi Sensing via Physical Data Augmentation. arXiv 2410.07230
|
||||
- Vision Reimagined: AI-Powered Breakthroughs in WiFi Indoor Imaging. arXiv 2401.04317
|
||||
- Electromagnetic Information Theory for 6G. arXiv 2401.08921
|
||||
@@ -0,0 +1,731 @@
|
||||
# State-of-the-Art Neural Decoding Landscape (2023–2026)
|
||||
|
||||
## SOTA Research Document — RF Topological Sensing Series (21/22)
|
||||
|
||||
**Date**: 2026-03-09
|
||||
**Domain**: Neural Decoding × Generative AI × Brain-Computer Interfaces × Quantum Sensing
|
||||
**Status**: Research Survey / Strategic Positioning
|
||||
|
||||
---
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
The field of neural decoding has undergone a phase transition between 2023 and 2026. Three
|
||||
technologies stacked together — sensors, decoders, and visualization/reconstruction systems —
|
||||
have collectively moved "brain reading" from science fiction to engineering challenge. Yet the
|
||||
popular narrative obscures a critical distinction: current systems decode *perceived* and
|
||||
*intended* content from neural activity, not arbitrary private thoughts.
|
||||
|
||||
This document maps the current state of the art across all three layers, positions the
|
||||
RuVector + dynamic mincut architecture within this landscape, and identifies the unexplored
|
||||
territory where topological brain modeling could open an entirely new research direction.
|
||||
|
||||
---
|
||||
|
||||
## 2. Layer 1: Neural Sensors — The Fidelity Floor
|
||||
|
||||
Everything in neural decoding is bounded by sensor fidelity. No algorithm can extract
|
||||
information that the sensor never captured.
|
||||
|
||||
### 2.1 Invasive Neural Interfaces (Highest Fidelity)
|
||||
|
||||
**Technology**: Microelectrode arrays implanted directly in brain tissue.
|
||||
|
||||
**Leading Systems**:
|
||||
- **Neuralink N1**: 1,024 electrodes on flexible threads, wireless telemetry
|
||||
- **Stanford BrainGate**: Utah microelectrode arrays (96 channels) in motor cortex
|
||||
- **ECoG grids**: Electrocorticography strips placed on cortical surface
|
||||
|
||||
**Capabilities Demonstrated**:
|
||||
- Decode speech intentions from motor cortex with ~74% accuracy (Stanford, 2023)
|
||||
- Control computer cursors and robotic arms in real time
|
||||
- Decode imagined handwriting at 90+ characters per minute
|
||||
- Reconstruct inner speech patterns from speech motor cortex
|
||||
|
||||
**Signal Characteristics**:
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Spatial resolution | Single neuron (~10 μm) |
|
||||
| Temporal resolution | Sub-millisecond |
|
||||
| Channel count | 96–1,024 |
|
||||
| Signal-to-noise ratio | 5–20 dB per neuron |
|
||||
| Coverage area | ~4×4 mm per array |
|
||||
| Bandwidth | DC to 10 kHz |
|
||||
|
||||
**Fundamental Limitation**: Requires brain surgery. Coverage area is tiny relative to the
|
||||
whole brain (~0.001% of cortical surface per array). Each implant covers one small patch.
|
||||
Network-level topology analysis requires coverage of many regions simultaneously — the exact
|
||||
opposite of what implants provide.
|
||||
|
||||
**Why This Matters for Mincut Architecture**: Implants give depth but not breadth. Dynamic
|
||||
mincut analysis of brain network topology requires simultaneous observation of dozens to
|
||||
hundreds of brain regions. This fundamentally favors non-invasive, whole-brain sensors.
|
||||
|
||||
### 2.2 Functional Magnetic Resonance Imaging (fMRI)
|
||||
|
||||
**Technology**: Measures blood-oxygen-level-dependent (BOLD) signal as proxy for neural
|
||||
activity.
|
||||
|
||||
**Signal Characteristics**:
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Spatial resolution | 1–3 mm voxels |
|
||||
| Temporal resolution | ~0.5–2 Hz (hemodynamic delay ~5–7 seconds) |
|
||||
| Coverage | Whole brain |
|
||||
| Cost | $2–5M per scanner |
|
||||
| Portability | None (fixed installation, 5+ ton magnet) |
|
||||
| Subject constraints | Must lie still in bore |
|
||||
|
||||
**Key Neural Decoding Results (2023–2026)**:
|
||||
- **Semantic decoding of continuous language** (Tang et al., 2023, University of Texas):
|
||||
Decoded continuous language from fMRI recordings of subjects listening to stories. Used
|
||||
GPT-based language model to map brain activity to word sequences. Achieved meaningful
|
||||
semantic recovery of story content, though not verbatim word-for-word accuracy.
|
||||
|
||||
- **Visual reconstruction** (Takagi & Nishimoto, 2023): High-fidelity reconstruction of
|
||||
viewed images from fMRI using latent diffusion models. Structural layout and semantic
|
||||
content recognizable, though fine details are lost.
|
||||
|
||||
- **Imagined image reconstruction**: Researchers achieved ~90% identification accuracy for
|
||||
seen images and ~75% for imagined images in constrained paradigms.
|
||||
|
||||
**Limitation for Topology Analysis**: The 5–7 second hemodynamic delay means fMRI cannot
|
||||
capture fast network topology transitions. Cognitive state changes that occur on millisecond
|
||||
timescales are invisible to fMRI. The technology is fundamentally a slow integrator, averaging
|
||||
neural activity over seconds.
|
||||
|
||||
### 2.3 Electroencephalography (EEG)
|
||||
|
||||
**Technology**: Scalp electrodes measuring voltage fluctuations from cortical neural activity.
|
||||
|
||||
**Signal Characteristics**:
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Spatial resolution | ~10–20 mm (severely blurred by skull) |
|
||||
| Temporal resolution | 1–1000 Hz |
|
||||
| Channel count | 32–256 |
|
||||
| Cost | $1K–50K |
|
||||
| Portability | High (wearable caps available) |
|
||||
| Setup time | 15–45 minutes |
|
||||
|
||||
**Neural Decoding Status**:
|
||||
- Motor imagery classification: 70–85% accuracy for 2–4 classes
|
||||
- P300-based BCI: reliable for character selection at ~5 characters/minute
|
||||
- Emotion recognition: 60–75% accuracy (limited by spatial resolution)
|
||||
- Cognitive workload detection: 80–90% accuracy in binary classification
|
||||
|
||||
**Limitation**: Skull conductivity smears spatial information severely. The volume conduction
|
||||
problem means that EEG measures a blurred weighted sum of many cortical sources. Source
|
||||
localization is ill-conditioned. Fine-grained network topology analysis is fundamentally
|
||||
limited by this spatial ambiguity.
|
||||
|
||||
### 2.4 Magnetoencephalography (MEG)
|
||||
|
||||
**Technology**: Measures magnetic fields generated by neuronal currents.
|
||||
|
||||
**Traditional SQUID-MEG**:
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Sensitivity | 3–5 fT/√Hz |
|
||||
| Spatial resolution | 3–5 mm (source localization) |
|
||||
| Temporal resolution | DC to 1000+ Hz |
|
||||
| Channel count | 275–306 |
|
||||
| Cost | $2–5M + $200K–2M shielded room |
|
||||
| Size | Fixed installation, liquid helium cooling |
|
||||
| Sensor-to-scalp distance | 20–30 mm (helmet gap) |
|
||||
|
||||
**Key Advantage for Topology Analysis**: MEG provides both high temporal resolution
|
||||
(millisecond) AND reasonable spatial resolution (millimeter-scale source localization). This
|
||||
combination is ideal for tracking dynamic network topology. Magnetic fields pass through the
|
||||
skull without distortion, unlike EEG.
|
||||
|
||||
**Emerging: OPM-MEG** (see Section 2.5)
|
||||
|
||||
### 2.5 Optically Pumped Magnetometers (OPMs)
|
||||
|
||||
**Technology**: Alkali vapor cells detect magnetic fields through spin-precession of
|
||||
optically pumped atoms. Operates in SERF (spin-exchange relaxation-free) regime for maximum
|
||||
sensitivity.
|
||||
|
||||
**Signal Characteristics**:
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Sensitivity | 7–15 fT/√Hz (on-head) |
|
||||
| Spatial resolution | ~3–5 mm |
|
||||
| Temporal resolution | DC to 200 Hz |
|
||||
| Sensor size | ~12×12×19 mm per channel |
|
||||
| Cost per sensor | $5K–15K |
|
||||
| Cryogenics | None (room temperature) |
|
||||
| Wearable | Yes (3D-printed helmets) |
|
||||
| Movement tolerance | High (subjects can move) |
|
||||
|
||||
**Why OPM is the Most Important Near-Term Sensor for This Architecture**:
|
||||
|
||||
1. **Wearable**: subjects can move naturally, enabling ecological paradigms
|
||||
2. **Close proximity**: sensor directly on scalp (~6 mm gap vs ~25 mm for SQUID)
|
||||
3. **Better SNR**: closer sensors → 2–3× better signal-to-noise ratio
|
||||
4. **Scalable**: add channels incrementally
|
||||
5. **Cost trajectory**: full system potentially $50K–200K vs $2M+ for SQUID
|
||||
6. **Temporal resolution**: millisecond-scale network dynamics visible
|
||||
7. **Spatial resolution**: adequate for 68–400 brain parcels
|
||||
|
||||
**Leading Groups**:
|
||||
- University of Nottingham / Cerca Magnetics: pioneered wearable OPM-MEG
|
||||
- FieldLine Inc: HEDscan commercial system
|
||||
- QuSpin: Gen-3 QZFM sensor modules
|
||||
|
||||
### 2.6 Quantum Sensors (Frontier)
|
||||
|
||||
**NV Diamond Magnetometers**:
|
||||
- Nitrogen-vacancy defects in diamond detect magnetic fields at femtotesla sensitivity
|
||||
- Room temperature operation, no cryogenics
|
||||
- Potential for miniaturization to chip scale
|
||||
- Current lab sensitivity: ~1–10 fT/√Hz
|
||||
- Advantage: can be fabricated as dense 2D arrays for high spatial resolution
|
||||
- Status: demonstrated in controlled lab conditions, not yet clinical
|
||||
|
||||
**Atomic Interferometers**:
|
||||
- Detect phase shifts in atomic wavefunctions
|
||||
- Extreme precision for magnetic and gravitational fields
|
||||
- Current status: large laboratory instruments
|
||||
- Potential: sub-femtotesla magnetic field measurement
|
||||
- Limitation: low bandwidth (1–10 Hz cycle rate), large apparatus
|
||||
|
||||
### 2.7 Sensor Comparison Matrix
|
||||
|
||||
| Sensor | Spatial Res. | Temporal Res. | Invasive | Portable | Cost | Network Topology Suitability |
|
||||
|--------|-------------|---------------|----------|----------|------|------------------------------|
|
||||
| Implants | 10 μm | <1 ms | Yes | No | $50K+ surgery | Poor (tiny coverage) |
|
||||
| fMRI | 1–3 mm | 0.5 Hz | No | No | $2–5M | Moderate (good spatial, poor temporal) |
|
||||
| EEG | 10–20 mm | 1 kHz | No | Yes | $1–50K | Poor (spatial smearing) |
|
||||
| SQUID-MEG | 3–5 mm | 1 kHz | No | No | $2–5M | Good (but fixed, expensive) |
|
||||
| OPM-MEG | 3–5 mm | 200 Hz | No | Yes | $50–200K | Excellent |
|
||||
| NV Diamond | <1 mm | 1 kHz | No | Potentially | $5–50K | Excellent (when mature) |
|
||||
| Atom Interf. | N/A | 1–10 Hz | No | No | $100K+ | Poor (bandwidth limited) |
|
||||
|
||||
**Conclusion**: OPM-MEG is the clear near-term choice for real-time brain network topology
|
||||
analysis. NV diamond arrays represent the medium-term upgrade path.
|
||||
|
||||
---
|
||||
|
||||
## 3. Layer 2: Neural Decoders — AI Meets Neuroscience
|
||||
|
||||
### 3.1 The Translation Paradigm
|
||||
|
||||
Modern neural decoding frames the problem as machine translation:
|
||||
- **Source language**: brain activity patterns (high-dimensional time series)
|
||||
- **Target language**: text, images, speech, or motor commands
|
||||
- **Translation model**: transformer or diffusion-based neural network
|
||||
|
||||
The pipeline is typically:
|
||||
```
|
||||
Brain signals → Feature extraction → Embedding space → Generative model → Output
|
||||
```
|
||||
|
||||
This paradigm has been remarkably successful for *perceived* content decoding.
|
||||
|
||||
### 3.2 Language Decoding
|
||||
|
||||
**Architecture**: Brain → embedding → language model → text
|
||||
|
||||
**Key Approaches**:
|
||||
|
||||
1. **Brain-to-embedding mapping**: Linear or nonlinear regression from brain activity
|
||||
(fMRI voxels or MEG sensors) to a shared embedding space (e.g., GPT embedding space).
|
||||
|
||||
2. **Embedding-to-text generation**: Pre-trained language model (GPT, LLaMA) generates
|
||||
text conditioned on the brain-derived embedding.
|
||||
|
||||
3. **End-to-end training**: Joint optimization of encoder and decoder, fine-tuned per
|
||||
subject.
|
||||
|
||||
**Results**:
|
||||
| Study | Modality | Task | Performance |
|
||||
|-------|----------|------|-------------|
|
||||
| Tang et al. (2023) | fMRI | Continuous speech decoding | Semantic gist recovery |
|
||||
| Défossez et al. (2023) | MEG/EEG | Speech perception | Word-level identification |
|
||||
| Willett et al. (2023) | Implant | Imagined handwriting | 94 characters/minute |
|
||||
| Metzger et al. (2023) | ECoG | Speech neuroprosthesis | 78 words/minute |
|
||||
|
||||
**Limitation**: All systems require extensive subject-specific training (typically 10–40 hours
|
||||
of calibration data). Cross-subject transfer is minimal. Decoding accuracy drops sharply for
|
||||
novel content not represented in training.
|
||||
|
||||
### 3.3 Image Reconstruction from Brain Activity
|
||||
|
||||
**Architecture**: Brain → latent vector → diffusion model → image
|
||||
|
||||
**Key Approaches**:
|
||||
|
||||
1. **fMRI-to-latent mapping**: Train a regression model from fMRI activation patterns to
|
||||
the latent space of a diffusion model (Stable Diffusion, DALL-E).
|
||||
|
||||
2. **Two-stage reconstruction**:
|
||||
- Stage 1: Decode semantic content (what is in the image)
|
||||
- Stage 2: Decode perceptual content (what it looks like)
|
||||
- Combine via conditional diffusion generation
|
||||
|
||||
3. **Brain Diffuser** (2023): Feeds fMRI representations through a variational autoencoder
|
||||
into a latent diffusion model. Reconstructs viewed images with recognizable structure
|
||||
and semantic content.
|
||||
|
||||
**Results**:
|
||||
- Viewed image reconstruction: structural layout and major objects identifiable
|
||||
- Imagined image reconstruction: ~75% identification accuracy (constrained set)
|
||||
- Cross-subject: poor (each subject needs individual model)
|
||||
|
||||
**What This Actually Recovers**:
|
||||
- High-level category (animal, building, face)
|
||||
- Spatial layout (left/right, center/periphery)
|
||||
- Color palette (approximate)
|
||||
- Semantic associations (beach scene, urban scene)
|
||||
|
||||
**What This Cannot Recover**:
|
||||
- Fine details (text, specific faces, exact objects)
|
||||
- Private imagination (untrained novel content)
|
||||
- Dreams (no training data exists during dreams)
|
||||
|
||||
### 3.4 Speech Synthesis from Neural Activity
|
||||
|
||||
**Architecture**: Motor cortex signals → articulatory model → speech synthesis
|
||||
|
||||
**Key Results**:
|
||||
- ECoG-based speech neuroprostheses decode attempted speech at 78 words/minute
|
||||
- Accuracy reaches 97% for 50-word vocabulary, drops to ~50% for open vocabulary
|
||||
- Real-time operation demonstrated for locked-in patients
|
||||
|
||||
**How This Works**:
|
||||
The motor cortex generates articulatory commands (tongue, lips, jaw, larynx positions) even
|
||||
when paralyzed. Electrodes on the motor cortex surface capture these attempted movements.
|
||||
A neural network maps motor signals to phoneme sequences, then a vocoder generates audio.
|
||||
|
||||
**Relevance to Mincut Architecture**: Speech decoding is a *content* problem. Mincut topology
|
||||
analysis is a *structure* problem. They are complementary, not competing. Mincut would detect
|
||||
when the speech network *activates* (pre-movement topology change), while the decoder would
|
||||
extract *what* is being said.
|
||||
|
||||
### 3.5 The Decoding Boundary
|
||||
|
||||
**What Current Decoders Can Access**:
|
||||
| Category | Accuracy | Modality | Training Required |
|
||||
|----------|----------|----------|-------------------|
|
||||
| Perceived speech (heard) | High | fMRI/ECoG | 10–40 hours |
|
||||
| Intended speech (attempted) | Moderate-High | ECoG/Implant | 10–40 hours |
|
||||
| Viewed images | Moderate | fMRI | 10–20 hours |
|
||||
| Imagined images | Low-Moderate | fMRI | 10–20 hours |
|
||||
| Motor intention (move left/right) | High | EEG/ECoG | 1–5 hours |
|
||||
| Semantic gist of thoughts | Low | fMRI | 10–40 hours |
|
||||
| Arbitrary private thoughts | None | Any | N/A |
|
||||
|
||||
**Why Arbitrary Thought Reading Is Extremely Unlikely**:
|
||||
|
||||
1. **Distributed representation**: Thoughts are encoded across millions of neurons in
|
||||
patterns that are not spatially localized.
|
||||
|
||||
2. **Individual specificity**: The neural code for the same concept differs between
|
||||
individuals. Transfer models fail across subjects.
|
||||
|
||||
3. **Context dependence**: The same neural pattern can represent different things depending
|
||||
on context, state, and history.
|
||||
|
||||
4. **Combinatorial complexity**: The space of possible thoughts is effectively infinite.
|
||||
Training data can never cover it.
|
||||
|
||||
5. **Temporal complexity**: Thoughts are not static patterns but dynamic trajectories
|
||||
through neural state space.
|
||||
|
||||
---
|
||||
|
||||
## 4. Layer 3: Visualization and Reconstruction
|
||||
|
||||
### 4.1 Visual Perception Reconstruction
|
||||
|
||||
**State of the Art Pipeline**:
|
||||
```
|
||||
Brain signal (fMRI/MEG)
|
||||
→ Feature extraction (voxel patterns or sensor topography)
|
||||
→ Embedding (mapped to CLIP or diffusion model latent space)
|
||||
→ Conditional generation (Stable Diffusion or similar)
|
||||
→ Reconstructed image
|
||||
```
|
||||
|
||||
**Meta AI (2023–2024)**: Demonstrated near-real-time reconstruction of visual stimuli from
|
||||
MEG signals. Used a large pre-trained visual model to map MEG topography to image embeddings,
|
||||
then generated images via diffusion. Temporal resolution was sufficient for video-like
|
||||
reconstruction of dynamic visual stimuli.
|
||||
|
||||
**Quality Assessment**:
|
||||
- High-level semantic content: 70–90% match
|
||||
- Spatial layout: 60–80% match
|
||||
- Color and texture: 40–60% match
|
||||
- Fine detail and text: <20% match
|
||||
- Novel/imagined content: 20–40% match
|
||||
|
||||
### 4.2 Speech Reconstruction
|
||||
|
||||
**Pipeline**:
|
||||
```
|
||||
Motor cortex signals (ECoG/Implant)
|
||||
→ Articulatory parameter extraction (tongue, jaw, lip positions)
|
||||
→ Phoneme sequence prediction
|
||||
→ Neural vocoder (WaveNet, HiFi-GAN)
|
||||
→ Synthesized speech audio
|
||||
```
|
||||
|
||||
**Performance**: Natural-sounding speech synthesis from neural signals demonstrated in
|
||||
multiple research groups. Quality sufficient for real-time communication in clinical BCI.
|
||||
|
||||
### 4.3 The Generative AI Amplifier
|
||||
|
||||
**Key Insight**: Generative AI (LLMs, diffusion models) dramatically amplified neural
|
||||
decoding capability by acting as a powerful *prior*. Instead of reconstructing output purely
|
||||
from neural data, the system uses neural data to *guide* a generative model that already
|
||||
knows what text and images look like.
|
||||
|
||||
This means:
|
||||
- **Less neural data needed**: The generative model fills in details
|
||||
- **Higher quality output**: Outputs look natural even with noisy input
|
||||
- **Risk of hallucination**: The model may generate plausible but incorrect content
|
||||
- **Overfitting to priors**: Reconstructions may reflect model biases, not actual thought
|
||||
|
||||
**Implication for Topology Analysis**: The RuVector/mincut approach sidesteps the hallucination
|
||||
problem entirely. It measures *structural properties* of brain activity (network topology,
|
||||
coherence boundaries) rather than trying to generate *content* (images, text). There is no
|
||||
generative prior to hallucinate — the topology either changes or it doesn't.
|
||||
|
||||
---
|
||||
|
||||
## 5. The Hard Limits
|
||||
|
||||
### 5.1 Physical Limits of Non-Invasive Sensing
|
||||
|
||||
**Magnetic field attenuation**: Neural magnetic fields drop as 1/r³ from the source.
|
||||
A cortical current dipole generating 100 fT at the scalp surface produces only ~10 fT at
|
||||
20 mm standoff (SQUID) and ~50 fT at 6 mm standoff (OPM). Deep brain structures (thalamus,
|
||||
hippocampus) generate signals attenuated by 10–100× at the scalp surface.
|
||||
|
||||
**Inverse problem ill-conditioning**: Reconstructing 3D current sources from 2D surface
|
||||
measurements is inherently ill-posed. Regularization is required, which limits spatial
|
||||
resolution. Typical resolution: 5–10 mm for cortical sources, 10–20 mm for deep sources.
|
||||
|
||||
**Noise floor**: Even with quantum sensors achieving fT/√Hz sensitivity, the fundamental
|
||||
noise floor limits signal detection from deep structures and weakly active regions.
|
||||
|
||||
### 5.2 Three Determinants of Decoding Capability
|
||||
|
||||
1. **Sensor fidelity**: Signal-to-noise ratio at the measurement point determines the
|
||||
information ceiling. No algorithm can recover information not captured by the sensor.
|
||||
|
||||
2. **Signal-to-noise ratio**: Environmental noise (urban electromagnetic interference,
|
||||
building vibrations, physiological artifacts) degrades achievable SNR in practice.
|
||||
|
||||
3. **Subject-specific training**: Neural representations are highly individual. Current
|
||||
decoders require 10–40 hours of calibration per subject. This is a fundamental barrier
|
||||
to scalable deployment.
|
||||
|
||||
### 5.3 What Is and Is Not Possible
|
||||
|
||||
**Confidently achievable with current technology**:
|
||||
- Binary cognitive state detection (focused vs. unfocused)
|
||||
- Gross motor intention (left hand vs. right hand)
|
||||
- Sleep stage classification
|
||||
- Epileptic activity detection
|
||||
- Perceived speech semantic gist (with fMRI and extensive training)
|
||||
|
||||
**Achievable with near-term advances (2–5 years)**:
|
||||
- Multi-class cognitive state classification (5–10 states)
|
||||
- Pre-movement intention detection (200–500 ms lead)
|
||||
- Real-time brain network topology visualization
|
||||
- Early neurological disease biomarkers from connectivity analysis
|
||||
- Non-invasive motor BCI with moderate accuracy
|
||||
|
||||
**Extremely unlikely**:
|
||||
- Real-time arbitrary thought reading
|
||||
- Cross-subject decoding without calibration
|
||||
- Covert brain scanning (sensors require cooperation)
|
||||
- Dream content reconstruction with meaningful accuracy
|
||||
|
||||
---
|
||||
|
||||
## 6. Where RuVector + Dynamic Mincut Fits
|
||||
|
||||
### 6.1 The Unexplored Niche
|
||||
|
||||
Most neural decoding research asks: **"What is the brain computing?"**
|
||||
|
||||
The RuVector + mincut architecture asks: **"How is the brain organizing its computation?"**
|
||||
|
||||
This is a fundamentally different question with different:
|
||||
- **Sensor requirements**: needs coverage breadth, not depth (favors non-invasive)
|
||||
- **Temporal requirements**: needs millisecond dynamics (favors MEG/OPM over fMRI)
|
||||
- **Output representation**: graphs and topology, not images or text
|
||||
- **Privacy implications**: measures state, not content
|
||||
|
||||
### 6.2 Positioning in the Landscape
|
||||
|
||||
```
|
||||
CONTENT-FOCUSED STRUCTURE-FOCUSED
|
||||
(What is thought?) (How does thought organize?)
|
||||
───────────────── ──────────────────────────────
|
||||
HIGH FIDELITY Implant BCI [Gap - no one here]
|
||||
Speech neuroprostheses
|
||||
|
||||
MEDIUM FIDELITY fMRI image reconstruction → RuVector + Mincut (OPM) ←
|
||||
fMRI language decoding Dynamic topology analysis
|
||||
|
||||
LOW FIDELITY EEG motor imagery EEG connectivity (basic)
|
||||
P300 BCI
|
||||
```
|
||||
|
||||
The RuVector + mincut architecture occupies the **medium-fidelity, structure-focused** quadrant
|
||||
— a space that is largely unexplored in current research.
|
||||
|
||||
### 6.3 What This Architecture Uniquely Enables
|
||||
|
||||
1. **Real-time network topology tracking**: No existing system monitors brain connectivity
|
||||
graph topology at millisecond resolution in real time.
|
||||
|
||||
2. **Structural transition detection**: Mincut identifies when brain networks reorganize,
|
||||
which correlates with cognitive state changes.
|
||||
|
||||
3. **Longitudinal tracking**: RuVector memory enables tracking of topology evolution over
|
||||
days, weeks, months — detecting gradual changes like neurodegeneration.
|
||||
|
||||
4. **Content-agnostic monitoring**: The system does not need to decode what is being thought.
|
||||
It detects how the brain organizes its processing, which is clinically and scientifically
|
||||
valuable without raising thought-privacy concerns.
|
||||
|
||||
5. **Cross-subject topology comparison**: While neural content representations differ between
|
||||
individuals, network *topology* properties (modularity, hub structure, integration) are
|
||||
more conserved across subjects.
|
||||
|
||||
### 6.4 Integration with Content Decoders
|
||||
|
||||
The topology analysis is complementary to content decoding, not competing:
|
||||
|
||||
```
|
||||
Quantum Sensors → Preprocessing → Source Localization → ┬─ Content Decoder (text/image)
|
||||
├─ Topology Analyzer (mincut)
|
||||
└─ Combined: state-aware decoding
|
||||
```
|
||||
|
||||
**Example**: A speech BCI could use mincut to detect when the speech network *activates*
|
||||
(pre-speech topology change at t = -300ms), then trigger the content decoder only when
|
||||
speech intention is detected. This reduces false activations and improves timing.
|
||||
|
||||
---
|
||||
|
||||
## 7. Neural Foundation Models
|
||||
|
||||
### 7.1 Emerging Direction
|
||||
|
||||
Training large models directly on brain data (analogous to LLMs trained on text):
|
||||
- **Brain-GPT** concepts: pre-train on large neural datasets, fine-tune per subject
|
||||
- **Cross-modal alignment**: align brain activity embeddings with CLIP/GPT embeddings
|
||||
- **Self-supervised learning**: predict masked brain regions from surrounding activity
|
||||
|
||||
### 7.2 Relevance to Topology Analysis
|
||||
|
||||
Foundation models could learn brain topology patterns from large datasets:
|
||||
- Pre-train on thousands of subjects' connectivity graphs
|
||||
- Learn universal topology transition patterns
|
||||
- Transfer: adapt to new subjects with minimal calibration
|
||||
- Enable cross-subject topology comparison in a shared embedding space
|
||||
|
||||
This is where RuVector's contrastive learning (AETHER) and geometric embedding become
|
||||
particularly valuable — they provide the representational framework for topology foundation
|
||||
models.
|
||||
|
||||
---
|
||||
|
||||
## 8. Five Landmark "Mind Reading" Experiments
|
||||
|
||||
### 8.1 Gallant Lab Visual Reconstruction (UC Berkeley, 2011)
|
||||
|
||||
**What they did**: Reconstructed movie clips from fMRI brain activity. Subjects watched movie
|
||||
trailers in an MRI scanner. A decoder predicted which of 1,000 random YouTube clips best
|
||||
matched the brain activity at each moment.
|
||||
|
||||
**Result**: Blurry but recognizable reconstructions of viewed video.
|
||||
|
||||
**Significance**: First demonstration that dynamic visual experience could be decoded from
|
||||
brain activity.
|
||||
|
||||
### 8.2 Tang et al. Continuous Language Decoder (UT Austin, 2023)
|
||||
|
||||
**What they did**: Decoded continuous speech from fMRI while subjects listened to stories.
|
||||
Used GPT-based language model to map fMRI activity to word sequences.
|
||||
|
||||
**Result**: Recovered semantic meaning of stories (not verbatim words).
|
||||
|
||||
**Significance**: First open-vocabulary language decoder from non-invasive imaging. Crucially,
|
||||
decoding failed when subjects were not cooperating — they could defeat the decoder by
|
||||
thinking about other things.
|
||||
|
||||
### 8.3 Takagi & Nishimoto Image Reconstruction (2023)
|
||||
|
||||
**What they did**: Fed fMRI patterns into a latent diffusion model (Stable Diffusion) to
|
||||
reconstruct viewed images.
|
||||
|
||||
**Result**: Recognizable reconstructions with correct semantic content and approximate layout.
|
||||
|
||||
**Significance**: Generative AI dramatically improved reconstruction quality over previous
|
||||
approaches.
|
||||
|
||||
### 8.4 Willett et al. Imagined Handwriting (Stanford, 2021)
|
||||
|
||||
**What they did**: Decoded imagined handwriting from motor cortex implant. Subject imagined
|
||||
writing letters; a neural network decoded the intended characters.
|
||||
|
||||
**Result**: 94.1 characters per minute with 94.1% accuracy (with language model correction).
|
||||
|
||||
**Significance**: Demonstrated that motor cortex retains detailed movement representations
|
||||
even years after paralysis.
|
||||
|
||||
### 8.5 Meta AI Real-Time MEG Reconstruction (2023–2024)
|
||||
|
||||
**What they did**: Trained a model to reconstruct viewed images from MEG signals in near
|
||||
real time.
|
||||
|
||||
**Result**: Decoded visual category and approximate layout with sub-second latency.
|
||||
|
||||
**Significance**: First demonstration of MEG-based visual decoding approaching real-time
|
||||
speed. MEG's temporal resolution enabled tracking of dynamic visual processing.
|
||||
|
||||
---
|
||||
|
||||
## 9. Strategic Implications for RuView Architecture
|
||||
|
||||
### 9.1 What the SOTA Map Tells Us
|
||||
|
||||
1. **Content decoding is advancing rapidly** but remains subject-specific and perception-bound.
|
||||
2. **Non-invasive sensors are reaching sufficient fidelity** for network-level analysis.
|
||||
3. **Generative AI amplifies decoding** but introduces hallucination risks.
|
||||
4. **Topology analysis is the unexplored dimension** — no major group is doing real-time
|
||||
mincut-based brain network analysis.
|
||||
5. **OPM-MEG is the enabling technology** — wearable, high-fidelity, affordable trajectory.
|
||||
|
||||
### 9.2 Recommended Architecture Priorities
|
||||
|
||||
| Priority | Rationale |
|
||||
|----------|-----------|
|
||||
| OPM-MEG integration first | Most mature quantum sensor, sufficient for network topology |
|
||||
| Real-time mincut pipeline | Unique capability, no competition |
|
||||
| RuVector longitudinal tracking | Clinical value for disease monitoring |
|
||||
| Content decoder integration later | Let others solve content; focus on topology |
|
||||
| NV diamond upgrade path | Higher spatial resolution when technology matures |
|
||||
|
||||
### 9.3 Competitive Landscape
|
||||
|
||||
**Who else is working on brain network topology?**
|
||||
|
||||
- **Graph neural network approaches**: Several groups apply GNNs to brain connectivity data,
|
||||
but primarily for static classification (disease vs. healthy), not real-time dynamic
|
||||
topology tracking.
|
||||
|
||||
- **Connectome analysis**: Human Connectome Project provides structural connectivity maps,
|
||||
but these are static (one scan per subject).
|
||||
|
||||
- **Dynamic functional connectivity (dFC)**: fMRI-based studies examine time-varying
|
||||
connectivity, but at ~0.5 Hz temporal resolution — too slow for real-time cognitive
|
||||
tracking.
|
||||
|
||||
- **No one is doing real-time mincut on brain networks from MEG/OPM data.** This is
|
||||
genuinely unexplored territory.
|
||||
|
||||
---
|
||||
|
||||
## 10. The Topological Difference
|
||||
|
||||
The critical reframing that separates this architecture from the mainstream neural decoding
|
||||
field:
|
||||
|
||||
**Mainstream Neural Decoding**:
|
||||
```
|
||||
Brain activity → What is the content? → Generate text/image/speech
|
||||
```
|
||||
- Requires subject-specific training
|
||||
- Limited to perceived/intended content
|
||||
- Raises profound privacy concerns
|
||||
- Subject can defeat the decoder by not cooperating
|
||||
|
||||
**Topological Brain Analysis (This Architecture)**:
|
||||
```
|
||||
Brain activity → How is the network organized? → Track topology changes
|
||||
```
|
||||
- More conserved across subjects (topology > content)
|
||||
- Measures cognitive state, not content
|
||||
- Privacy-preserving by design
|
||||
- Cannot be easily defeated (topology is involuntary)
|
||||
- Clinically valuable (disease signatures)
|
||||
- Scientifically novel (unexplored direction)
|
||||
|
||||
This is not a weaker version of mind reading. It is a fundamentally different measurement
|
||||
that reveals aspects of brain function that content decoders cannot access.
|
||||
|
||||
---
|
||||
|
||||
## 11. Conclusion
|
||||
|
||||
The 2023–2026 SOTA landscape shows that neural decoding has made remarkable progress on
|
||||
content recovery from brain activity, driven by the convergence of better sensors (OPM),
|
||||
better algorithms (transformers, diffusion models), and better training data. Yet this
|
||||
progress has not addressed the fundamental question of how cognition organizes itself
|
||||
topologically.
|
||||
|
||||
The RuVector + dynamic mincut architecture positions itself in this gap — not competing with
|
||||
content decoders but opening an entirely new dimension of brain observation. Combined with
|
||||
OPM quantum sensors, this becomes a "topological brain observatory" that measures the
|
||||
architecture of thought rather than its content.
|
||||
|
||||
The sensor fidelity is nearly sufficient. The algorithms exist. The software architecture
|
||||
(RuVector, mincut, temporal tracking) maps directly from the existing RF sensing codebase.
|
||||
The application space (clinical diagnostics, cognitive monitoring, BCI augmentation) is
|
||||
commercially viable.
|
||||
|
||||
The question is no longer "can this work?" but "who will build it first?"
|
||||
|
||||
---
|
||||
|
||||
## 12. References and Further Reading
|
||||
|
||||
### Sensor Technology
|
||||
- Boto et al. (2018). "Moving magnetoencephalography towards real-world applications with a
|
||||
wearable system." Nature.
|
||||
- Barry et al. (2020). "Sensitivity optimization for NV-diamond magnetometry." Reviews of
|
||||
Modern Physics.
|
||||
- Tierney et al. (2019). "Optically pumped magnetometers: From quantum origins to
|
||||
multi-channel magnetoencephalography." NeuroImage.
|
||||
|
||||
### Neural Decoding
|
||||
- Tang et al. (2023). "Semantic reconstruction of continuous language from non-invasive brain
|
||||
recordings." Nature Neuroscience.
|
||||
- Takagi & Nishimoto (2023). "High-resolution image reconstruction with latent diffusion
|
||||
models from human brain activity." CVPR.
|
||||
- Défossez et al. (2023). "Decoding speech perception from non-invasive brain recordings."
|
||||
Nature Machine Intelligence.
|
||||
|
||||
### Brain Network Analysis
|
||||
- Bullmore & Sporns (2009). "Complex brain networks: graph theoretical analysis." Nature
|
||||
Reviews Neuroscience.
|
||||
- Bassett & Sporns (2017). "Network neuroscience." Nature Neuroscience.
|
||||
- Vidaurre et al. (2018). "Spontaneous cortical activity transiently organises into frequency
|
||||
specific phase-coupling networks." Nature Communications.
|
||||
|
||||
### Visual Reconstruction
|
||||
- Nishimoto et al. (2011). "Reconstructing visual experiences from brain activity evoked by
|
||||
natural movies." Current Biology.
|
||||
- Ozcelik & VanRullen (2023). "Natural scene reconstruction from fMRI signals using
|
||||
generative latent diffusion." Scientific Reports.
|
||||
|
||||
### Speech BCI
|
||||
- Willett et al. (2021). "High-performance brain-to-text communication via handwriting."
|
||||
Nature.
|
||||
- Metzger et al. (2023). "A high-performance neuroprosthesis for speech decoding and avatar
|
||||
control." Nature.
|
||||
|
||||
---
|
||||
|
||||
*This document is part of the RF Topological Sensing research series. It positions the
|
||||
RuVector + dynamic mincut architecture within the 2023–2026 neural decoding landscape,
|
||||
identifying the unexplored niche of real-time brain network topology analysis.*
|
||||
@@ -0,0 +1,877 @@
|
||||
# Brain State Observatory — Ten Application Domains
|
||||
|
||||
## SOTA Research Document — RF Topological Sensing Series (22/22)
|
||||
|
||||
**Date**: 2026-03-09
|
||||
**Domain**: Clinical Diagnostics × BCI × Cognitive Science × Commercial Applications
|
||||
**Status**: Applications Roadmap / Strategic Analysis
|
||||
|
||||
---
|
||||
|
||||
## 1. Introduction — Not Mind Reading, Something Better
|
||||
|
||||
If you build a system that combines high-sensitivity neural sensing, RuVector-style geometric
|
||||
memory, and dynamic mincut topology analysis, you are not building a mind reader. You are
|
||||
building a **brain state observatory**.
|
||||
|
||||
The most valuable applications are not "reading thoughts." They are systems that measure how
|
||||
cognition organizes itself over time — and detect when that organization goes wrong.
|
||||
|
||||
This document maps ten application domains where the RuVector + dynamic mincut architecture
|
||||
becomes unusually powerful, with honest assessment of feasibility, market reality, and
|
||||
technical requirements for each.
|
||||
|
||||
---
|
||||
|
||||
## 2. Domain 1: Neurological Disease Detection
|
||||
|
||||
### 2.1 Clinical Need
|
||||
|
||||
Neurological diseases are diagnosed late. By the time symptoms are visible:
|
||||
- Alzheimer's: 40–60% of neurons in affected regions are already dead
|
||||
- Parkinson's: 60–80% of dopaminergic neurons in substantia nigra are lost
|
||||
- Epilepsy: seizures may have been building for years before clinical onset
|
||||
- Multiple Sclerosis: demyelination is often widespread before first relapse
|
||||
|
||||
The fundamental problem: structural damage is detectable only after it becomes severe.
|
||||
Functional network changes precede structural damage by years.
|
||||
|
||||
### 2.2 How Mincut Detects Disease
|
||||
|
||||
Each neurological condition has a characteristic topology signature:
|
||||
|
||||
**Alzheimer's Disease**:
|
||||
- Progressive disconnection of the default mode network (DMN)
|
||||
- Loss of hub connectivity (especially posterior cingulate, medial prefrontal)
|
||||
- Increased graph fragmentation → mincut value decreases over months/years
|
||||
- Mincut tracking detects gradual network dissolution before clinical symptoms
|
||||
|
||||
Topology signature:
|
||||
```
|
||||
Healthy: mc(DMN) = 0.82 ± 0.05 (strongly integrated)
|
||||
Prodromal: mc(DMN) = 0.61 ± 0.08 (beginning to fragment)
|
||||
Clinical: mc(DMN) = 0.34 ± 0.12 (severely fragmented)
|
||||
```
|
||||
|
||||
**Epilepsy**:
|
||||
- Pre-ictal phase: abnormal hypersynchronization of local networks
|
||||
- Focal region becomes increasingly connected internally while disconnecting from surround
|
||||
- Mincut detects the pre-seizure topology: high local coupling, low global integration
|
||||
- Prediction window: 30 seconds to 5 minutes before seizure onset
|
||||
|
||||
Topology signature:
|
||||
```
|
||||
Inter-ictal: mc(focus) = 0.45 mc(global) = 0.72
|
||||
Pre-ictal: mc(focus) = 0.12 mc(global) = 0.83 ← focus isolating
|
||||
Ictal: mc(focus) = 0.03 mc(global) = 0.95 ← hypersync
|
||||
```
|
||||
|
||||
**Parkinson's Disease**:
|
||||
- Disruption of basal ganglia–cortical motor loops
|
||||
- Beta oscillation network topology changes
|
||||
- Asymmetric degradation (one hemisphere typically leads)
|
||||
- Mincut across motor network correlates with motor symptom severity
|
||||
|
||||
**Traumatic Brain Injury (TBI)**:
|
||||
- Acute: diffuse disconnection, globally elevated mincut
|
||||
- Recovery: gradual re-integration of network modules
|
||||
- Chronic: persistent topology abnormalities correlate with cognitive deficits
|
||||
- Mincut tracking provides objective recovery metric
|
||||
|
||||
### 2.3 Clinical Implementation
|
||||
|
||||
**Input**: Neural signals from OPM-MEG or NV magnetometer array
|
||||
**Processing**: Dynamic connectivity graph → mincut analysis → longitudinal tracking
|
||||
**Output**: Network integrity report, early warning alerts, progression tracking
|
||||
|
||||
**Regulatory Pathway**: Medical device (FDA 510(k) or De Novo for diagnostic aid)
|
||||
- Predicate devices: existing MEG diagnostic systems
|
||||
- Clinical validation: prospective cohort studies comparing mincut biomarkers to
|
||||
established diagnostic criteria
|
||||
- Timeline: 3–5 years from first prototype to regulatory submission
|
||||
|
||||
### 2.4 Market Reality
|
||||
|
||||
Hospitals spend billions annually on diagnostic neuroimaging (MRI, CT, PET). Current tools
|
||||
provide structural images or slow functional snapshots (fMRI). No tool provides real-time
|
||||
functional network topology monitoring.
|
||||
|
||||
**Market size estimates**:
|
||||
| Application | Annual Market | Current Gap |
|
||||
|-------------|-------------|-------------|
|
||||
| Alzheimer's diagnostics | $6B globally | No early functional biomarker |
|
||||
| Epilepsy monitoring | $2B globally | Poor seizure prediction |
|
||||
| TBI assessment | $1.5B globally | No objective recovery metric |
|
||||
| Parkinson's monitoring | $1B globally | Limited progression tracking |
|
||||
|
||||
---
|
||||
|
||||
## 3. Domain 2: Brain-Computer Interfaces
|
||||
|
||||
### 3.1 Architecture
|
||||
|
||||
```
|
||||
Neural signals → RuVector embeddings → State memory → Decode intent → Device control
|
||||
```
|
||||
|
||||
### 3.2 Capabilities
|
||||
|
||||
| Application | Signal Source | Accuracy Target | Latency Target |
|
||||
|-------------|-------------|-----------------|----------------|
|
||||
| Prosthetic control | Motor cortex topology | 90%+ for 6 DOF | <100 ms |
|
||||
| Typing/communication | Speech network topology | 95%+ characters | <200 ms |
|
||||
| Computer cursor control | Motor intention states | 95%+ directions | <50 ms |
|
||||
| Environmental control | Cognitive state | 85%+ for 4 commands | <500 ms |
|
||||
|
||||
### 3.3 Topology-Based BCI Advantages
|
||||
|
||||
Traditional BCI decodes amplitude patterns (which neurons fire, how strongly).
|
||||
Topology-based BCI decodes network reorganization patterns.
|
||||
|
||||
**Advantages**:
|
||||
1. **More robust**: Network topology is less variable than amplitude patterns across sessions
|
||||
2. **Self-calibrating**: Topology features normalize automatically (relative, not absolute)
|
||||
3. **State-aware**: Detects when the user is "ready" vs "idle" from network structure
|
||||
4. **Pre-movement detection**: Topology changes precede motor output by 200–500 ms
|
||||
|
||||
**Disadvantage**:
|
||||
- Lower spatial specificity than invasive implants (cannot decode individual finger movements)
|
||||
- Best for categorical commands, not continuous analog control
|
||||
|
||||
### 3.4 Non-Invasive BCI Breakthrough Potential
|
||||
|
||||
Current non-invasive BCI (EEG-based) achieves ~70–85% accuracy for binary classification.
|
||||
The limitation is EEG's poor spatial resolution.
|
||||
|
||||
OPM-MEG + mincut could provide:
|
||||
- Better spatial resolution → more distinguishable states
|
||||
- Topology features that are more stable across sessions
|
||||
- Reduced calibration time (topology patterns are more conserved)
|
||||
- Potential accuracy: 85–95% for 4–8 state classification
|
||||
|
||||
**This could be the first non-invasive BCI that approaches implant-level utility for
|
||||
categorical control tasks.**
|
||||
|
||||
### 3.5 Speech Reconstruction for Paralyzed Patients
|
||||
|
||||
The most impactful near-term BCI application:
|
||||
- Detect speech intention from motor cortex network activation
|
||||
- Classify attempted speech from topology of speech motor network
|
||||
- Combine with language model for error correction
|
||||
- Target: 30–50 words per minute (current ECoG: 78 wpm)
|
||||
|
||||
Even at lower throughput, a non-invasive speech BCI eliminates the need for brain surgery.
|
||||
|
||||
---
|
||||
|
||||
## 4. Domain 3: Cognitive State Monitoring
|
||||
|
||||
### 4.1 Core Capability
|
||||
|
||||
Measure brain network organization to infer mental states without decoding content.
|
||||
|
||||
The system answers: "Is this person focused, fatigued, overloaded, or disengaged?"
|
||||
It does NOT answer: "What is this person thinking about?"
|
||||
|
||||
### 4.2 Metrics
|
||||
|
||||
| Metric | Computation | Cognitive Correlate |
|
||||
|--------|-------------|---------------------|
|
||||
| Global mincut value | Minimum cut of whole-brain graph | Integration level |
|
||||
| Modular structure | Number and size of graph modules | Cognitive mode |
|
||||
| Hub connectivity | Degree centrality of hub regions | Executive function |
|
||||
| Graph entropy | Shannon entropy of edge weight distribution | Cognitive complexity |
|
||||
| Temporal variability | Rate of topology change | Engagement level |
|
||||
| Inter-hemispheric mincut | Left-right partition strength | Lateralized processing |
|
||||
|
||||
### 4.3 Industry Applications
|
||||
|
||||
**Aviation**:
|
||||
- Pilot cognitive workload monitoring
|
||||
- Fatigue detection during long-haul flights
|
||||
- Attention allocation tracking (scan pattern vs focus)
|
||||
- Regulatory interest: FAA/EASA fatigue risk management
|
||||
|
||||
**Military**:
|
||||
- Operator cognitive load in command centers
|
||||
- Fatigue monitoring for extended missions
|
||||
- Stress detection in high-threat environments
|
||||
- DARPA has funded cognitive workload research for decades
|
||||
|
||||
**Spaceflight**:
|
||||
- Astronaut cognitive performance monitoring
|
||||
- Sleep quality assessment in microgravity
|
||||
- Isolation and confinement effects on brain topology
|
||||
- NASA human factors research priorities
|
||||
|
||||
**High-Performance Work**:
|
||||
- Surgeon fatigue monitoring during long procedures
|
||||
- Air traffic controller workload assessment
|
||||
- Nuclear plant operator vigilance monitoring
|
||||
- Financial trading desk cognitive load optimization
|
||||
|
||||
### 4.4 Latency Requirements
|
||||
|
||||
| Application | Max Latency | Consequence of Late Detection |
|
||||
|-------------|-------------|-------------------------------|
|
||||
| Aviation (fatigue alert) | <5 seconds | Delayed warning |
|
||||
| Military (overload) | <2 seconds | Decision error |
|
||||
| Surgery (fatigue) | <10 seconds | Delayed warning |
|
||||
| Industrial safety | <1 second | Accident risk |
|
||||
|
||||
### 4.5 DARPA and NASA Context
|
||||
|
||||
DARPA programs funding cognitive monitoring:
|
||||
- **DARPA N3**: Next-generation non-surgical neurotechnology
|
||||
- **DARPA NESD**: Neural Engineering System Design
|
||||
- **DARPA RAM**: Restoring Active Memory
|
||||
|
||||
NASA research:
|
||||
- Human Research Program: cognitive performance in spaceflight
|
||||
- Behavioral Health and Performance: monitoring astronaut brain function
|
||||
- Gateway lunar station: long-duration crew monitoring needs
|
||||
|
||||
---
|
||||
|
||||
## 5. Domain 4: Mental Health Diagnostics
|
||||
|
||||
### 5.1 The Diagnostic Gap
|
||||
|
||||
Most psychiatric diagnoses rely on subjective questionnaires (PHQ-9, GAD-7, DSM-5 criteria).
|
||||
There are no objective biomarkers for most mental health conditions. This leads to:
|
||||
- Diagnostic uncertainty (40% of depression cases misdiagnosed initially)
|
||||
- Treatment selection by trial-and-error
|
||||
- No objective measure of treatment response
|
||||
- Stigma from perceived subjectivity of diagnosis
|
||||
|
||||
### 5.2 Neural Topology Biomarkers
|
||||
|
||||
Each psychiatric condition has characteristic network topology disruptions:
|
||||
|
||||
**Major Depression**:
|
||||
- Default mode network (DMN) over-integration: abnormally low mincut within DMN
|
||||
- Reduced executive network connectivity
|
||||
- Disrupted DMN–executive network anticorrelation
|
||||
- Topology signature: mc(DMN) low, mc(DMN↔Executive) high
|
||||
|
||||
**Generalized Anxiety**:
|
||||
- Amygdala–prefrontal connectivity disruption
|
||||
- Hyperconnectivity of threat-processing networks
|
||||
- Reduced top-down regulation from prefrontal cortex
|
||||
- Topology signature: abnormal hub structure in salience network
|
||||
|
||||
**PTSD**:
|
||||
- Hippocampal disconnection from cortical networks
|
||||
- Amygdala hyperconnectivity
|
||||
- Disrupted fear extinction network (ventromedial PFC)
|
||||
- Topology signature: fragmented memory encoding network
|
||||
|
||||
**Schizophrenia**:
|
||||
- Global disruption of integration-segregation balance
|
||||
- Reduced small-world properties
|
||||
- Disrupted thalamo-cortical connectivity
|
||||
- Topology signature: globally altered graph metrics
|
||||
|
||||
### 5.3 Treatment Monitoring
|
||||
|
||||
**Antidepressant response tracking**:
|
||||
- Baseline topology assessment before treatment
|
||||
- Weekly/monthly topology monitoring during treatment
|
||||
- Objective measure: is the network topology normalizing?
|
||||
- Predict treatment response from early topology changes (week 1–2)
|
||||
|
||||
**Psychotherapy monitoring**:
|
||||
- Track network changes during cognitive behavioral therapy
|
||||
- Measure: is the DMN–executive anticorrelation restoring?
|
||||
- Objective progress metric for therapist and patient
|
||||
|
||||
### 5.4 Functional Brain Biomarker Platform
|
||||
|
||||
The RuVector + mincut system could become a **general-purpose functional brain biomarker
|
||||
platform**:
|
||||
|
||||
```
|
||||
Patient Assessment Flow:
|
||||
1. 15-minute OPM recording (resting state + brief tasks)
|
||||
2. Real-time connectivity graph construction
|
||||
3. Mincut analysis → topology feature extraction
|
||||
4. Compare to normative database (age/sex matched)
|
||||
5. Generate biomarker report:
|
||||
- Network integration score
|
||||
- Modular structure comparison
|
||||
- Hub connectivity profile
|
||||
- Anomaly flags for specific conditions
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Domain 5: Neurofeedback and Brain Training
|
||||
|
||||
### 6.1 Real-Time Feedback Loop
|
||||
|
||||
```
|
||||
Brain activity → Topology analysis → Feedback signal → Cognitive adjustment
|
||||
↑ ↓
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 6.2 Applications
|
||||
|
||||
**Focus Training**:
|
||||
- Target: increase frontal-parietal network integration (mincut decrease in attention network)
|
||||
- Feedback: visual/auditory signal indicating network state
|
||||
- Training: 20–30 sessions of 30 minutes each
|
||||
- Evidence: EEG neurofeedback for attention has moderate effect sizes (d = 0.4–0.6)
|
||||
- OPM-based topology feedback could improve by providing more specific targets
|
||||
|
||||
**ADHD Therapy**:
|
||||
- Target: normalize fronto-striatal network connectivity
|
||||
- Current EEG neurofeedback for ADHD: some evidence, controversial
|
||||
- Topology-based approach may be more specific → better outcomes
|
||||
- Insurance coverage potential if clinical trials succeed
|
||||
|
||||
**Stress Reduction**:
|
||||
- Target: reduce amygdala–prefrontal hyperconnectivity
|
||||
- Feedback when topology normalizes toward calm-state pattern
|
||||
- Combine with meditation/breathing guidance
|
||||
- Corporate wellness and clinical stress management
|
||||
|
||||
**Peak Performance Training**:
|
||||
- Target: optimize integration-segregation balance for specific tasks
|
||||
- Elite athletes: motor network optimization
|
||||
- Musicians: auditory-motor coupling refinement
|
||||
- Financial traders: decision network optimization under pressure
|
||||
|
||||
### 6.3 Technical Requirements for Neurofeedback
|
||||
|
||||
| Parameter | Requirement | Current Capability |
|
||||
|-----------|------------|-------------------|
|
||||
| Feedback latency | <250 ms | ~100 ms achievable |
|
||||
| Session duration | 30 minutes | Battery/comfort limits |
|
||||
| Feature stability | <5% variance | Topology features stable |
|
||||
| Wearability | Comfortable helmet | OPM helmets demonstrated |
|
||||
| Home use | Portable setup | Not yet (shielding needed) |
|
||||
|
||||
---
|
||||
|
||||
## 7. Domain 6: Dream and Imagination Reconstruction
|
||||
|
||||
### 7.1 Current State
|
||||
|
||||
**What has been demonstrated**:
|
||||
- fMRI reconstruction of viewed images (waking state) using diffusion models
|
||||
- Basic decoding of imagined visual categories from fMRI
|
||||
- Sleep stage classification from EEG/MEG
|
||||
|
||||
**What has NOT been demonstrated**:
|
||||
- Real-time dream content reconstruction
|
||||
- Imagined scene reconstruction with meaningful detail
|
||||
- Dream-to-image generation
|
||||
|
||||
### 7.2 What Topology Analysis Adds
|
||||
|
||||
Mincut analysis during sleep/dreaming could:
|
||||
- **Map dream network topology**: which brain regions are co-active during dreams?
|
||||
- **Detect lucid dreaming**: characterized by frontal network re-integration
|
||||
- **Track REM vs NREM topology**: distinct network organizations
|
||||
- **Identify replay events**: hippocampal-cortical coupling during memory consolidation
|
||||
|
||||
### 7.3 Brain-to-Art Interface
|
||||
|
||||
Creative application:
|
||||
- Artist wears OPM helmet during ideation
|
||||
- Topology analysis captures network states during creative thought
|
||||
- Map topology states to generative model parameters
|
||||
- Generate visual art that reflects brain network organization (not thought content)
|
||||
- The art represents HOW the brain is organizing, not WHAT it is imagining
|
||||
|
||||
### 7.4 Honest Assessment
|
||||
|
||||
Dream reconstruction remains the most speculative application. Current technology cannot
|
||||
meaningfully decode dream content. Topology analysis during sleep is feasible but interpretation
|
||||
is limited. This domain is 10+ years from practical application.
|
||||
|
||||
---
|
||||
|
||||
## 8. Domain 7: Cognitive Research
|
||||
|
||||
### 8.1 The Scientific Opportunity
|
||||
|
||||
Instead of static brain scans, researchers get continuous graph topology of cognition. This
|
||||
enables entirely new categories of scientific questions.
|
||||
|
||||
### 8.2 Research Questions This Architecture Could Answer
|
||||
|
||||
**How do thoughts form?**
|
||||
- Track topology transitions from idle state to focused cognition
|
||||
- Measure network integration speed and sequence
|
||||
- Compare across individuals, age groups, expertise levels
|
||||
- Temporal resolution: millisecond-by-millisecond topology evolution
|
||||
|
||||
**How do ideas propagate through brain networks?**
|
||||
- Present stimulus → track topology wave propagation
|
||||
- Measure information flow direction from mincut asymmetry
|
||||
- Identify bottleneck regions (high betweenness centrality)
|
||||
- Compare sensory processing paths across modalities
|
||||
|
||||
**How does memory recall reorganize connectivity?**
|
||||
- Cue presentation → hippocampal network activation → cortical reinstatement
|
||||
- Topology signature of successful vs failed recall
|
||||
- Reconsolidation: how does recalled memory modify the network?
|
||||
- Longitudinal: how do memory networks change over weeks?
|
||||
|
||||
**How does creativity emerge?**
|
||||
- Divergent thinking: loosened topology constraints, more random connections
|
||||
- Convergent thinking: tightened topology, focused integration
|
||||
- Creative insight (aha moment): sudden topology reorganization
|
||||
- Compare creative vs non-creative individuals' topology dynamics
|
||||
|
||||
**Developmental neuroscience**:
|
||||
- How do children's brain topologies differ from adults?
|
||||
- Track topology development across childhood and adolescence
|
||||
- Sensitive periods: when do specific network topologies crystallize?
|
||||
- OPM's wearability makes pediatric studies practical
|
||||
|
||||
**Aging and neurodegeneration**:
|
||||
- Healthy aging: gradual topology changes over decades
|
||||
- Pathological aging: accelerated topology degradation
|
||||
- Cognitive reserve: maintained topology despite structural damage
|
||||
- Can topology analysis predict cognitive decline years in advance?
|
||||
|
||||
### 8.3 Methodological Advantages
|
||||
|
||||
| Current Methods | Topology Approach |
|
||||
|----------------|-------------------|
|
||||
| fMRI: 0.5 Hz temporal resolution | OPM: 200+ Hz dynamics |
|
||||
| EEG: poor spatial resolution | OPM: 3–5 mm source localization |
|
||||
| Static connectivity matrices | Dynamic time-varying graphs |
|
||||
| Single-session snapshots | Longitudinal RuVector tracking |
|
||||
| Group-level statistics | Individual topology fingerprints |
|
||||
|
||||
### 8.4 This Is Network Science of Cognition
|
||||
|
||||
The field has studied individual brain regions and pairwise connections. Topology analysis
|
||||
studies the emergent organizational principles — how the whole network self-organizes to
|
||||
produce cognition. This is analogous to studying traffic patterns in a city rather than
|
||||
individual cars.
|
||||
|
||||
---
|
||||
|
||||
## 9. Domain 8: Human-Computer Interaction
|
||||
|
||||
### 9.1 Cognition-Aware Computing
|
||||
|
||||
Computers could adapt their behavior based on the user's cognitive state.
|
||||
|
||||
### 9.2 Applications
|
||||
|
||||
**Adaptive Software Interfaces**:
|
||||
- Detect cognitive overload → simplify interface, reduce information density
|
||||
- Detect high focus → minimize interruptions, defer notifications
|
||||
- Detect confusion → provide contextual help, slow down tutorial pace
|
||||
- Detect fatigue → suggest breaks, reduce task complexity
|
||||
|
||||
**Learning Systems**:
|
||||
- Detect when student is confused (topology disruption in comprehension networks)
|
||||
- Adjust difficulty and presentation style in real time
|
||||
- Identify optimal learning moments (high engagement topology)
|
||||
- Personalize educational content to individual learning topology
|
||||
|
||||
**Immersive Experiences**:
|
||||
- VR/AR systems that respond to cognitive state
|
||||
- Game difficulty that adapts to engagement level
|
||||
- Meditation/mindfulness apps with real-time topology feedback
|
||||
- Therapeutic VR guided by brain network state
|
||||
|
||||
### 9.3 Cognition-Aware Operating System Concept
|
||||
|
||||
```
|
||||
Sensor Layer: OPM headband → continuous topology stream
|
||||
Analysis Layer: Real-time mincut → cognitive state classification
|
||||
OS Layer: CogState API → applications query current state
|
||||
App Layer: Notifications, UI complexity, timing adapt automatically
|
||||
```
|
||||
|
||||
**States the OS tracks**:
|
||||
| State | Topology Signature | OS Action |
|
||||
|-------|-------------------|-----------|
|
||||
| Deep focus | High frontal integration | Block notifications |
|
||||
| Low attention | Fragmented topology | Suggest break |
|
||||
| Creative mode | Loose coupling, high entropy | Expand workspace |
|
||||
| Stress | Amygdala-PFC disruption | Calming UI adjustments |
|
||||
| Fatigue | Reduced graph energy | Reduce complexity |
|
||||
|
||||
### 9.4 Timeline
|
||||
|
||||
- Near-term (1–3 years): Research prototypes in controlled settings
|
||||
- Medium-term (3–7 years): Professional applications (aviation, surgery)
|
||||
- Long-term (7–15 years): Consumer-grade cognition-aware computing
|
||||
|
||||
---
|
||||
|
||||
## 10. Domain 9: Brain Health Monitoring Wearables
|
||||
|
||||
### 10.1 The Brain's Apple Watch
|
||||
|
||||
If sensors become sufficiently small and affordable, continuous brain topology monitoring
|
||||
becomes possible in a wearable form factor.
|
||||
|
||||
### 10.2 Target Device
|
||||
|
||||
**Form factor**: Helmet, headband, or behind-ear device with magnetometer array
|
||||
**Sensors**: 8–32 miniaturized OPM or NV diamond sensors
|
||||
**Processing**: Edge AI chip for real-time topology analysis
|
||||
**Battery**: 8–12 hour operation
|
||||
**Connectivity**: Bluetooth/WiFi to smartphone app
|
||||
**Data**: Continuous topology metrics, alerts, daily reports
|
||||
|
||||
### 10.3 Monitoring Capabilities
|
||||
|
||||
**Sleep Quality**:
|
||||
- Sleep staging from topology transitions (wake → N1 → N2 → N3 → REM)
|
||||
- Sleep architecture quality score
|
||||
- Sleep spindle and slow wave detection
|
||||
- REM density and distribution
|
||||
- Compare to age-matched normative database
|
||||
|
||||
**Brain Health Baseline**:
|
||||
- Monthly topology assessment
|
||||
- Track gradual changes over years
|
||||
- Early warning for neurodegeneration
|
||||
- Concussion detection and recovery monitoring
|
||||
|
||||
**Concussion/TBI Risk**:
|
||||
- Pre-exposure baseline (for athletes, military)
|
||||
- Post-impact assessment: compare topology to baseline
|
||||
- Return-to-play/return-to-duty decision support
|
||||
- Longitudinal tracking during recovery
|
||||
|
||||
**Stress and Mental Health**:
|
||||
- Daily stress topology patterns
|
||||
- Chronic stress detection from sustained topology disruption
|
||||
- Correlation with self-reported well-being
|
||||
- Trigger identification from topology-event correlation
|
||||
|
||||
### 10.4 Technical Barriers to Consumer Deployment
|
||||
|
||||
| Barrier | Current Status | Required for Consumer |
|
||||
|---------|---------------|----------------------|
|
||||
| Sensor size | 12×12×19 mm (OPM) | <5×5×5 mm |
|
||||
| Magnetic shielding | Room or active coils | Integrated micro-shielding |
|
||||
| Power consumption | ~1W per sensor | <100 mW per sensor |
|
||||
| Cost per sensor | $5–15K | <$100 |
|
||||
| Ease of use | Expert setup | Self-applied in <30 seconds |
|
||||
|
||||
**Realistic timeline**: 10–15 years for consumer wearable. Near-term: clinical/professional
|
||||
devices that accept larger form factor.
|
||||
|
||||
---
|
||||
|
||||
## 11. Domain 10: Brain Network Digital Twins
|
||||
|
||||
### 11.1 The Most Advanced Concept
|
||||
|
||||
A digital twin of a person's brain network: a dynamic graph model that captures their unique
|
||||
neural topology and tracks how it evolves over time.
|
||||
|
||||
### 11.2 Architecture
|
||||
|
||||
```
|
||||
Physical Brain: Periodic OPM recordings → topology snapshots
|
||||
Digital Twin: Personalized brain graph model in RuVector
|
||||
├─ Structural connectivity (from MRI/DTI)
|
||||
├─ Functional topology (from OPM, updated periodically)
|
||||
├─ Dynamic model (predict topology transitions)
|
||||
└─ Response model (predict effects of interventions)
|
||||
|
||||
Applications:
|
||||
├─ Track brain aging trajectory
|
||||
├─ Simulate treatment responses
|
||||
├─ Personalize intervention targets
|
||||
├─ Predict cognitive decline
|
||||
└─ Optimize rehabilitation protocols
|
||||
```
|
||||
|
||||
### 11.3 Applications
|
||||
|
||||
**Tracking Brain Aging**:
|
||||
- Build topology trajectory from age 40 onwards
|
||||
- Compare individual trajectory to population norms
|
||||
- Detect accelerated aging patterns
|
||||
- Correlate with lifestyle factors (exercise, sleep, diet, social)
|
||||
- Personalized brain health optimization
|
||||
|
||||
**Simulating Treatment Responses**:
|
||||
- Patient's brain topology model + proposed treatment → predicted outcome
|
||||
- Compare: antidepressant A vs B, which normalizes topology better?
|
||||
- TMS target selection: simulate topology effects of stimulating different regions
|
||||
- Reduce trial-and-error in psychiatric treatment
|
||||
|
||||
**Personalized Neurology**:
|
||||
- Individual topology fingerprint as clinical identifier
|
||||
- Track topology before, during, and after treatment
|
||||
- Adjust treatment based on individual topology response
|
||||
- Enable precision neurology (like precision oncology)
|
||||
|
||||
**Brain Rehabilitation Modeling**:
|
||||
- Stroke recovery: model which topology trajectories lead to best outcomes
|
||||
- TBI rehabilitation: identify when topology has recovered sufficiently
|
||||
- Physical therapy optimization: correlate movement training with topology changes
|
||||
- Cognitive rehabilitation: target specific topology deficits
|
||||
|
||||
### 11.4 Data Requirements
|
||||
|
||||
| Component | Data Source | Frequency | Storage |
|
||||
|-----------|-----------|-----------|---------|
|
||||
| Structural connectome | MRI/DTI | Once (baseline) + yearly | ~1 GB |
|
||||
| Functional topology | OPM recording | Monthly 1-hour sessions | ~2 GB/session |
|
||||
| Dynamic model | Computed from above | Updated per session | ~100 MB |
|
||||
| Longitudinal trajectory | Accumulated | Growing database | ~50 GB/decade |
|
||||
|
||||
### 11.5 RuVector's Role
|
||||
|
||||
RuVector provides the embedding space for storing and comparing brain topology states:
|
||||
- Each session → set of topology embeddings stored in RuVector memory
|
||||
- Nearest-neighbor search: find past states most similar to current
|
||||
- Trajectory analysis: is the topology trajectory trending toward health or disease?
|
||||
- Cross-subject comparison: find patients with similar topology profiles
|
||||
- HNSW indexing: fast retrieval from growing longitudinal database
|
||||
|
||||
---
|
||||
|
||||
## 12. Where Dynamic Mincut Becomes Unique
|
||||
|
||||
### 12.1 Beyond Deep Learning
|
||||
|
||||
Most brain decoding systems use deep learning exclusively: neural signals → neural network →
|
||||
output labels. The model is a black box that maps input patterns to outputs.
|
||||
|
||||
Dynamic mincut adds **structural intelligence**: instead of pattern matching, it computes
|
||||
a mathematically precise property of the brain's connectivity graph.
|
||||
|
||||
### 12.2 The Key Question Shift
|
||||
|
||||
| Traditional Approach | Mincut Approach |
|
||||
|---------------------|-----------------|
|
||||
| "What is the signal?" | "Where does the network break?" |
|
||||
| Pattern matching | Structural analysis |
|
||||
| Requires large training data | Requires graph construction |
|
||||
| Black box | Interpretable (the cut is visible) |
|
||||
| Content-dependent | Content-independent |
|
||||
| Subject-specific | More transferable |
|
||||
|
||||
### 12.3 Interpretability Advantage
|
||||
|
||||
When a deep learning model classifies a brain state, explaining *why* it made that
|
||||
classification is difficult (interpretability problem). When mincut identifies a network
|
||||
partition, the explanation is inherent: "These brain regions disconnected from those brain
|
||||
regions." A clinician can directly inspect the partition and relate it to known functional
|
||||
neuroanatomy.
|
||||
|
||||
### 12.4 Mathematical Properties
|
||||
|
||||
Mincut has well-defined mathematical properties that deep learning lacks:
|
||||
- **Duality**: Max-flow/min-cut theorem provides dual interpretation
|
||||
- **Stability**: small perturbations produce small changes in cut value
|
||||
- **Monotonicity**: adding edges can only decrease mincut
|
||||
- **Submodularity**: enables efficient optimization
|
||||
- **Spectral connection**: Cheeger inequality links cut to graph Laplacian eigenvalues
|
||||
|
||||
These properties provide formal guarantees about the behavior of the analysis, unlike
|
||||
neural network classifiers which can fail unpredictably.
|
||||
|
||||
---
|
||||
|
||||
## 13. The Most Powerful Future Use — Google Maps for Cognition
|
||||
|
||||
### 13.1 The Vision
|
||||
|
||||
A real-time neural topology map. Think of it like Google Maps for the brain:
|
||||
|
||||
| Google Maps | Brain Topology Observatory |
|
||||
|------------|--------------------------|
|
||||
| Roads and highways | Neural pathways |
|
||||
| Traffic flow | Information flow |
|
||||
| Districts and neighborhoods | Functional brain modules |
|
||||
| Traffic jams | Processing bottlenecks |
|
||||
| Road closures | Disconnected pathways |
|
||||
| Construction zones | Reorganizing networks |
|
||||
| Rush hour patterns | Cognitive state patterns |
|
||||
| Navigation routing | Information routing |
|
||||
|
||||
### 13.2 What You Would See
|
||||
|
||||
A real-time display showing:
|
||||
1. **Brain regions** as nodes, colored by activity level
|
||||
2. **Connections** as edges, thickness proportional to coupling strength
|
||||
3. **Module boundaries** highlighted by mincut analysis
|
||||
4. **State transitions** animated as boundaries shift
|
||||
5. **Timeline** showing topology history
|
||||
6. **Anomaly markers** where topology deviates from baseline
|
||||
|
||||
### 13.3 How This Changes Neuroscience
|
||||
|
||||
Current neuroscience is like having satellite photos of a city — you see the buildings but
|
||||
not the traffic. This observatory adds the traffic layer: real-time flow, congestion,
|
||||
routing, and reorganization.
|
||||
|
||||
**Questions that become answerable**:
|
||||
- Which brain networks activate first during decision-making?
|
||||
- How does the network reorganize during insight?
|
||||
- What topology predicts memory formation success?
|
||||
- How does anesthesia progressively disconnect brain modules?
|
||||
- What is the topology of consciousness?
|
||||
|
||||
---
|
||||
|
||||
## 14. Hard Reality Check
|
||||
|
||||
### 14.1 Three Things That Determine Success
|
||||
|
||||
1. **Sensor fidelity**: SNR at the measurement point sets the information ceiling. Current
|
||||
OPMs: 7–15 fT/√Hz, adequate for cortical sources, marginal for deep structures.
|
||||
|
||||
2. **Signal-to-noise ratio in practice**: Environmental noise, physiological artifacts, and
|
||||
movement artifacts degrade achievable SNR. Magnetic shielding is currently required.
|
||||
|
||||
3. **Subject-specific calibration**: While topology features are more transferable than
|
||||
content features, some individual calibration is still needed for source localization
|
||||
and parcellation mapping.
|
||||
|
||||
### 14.2 What Must Improve
|
||||
|
||||
| Technology | Current | Required for Clinical Use | Timeline |
|
||||
|-----------|---------|--------------------------|----------|
|
||||
| OPM sensitivity | 7–15 fT/√Hz | 3–5 fT/√Hz | 2–3 years |
|
||||
| Magnetic shielding | Room-scale | Portable/head-mounted | 5–7 years |
|
||||
| Sensor cost | $5–15K each | $500–1K each | 5–10 years |
|
||||
| Real-time processing | Research prototype | Clinical-grade software | 2–4 years |
|
||||
| Normative database | Small research studies | 10,000+ subjects | 5–8 years |
|
||||
|
||||
### 14.3 Honest Feasibility Assessment
|
||||
|
||||
| Domain | Technical Feasibility | Timeline | Market Size |
|
||||
|--------|---------------------|----------|-------------|
|
||||
| 1. Disease detection | High | 3–5 years to pilot | $10B+ |
|
||||
| 2. BCI | Medium-High | 2–4 years to prototype | $5B |
|
||||
| 3. Cognitive monitoring | High | 1–3 years to demo | $2B |
|
||||
| 4. Mental health dx | Medium | 4–7 years to validate | $8B |
|
||||
| 5. Neurofeedback | Medium-High | 2–4 years to product | $1B |
|
||||
| 6. Dream/imagination | Low | 10+ years | Unknown |
|
||||
| 7. Cognitive research | High | 1–2 years to use | $500M (grants) |
|
||||
| 8. HCI | Medium | 5–10 years to product | $3B |
|
||||
| 9. Wearables | Low-Medium | 10–15 years | $20B+ |
|
||||
| 10. Digital twins | Low-Medium | 7–12 years | $5B+ |
|
||||
|
||||
---
|
||||
|
||||
## 15. Strategic Roadmap
|
||||
|
||||
### Phase 1: Research Platform (Year 1–2)
|
||||
|
||||
**Goal**: Demonstrate real-time brain topology tracking from OPM-MEG data.
|
||||
|
||||
**Deliverables**:
|
||||
- Software pipeline: OPM data → connectivity graph → mincut analysis → visualization
|
||||
- Proof-of-concept: distinguish rest/task/sleep from topology features
|
||||
- RuVector integration: longitudinal topology tracking across sessions
|
||||
- Publication: first paper on real-time mincut-based brain topology analysis
|
||||
|
||||
**Hardware**: 32-channel OPM system in magnetically shielded room
|
||||
**Cost**: ~$200K (sensors) + $300K (shielding) + $100K (computing) = ~$600K
|
||||
**Team**: 3–5 researchers (signal processing, neuroscience, software engineering)
|
||||
|
||||
### Phase 2: Clinical Validation (Year 2–4)
|
||||
|
||||
**Goal**: Validate topology biomarkers against clinical diagnoses.
|
||||
|
||||
**Deliverables**:
|
||||
- Clinical study: 100+ patients with known neurological conditions
|
||||
- Normative database: 500+ healthy controls
|
||||
- Sensitivity/specificity for each disease topology signature
|
||||
- Regulatory pre-submission meeting with FDA
|
||||
|
||||
**Applications to validate**:
|
||||
1. Epilepsy seizure prediction (most clear-cut clinical signal)
|
||||
2. Alzheimer's early detection (largest market need)
|
||||
3. Cognitive workload monitoring (simplest to commercialize)
|
||||
|
||||
### Phase 3: Product Development (Year 3–6)
|
||||
|
||||
**Goal**: First commercial topology monitoring system.
|
||||
|
||||
**Two parallel tracks**:
|
||||
1. **Clinical diagnostic**: OPM + topology software for hospitals
|
||||
2. **Professional monitoring**: simplified system for aviation/military
|
||||
|
||||
**Commercialization priorities**:
|
||||
- Cognitive workload monitoring (defense/aviation contracts) — fastest revenue
|
||||
- Epilepsy topology monitoring (clinical need, clear regulatory path) — largest impact
|
||||
- Brain health assessment (wellness market) — largest eventual market
|
||||
|
||||
### Phase 4: Platform Expansion (Year 5–10)
|
||||
|
||||
**Goal**: General-purpose brain topology platform.
|
||||
|
||||
**Capabilities**:
|
||||
- Digital twin construction and tracking
|
||||
- Treatment response prediction
|
||||
- Neurofeedback with topology targets
|
||||
- Consumer wearable (as sensor technology miniaturizes)
|
||||
|
||||
---
|
||||
|
||||
## 16. Two Strategic Questions
|
||||
|
||||
### Question 1: Research Platform vs. Commercial Product?
|
||||
|
||||
**Answer**: Start as research platform, spin into commercial products.
|
||||
|
||||
The RuVector + mincut core engine is the reusable technology. It should be:
|
||||
- Open-source for research adoption → builds community and validation
|
||||
- Licensed commercially for clinical and professional applications
|
||||
- The research platform generates the clinical evidence needed for commercial products
|
||||
|
||||
### Question 2: Non-Invasive Only vs. Clinical Implant Research?
|
||||
|
||||
**Answer**: Non-invasive first, implant collaboration later.
|
||||
|
||||
**Why non-invasive is the right starting point**:
|
||||
1. Mincut topology analysis needs *breadth* of coverage (many regions), which non-invasive
|
||||
excels at
|
||||
2. Implants provide *depth* (single neuron) but only from tiny patches — the opposite of
|
||||
what topology analysis needs
|
||||
3. OPM-MEG fidelity is sufficient for network-level topology analysis
|
||||
4. Regulatory pathway is simpler for non-invasive devices
|
||||
5. Market is larger (no surgery required)
|
||||
|
||||
**Future implant collaboration**:
|
||||
Once the topology framework is validated non-invasively, combine with implant data for:
|
||||
- Ground-truth validation of topology features
|
||||
- Hybrid decoding: topology (non-invasive) + content (implant)
|
||||
- Closed-loop stimulation guided by topology analysis
|
||||
|
||||
---
|
||||
|
||||
## 17. Conclusion
|
||||
|
||||
The ten application domains for a brain state observatory are not speculative science fiction.
|
||||
They are engineering challenges with clear technical requirements, identifiable markets, and
|
||||
realistic development timelines. The enabling technologies — OPM sensors, graph algorithms,
|
||||
RuVector memory, dynamic mincut — exist today or are within reach.
|
||||
|
||||
The strategic insight is this: while the rest of the field races to decode brain *content*
|
||||
(what people think, see, imagine), there is an entirely unexplored dimension of brain
|
||||
*structure* (how networks organize, reorganize, and degrade). Dynamic mincut analysis is
|
||||
the mathematical tool that makes this dimension measurable.
|
||||
|
||||
The most interesting frontier idea remains: combine quantum magnetometers, RuVector neural
|
||||
memory, and dynamic mincut coherence detection to build a topological brain observatory that
|
||||
measures how cognition organizes itself in real time. That is genuinely unexplored territory,
|
||||
and it could fundamentally change neuroscience.
|
||||
|
||||
---
|
||||
|
||||
*This document is the applications capstone of the RF Topological Sensing research series.
|
||||
It maps ten application domains for the RuVector + dynamic mincut brain state observatory,
|
||||
with honest feasibility assessment and a phased strategic roadmap.*
|
||||
@@ -0,0 +1,934 @@
|
||||
# Quantum-Level Sensors for RF Topological Sensing
|
||||
|
||||
## SOTA Research Document — RF Topological Sensing Series (11/12)
|
||||
|
||||
**Date**: 2026-03-08
|
||||
**Domain**: Quantum Sensing × RF Topology × Graph-Based Detection
|
||||
**Status**: Research Survey
|
||||
|
||||
---
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
Classical RF sensing using ESP32 WiFi mesh nodes operates at milliwatt power levels with
|
||||
sensitivity limited by thermal noise floors (~-90 dBm). Quantum sensors offer fundamentally
|
||||
different detection mechanisms that can surpass classical limits by orders of magnitude,
|
||||
potentially transforming RF topological sensing from room-scale detection to single-photon
|
||||
field measurement.
|
||||
|
||||
This document surveys quantum sensing technologies relevant to RF topological sensing,
|
||||
evaluates their integration potential with the existing RuVector/mincut architecture, and
|
||||
identifies near-term and long-term opportunities.
|
||||
|
||||
---
|
||||
|
||||
## 2. Quantum Sensing Fundamentals
|
||||
|
||||
### 2.1 Nitrogen-Vacancy (NV) Centers in Diamond
|
||||
|
||||
NV centers are point defects in diamond crystal lattice where a nitrogen atom replaces a
|
||||
carbon atom adjacent to a vacancy. Key properties:
|
||||
|
||||
- **Sensitivity**: ~1 pT/√Hz at room temperature for magnetic fields
|
||||
- **Operating temperature**: Room temperature (unique advantage)
|
||||
- **Frequency range**: DC to ~10 GHz (microwave)
|
||||
- **Spatial resolution**: Nanometer-scale (single NV) to micrometer (ensemble)
|
||||
- **Detection mechanism**: Optically detected magnetic resonance (ODMR)
|
||||
|
||||
```
|
||||
Diamond Crystal with NV Center:
|
||||
|
||||
C---C---C---C
|
||||
| | | |
|
||||
C---N V---C N = Nitrogen atom
|
||||
| | | V = Vacancy
|
||||
C---C---C---C C = Carbon atoms
|
||||
| | | |
|
||||
C---C---C---C
|
||||
|
||||
ODMR Protocol:
|
||||
Green Laser → NV → Red Fluorescence
|
||||
↕
|
||||
Microwave Drive
|
||||
|
||||
Resonance frequency shifts with local B-field
|
||||
ΔfNV = γNV × B_local
|
||||
γNV = 28 GHz/T
|
||||
```
|
||||
|
||||
### 2.2 Superconducting Quantum Interference Devices (SQUIDs)
|
||||
|
||||
- **Sensitivity**: ~1 fT/√Hz (femtotesla — 1000× better than NV)
|
||||
- **Operating temperature**: 4 K (liquid helium) or 77 K (high-Tc)
|
||||
- **Frequency range**: DC to ~1 GHz
|
||||
- **Detection mechanism**: Josephson junction flux quantization
|
||||
- **Limitation**: Requires cryogenic cooling
|
||||
|
||||
```
|
||||
SQUID Loop:
|
||||
|
||||
┌──────[JJ1]──────┐
|
||||
│ │ JJ = Josephson Junction
|
||||
│ Φ_ext → │ Φ = Magnetic flux
|
||||
│ (flux) │
|
||||
│ │ V = Φ₀/(2π) × dφ/dt
|
||||
└──────[JJ2]──────┘ Φ₀ = 2.07 × 10⁻¹⁵ Wb
|
||||
|
||||
Critical current: Ic = 2I₀|cos(πΦ_ext/Φ₀)|
|
||||
Voltage oscillates with period Φ₀
|
||||
```
|
||||
|
||||
### 2.3 Rydberg Atom Sensors
|
||||
|
||||
Atoms excited to high principal quantum number (n > 30) become extraordinarily sensitive
|
||||
to electric fields:
|
||||
|
||||
- **Sensitivity**: ~1 µV/m/√Hz (electric field)
|
||||
- **Operating temperature**: Room temperature (vapor cell)
|
||||
- **Frequency range**: DC to THz (broadband, tunable)
|
||||
- **Detection mechanism**: Electromagnetically Induced Transparency (EIT)
|
||||
- **Key advantage**: Self-calibrated, SI-traceable (no calibration needed)
|
||||
|
||||
```
|
||||
Rydberg EIT Level Scheme:
|
||||
|
||||
|r⟩ -------- Rydberg state (n~50) ← RF field couples |r⟩↔|r'⟩
|
||||
↕ Ωc (coupling laser)
|
||||
|e⟩ -------- Excited state
|
||||
↕ Ωp (probe laser)
|
||||
|g⟩ -------- Ground state
|
||||
|
||||
Without RF: EIT window → transparent to probe
|
||||
With RF: Autler-Townes splitting → absorption changes
|
||||
|
||||
Splitting: Ω_RF = μ_rr' × E_RF / ℏ
|
||||
where μ_rr' = n² × e × a₀ (scales as n²!)
|
||||
```
|
||||
|
||||
### 2.4 Atomic Magnetometers
|
||||
|
||||
Spin-exchange relaxation-free (SERF) magnetometers using alkali vapor:
|
||||
|
||||
- **Sensitivity**: ~0.16 fT/√Hz (best demonstrated)
|
||||
- **Operating temperature**: ~150°C (heated vapor cell)
|
||||
- **Frequency range**: DC to ~1 kHz
|
||||
- **Size**: Can be miniaturized to chip-scale (CSAM)
|
||||
- **Limitation**: Low bandwidth, requires magnetic shielding
|
||||
|
||||
### 2.5 Comparison Table
|
||||
|
||||
| Sensor Type | Sensitivity | Temp | Bandwidth | Size | Cost Est. |
|
||||
|------------|-------------|------|-----------|------|-----------|
|
||||
| NV Diamond | ~1 pT/√Hz | 300K | DC-10 GHz | cm | $1K-10K |
|
||||
| SQUID | ~1 fT/√Hz | 4-77K | DC-1 GHz | cm | $10K-100K |
|
||||
| Rydberg | ~1 µV/m/√Hz | 300K | DC-THz | 10 cm | $5K-50K |
|
||||
| SERF | ~0.16 fT/√Hz | 420K | DC-1 kHz | cm | $5K-50K |
|
||||
| ESP32 (classical) | ~-90 dBm | 300K | 2.4/5 GHz | cm | $5 |
|
||||
|
||||
---
|
||||
|
||||
## 3. Quantum-Enhanced RF Detection
|
||||
|
||||
### 3.1 Classical vs Quantum Noise Limits
|
||||
|
||||
Classical RF detection is limited by thermal (Johnson-Nyquist) noise:
|
||||
|
||||
```
|
||||
Classical thermal noise floor:
|
||||
P_noise = k_B × T × B
|
||||
|
||||
At T = 300K, B = 20 MHz (WiFi channel):
|
||||
P_noise = 1.38e-23 × 300 × 20e6 = 8.3 × 10⁻¹⁴ W
|
||||
P_noise = -101 dBm
|
||||
|
||||
Shot noise limit (coherent state):
|
||||
ΔE = √(ℏω/(2ε₀V)) per photon
|
||||
SNR_shot ∝ √N_photons
|
||||
|
||||
Heisenberg limit (entangled state):
|
||||
SNR_Heisenberg ∝ N_photons
|
||||
|
||||
Quantum advantage: √N improvement over shot noise
|
||||
For N = 10⁶ photons → 1000× SNR improvement
|
||||
```
|
||||
|
||||
### 3.2 Quantum Advantage Regimes
|
||||
|
||||
The quantum advantage for RF sensing depends on the signal regime:
|
||||
|
||||
| Regime | Classical | Quantum | Advantage |
|
||||
|--------|-----------|---------|-----------|
|
||||
| Strong signal (>-60 dBm) | Adequate | Unnecessary | None |
|
||||
| Medium (-60 to -90 dBm) | Noisy | Cleaner | 10-100× SNR |
|
||||
| Weak (<-90 dBm) | Undetectable | Detectable | Enabling |
|
||||
| Single-photon | Impossible | Feasible | Infinite |
|
||||
|
||||
For RF topological sensing, the quantum advantage is most relevant for:
|
||||
- Detecting very subtle field perturbations (breathing, heartbeat)
|
||||
- Sensing through walls or at extended range
|
||||
- Distinguishing multiple overlapping perturbations
|
||||
|
||||
### 3.3 Quantum Noise Reduction Techniques
|
||||
|
||||
**Squeezed States**: Reduce noise in one quadrature at expense of other:
|
||||
```
|
||||
ΔX₁ × ΔX₂ ≥ ℏ/2
|
||||
Squeeze X₁: ΔX₁ = e⁻ʳ × √(ℏ/2) (reduced)
|
||||
ΔX₂ = e⁺ʳ × √(ℏ/2) (increased)
|
||||
|
||||
For r = 2 (17.4 dB squeezing):
|
||||
Noise reduction in amplitude: 7.4×
|
||||
Demonstrated: 15 dB squeezing (LIGO)
|
||||
```
|
||||
|
||||
**Quantum Error Correction**: Protect quantum states from decoherence:
|
||||
- Repetition codes for phase noise
|
||||
- Surface codes for general errors
|
||||
- Overhead: ~1000 physical qubits per logical qubit (current)
|
||||
|
||||
---
|
||||
|
||||
## 4. Rydberg Atom RF Sensors — Deep Dive
|
||||
|
||||
### 4.1 Broadband RF Detection via EIT
|
||||
|
||||
Rydberg atoms provide the most promising near-term quantum RF sensor for topological
|
||||
sensing because:
|
||||
|
||||
1. **Room temperature operation** — no cryogenics
|
||||
2. **Broadband** — single vapor cell covers MHz to THz by tuning laser wavelength
|
||||
3. **Self-calibrated** — response depends only on atomic constants
|
||||
4. **Compact** — vapor cell can be cm-scale
|
||||
|
||||
```
|
||||
Rydberg Sensor Architecture:
|
||||
|
||||
┌─────────────────────────────┐
|
||||
│ Cesium Vapor Cell │
|
||||
│ │
|
||||
│ Probe (852nm) ───────→ │──→ Photodetector
|
||||
│ Coupling (509nm) ───→ │
|
||||
│ │
|
||||
│ ↕ RF field enters │
|
||||
└─────────────────────────────┘
|
||||
|
||||
Frequency tuning:
|
||||
n=30: ~300 GHz transitions
|
||||
n=50: ~50 GHz transitions
|
||||
n=70: ~10 GHz transitions (WiFi band!)
|
||||
n=100: ~1 GHz transitions
|
||||
```
|
||||
|
||||
### 4.2 Sensitivity at WiFi Frequencies
|
||||
|
||||
For 2.4 GHz detection using Rydberg states near n=70:
|
||||
|
||||
```
|
||||
Transition dipole moment:
|
||||
μ = n² × e × a₀ ≈ 70² × 1.6e-19 × 5.3e-11
|
||||
μ ≈ 4.1 × 10⁻²⁶ C·m
|
||||
|
||||
Minimum detectable field:
|
||||
E_min = ℏ × Γ / (2μ)
|
||||
where Γ = EIT linewidth ≈ 1 MHz
|
||||
|
||||
E_min ≈ 1.05e-34 × 2π × 1e6 / (2 × 4.1e-26)
|
||||
E_min ≈ 8 µV/m
|
||||
|
||||
Compare to ESP32 sensitivity: ~1 mV/m
|
||||
Quantum advantage: ~125× in field sensitivity
|
||||
```
|
||||
|
||||
### 4.3 NIST and Army Research Lab Advances
|
||||
|
||||
Key milestones in Rydberg RF sensing:
|
||||
- **2012**: First demonstration of Rydberg EIT for RF measurement (Sedlacek et al.)
|
||||
- **2018**: Broadband electric field sensing 1-500 GHz (Holloway et al., NIST)
|
||||
- **2020**: Rydberg atom receiver for AM/FM radio signals
|
||||
- **2022**: Multi-band simultaneous detection using multiple Rydberg transitions
|
||||
- **2024**: Chip-scale vapor cells with integrated photonics
|
||||
- **2025**: Field demonstrations of Rydberg receivers for communications
|
||||
|
||||
### 4.4 Integration with ESP32 Mesh
|
||||
|
||||
```
|
||||
Hybrid Rydberg-ESP32 Architecture:
|
||||
|
||||
Classical Layer (ESP32 mesh):
|
||||
┌────┐ ┌────┐ ┌────┐
|
||||
│ESP1│────│ESP2│────│ESP3│ 120 classical edges
|
||||
└────┘ └────┘ └────┘ CSI coherence weights
|
||||
│ │ │
|
||||
│ ┌────┴────┐ │
|
||||
└────│Rydberg │────┘ Quantum sensor node
|
||||
│ Sensor │ High-sensitivity edges
|
||||
└─────────┘
|
||||
|
||||
The Rydberg sensor provides:
|
||||
1. Ultra-sensitive reference measurements
|
||||
2. Ground truth calibration for classical edges
|
||||
3. Detection of sub-threshold perturbations
|
||||
4. Phase reference for coherence estimation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Quantum Illumination for Object Detection
|
||||
|
||||
### 5.1 Lloyd's Quantum Illumination Protocol
|
||||
|
||||
Quantum illumination uses entangled photon pairs to detect objects in noisy environments:
|
||||
|
||||
```
|
||||
Protocol:
|
||||
1. Generate entangled signal-idler pair: |Ψ⟩ = Σ cₙ|n⟩_S|n⟩_I
|
||||
2. Send signal photon toward target, keep idler
|
||||
3. Collect reflected signal (buried in thermal noise)
|
||||
4. Joint measurement on returned signal + stored idler
|
||||
|
||||
Classical detection: SNR = N_S / N_B
|
||||
Quantum detection: SNR = N_S × (N_B + 1) / N_B
|
||||
|
||||
Advantage: 6 dB in error exponent (factor of 4)
|
||||
|
||||
Critical: Advantage persists even when entanglement is destroyed
|
||||
by the noisy channel (unlike most quantum protocols)
|
||||
```
|
||||
|
||||
### 5.2 Microwave Quantum Illumination
|
||||
|
||||
For RF topological sensing at 2.4 GHz:
|
||||
|
||||
```
|
||||
Microwave entangled source:
|
||||
Josephson Parametric Amplifier (JPA)
|
||||
→ Generates entangled microwave-microwave pairs
|
||||
→ Or microwave-optical pairs (for optical idler storage)
|
||||
|
||||
Challenge: thermal photon number at 2.4 GHz, 300K:
|
||||
n_th = 1/(exp(hf/kT) - 1) = 1/(exp(4.8e-5) - 1) ≈ 2600
|
||||
|
||||
Background: ~2600 thermal photons per mode
|
||||
→ Classical detection hopeless for single-photon signals
|
||||
→ Quantum illumination still provides 6 dB advantage
|
||||
```
|
||||
|
||||
### 5.3 Application to RF Topology
|
||||
|
||||
Quantum illumination could enhance RF topological sensing by:
|
||||
- Detecting very weak reflections from small objects
|
||||
- Operating in high-noise environments (industrial, urban)
|
||||
- Distinguishing target-reflected signals from multipath clutter
|
||||
- Providing phase-coherent measurements for graph edge weights
|
||||
|
||||
---
|
||||
|
||||
## 6. Quantum Graph Theory
|
||||
|
||||
### 6.1 Quantum Walks on Graphs
|
||||
|
||||
Quantum walks are the quantum analog of random walks, with superposition and interference:
|
||||
|
||||
```
|
||||
Continuous-time quantum walk on graph G:
|
||||
|ψ(t)⟩ = e^{-iHt} |ψ(0)⟩
|
||||
where H = adjacency matrix A or Laplacian L
|
||||
|
||||
Key property: Quantum walk spreads quadratically faster
|
||||
Classical: ⟨x²⟩ ~ t (diffusive)
|
||||
Quantum: ⟨x²⟩ ~ t² (ballistic)
|
||||
|
||||
For graph topology detection:
|
||||
- Walk dynamics encode graph structure
|
||||
- Interference patterns reveal symmetries
|
||||
- Hitting times indicate connectivity
|
||||
```
|
||||
|
||||
### 6.2 Quantum Minimum Cut
|
||||
|
||||
**Grover-accelerated graph search**:
|
||||
```
|
||||
Classical min-cut (Stoer-Wagner): O(VE + V² log V)
|
||||
For V=16, E=120: ~4,000 operations
|
||||
|
||||
Quantum search for min-cut:
|
||||
Use Grover's algorithm to search over cuts
|
||||
Number of possible cuts: 2^V = 2^16 = 65,536
|
||||
|
||||
Classical brute force: O(2^V) = 65,536 evaluations
|
||||
Quantum (Grover): O(√(2^V)) = 256 evaluations
|
||||
|
||||
Quadratic speedup for brute-force approach
|
||||
|
||||
However: For V=16, Stoer-Wagner (4,000 ops) beats Grover (256 oracle calls)
|
||||
because each oracle call has overhead
|
||||
|
||||
Quantum advantage threshold: V > ~100 nodes
|
||||
```
|
||||
|
||||
**Quantum spectral analysis**:
|
||||
```
|
||||
Quantum Phase Estimation (QPE) for graph Laplacian:
|
||||
Input: L = D - A (graph Laplacian)
|
||||
Output: eigenvalues λ₁ ≤ λ₂ ≤ ... ≤ λ_V
|
||||
|
||||
Fiedler value λ₂ → algebraic connectivity
|
||||
Cheeger inequality: λ₂/2 ≤ h(G) ≤ √(2λ₂)
|
||||
where h(G) = min-cut / min-volume (Cheeger constant)
|
||||
|
||||
QPE complexity: O(poly(log V)) per eigenvalue
|
||||
Classical: O(V³) for full eigendecomposition
|
||||
|
||||
Quantum advantage for spectral analysis: exponential
|
||||
for V >> 100
|
||||
```
|
||||
|
||||
### 6.3 Quantum Graph Partitioning
|
||||
|
||||
```
|
||||
Variational Quantum Eigensolver (VQE) for normalized cut:
|
||||
|
||||
Minimize: NCut = cut(A,B) × (1/vol(A) + 1/vol(B))
|
||||
|
||||
Encode as QUBO:
|
||||
min x^T Q x where x ∈ {0,1}^V
|
||||
Q_ij = -w_ij + d_i × δ_ij × balance_penalty
|
||||
|
||||
Map to Ising Hamiltonian:
|
||||
H = Σ_ij J_ij σ_i^z σ_j^z + Σ_i h_i σ_i^z
|
||||
|
||||
Solve with:
|
||||
- VQE (gate-based): variational ansatz circuit
|
||||
- QAOA: alternating cost/mixer unitaries
|
||||
- Quantum annealing (D-Wave): native QUBO solver
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Hybrid Classical-Quantum RF Sensing Architecture
|
||||
|
||||
### 7.1 Where Quantum Advantage Matters
|
||||
|
||||
Not every edge in the RF sensing graph benefits from quantum sensing. The advantage
|
||||
is concentrated in specific scenarios:
|
||||
|
||||
| Scenario | Classical | Quantum | Benefit |
|
||||
|----------|-----------|---------|---------|
|
||||
| Strong LOS links | Adequate | Overkill | None |
|
||||
| Weak NLOS links | Noisy/lost | Detectable | Enables new edges |
|
||||
| Sub-threshold perturbations | Invisible | Detectable | Breathing, heartbeat |
|
||||
| Phase coherence measurement | Clock-limited | Fundamental | Better edge weights |
|
||||
| Multi-target disambiguation | Ambiguous | Resolvable | More accurate cuts |
|
||||
|
||||
### 7.2 Hybrid Architecture
|
||||
|
||||
```
|
||||
Three-Tier Hybrid Sensing:
|
||||
|
||||
Tier 1: ESP32 Classical Mesh (16 nodes, $80 total)
|
||||
┌─────────────────────────────────────┐
|
||||
│ Standard CSI extraction │
|
||||
│ 120 TX-RX edges │
|
||||
│ ~30-60 cm resolution │
|
||||
│ Person-scale detection │
|
||||
└──────────────┬──────────────────────┘
|
||||
│
|
||||
Tier 2: NV Diamond Enhancement (4 nodes, ~$20K)
|
||||
┌──────────────┴──────────────────────┐
|
||||
│ pT-level magnetic field sensing │
|
||||
│ Room-temperature operation │
|
||||
│ Complements RF with B-field edges │
|
||||
│ Breathing/heartbeat detection │
|
||||
└──────────────┬──────────────────────┘
|
||||
│
|
||||
Tier 3: Rydberg Reference (1 node, ~$50K)
|
||||
┌──────────────┴──────────────────────┐
|
||||
│ µV/m electric field sensitivity │
|
||||
│ Self-calibrated SI-traceable │
|
||||
│ Ground truth for classical edges │
|
||||
│ Sub-threshold perturbation detect │
|
||||
└─────────────────────────────────────┘
|
||||
|
||||
Graph construction:
|
||||
G_hybrid = G_classical ∪ G_magnetic ∪ G_quantum
|
||||
|
||||
Edge weight fusion:
|
||||
w_ij = α × w_classical + β × w_magnetic + γ × w_quantum
|
||||
where α + β + γ = 1, learned per-edge
|
||||
```
|
||||
|
||||
### 7.3 Quantum-Enhanced Edge Weight Computation
|
||||
|
||||
```
|
||||
Classical edge weight (ESP32):
|
||||
w_ij = coherence(CSI_i→j)
|
||||
Noise floor: ~-90 dBm
|
||||
Phase noise: ~5° RMS (clock drift limited)
|
||||
|
||||
Quantum-enhanced edge weight:
|
||||
w_ij = f(CSI_ij, B_field_ij, E_field_ij)
|
||||
|
||||
NV contribution:
|
||||
- Local magnetic field map at pT resolution
|
||||
- Detects metallic object perturbations
|
||||
- Measures eddy current signatures
|
||||
|
||||
Rydberg contribution:
|
||||
- Electric field at µV/m resolution
|
||||
- Phase-accurate reference measurement
|
||||
- Calibrates classical CSI phase errors
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Quantum Coherence for RF Field Mapping
|
||||
|
||||
### 8.1 Decoherence as Environmental Sensor
|
||||
|
||||
Quantum sensors naturally measure their environment through decoherence:
|
||||
|
||||
```
|
||||
NV Center Decoherence:
|
||||
T₁ (spin-lattice relaxation): ~6 ms at 300K
|
||||
T₂ (spin-spin dephasing): ~1 ms at 300K
|
||||
T₂* (inhomogeneous): ~1 µs
|
||||
|
||||
Environmental perturbation → T₂* change
|
||||
|
||||
Sensitivity:
|
||||
ΔB_min = (1/γ) × 1/(T₂* × √(η × T_meas))
|
||||
|
||||
where η = photon collection efficiency
|
||||
T_meas = measurement time
|
||||
|
||||
At η=0.1, T_meas=1s:
|
||||
ΔB_min ≈ 1 pT
|
||||
```
|
||||
|
||||
The key insight: **decoherence signatures encode environmental structure**. Different
|
||||
objects and materials produce different decoherence profiles:
|
||||
|
||||
| Object | Decoherence Mechanism | Signature |
|
||||
|--------|----------------------|-----------|
|
||||
| Metal | Eddy currents, Johnson noise | T₂* reduction, broadband |
|
||||
| Human body | Ionic currents, diamagnetism | T₁ modulation, low-freq |
|
||||
| Water | Diamagnetic susceptibility | Subtle T₂ shift |
|
||||
| Electronics | EM emission | Discrete frequency peaks |
|
||||
|
||||
### 8.2 Quantum Fisher Information for Optimal Placement
|
||||
|
||||
```
|
||||
Quantum Fisher Information (QFI):
|
||||
F_Q(θ) = 4(⟨∂_θψ|∂_θψ⟩ - |⟨ψ|∂_θψ⟩|²)
|
||||
|
||||
Quantum Cramér-Rao Bound:
|
||||
Var(θ̂) ≥ 1/(N × F_Q(θ))
|
||||
|
||||
For sensor placement optimization:
|
||||
- Compute F_Q at each candidate position
|
||||
- Place quantum sensors where F_Q is maximized
|
||||
- Typically: room center, doorways, narrow passages
|
||||
|
||||
Optimal placement for V=16 classical + 4 quantum:
|
||||
┌─────────────────────────┐
|
||||
│ E E E E E E │ E = ESP32 (perimeter)
|
||||
│ │
|
||||
│ E Q Q E │ Q = Quantum sensor
|
||||
│ │ (high-FI positions)
|
||||
│ E Q Q E │
|
||||
│ │
|
||||
│ E E E E E E │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Quantum Machine Learning for RF
|
||||
|
||||
### 9.1 Variational Quantum Circuits for Graph Classification
|
||||
|
||||
```
|
||||
Quantum Graph Neural Network:
|
||||
|
||||
Input: Edge weights w_ij from RF sensing graph
|
||||
|
||||
Encoding: Amplitude encoding of adjacency matrix
|
||||
|ψ_G⟩ = Σ_ij w_ij |i⟩|j⟩ / ||w||
|
||||
|
||||
Variational circuit:
|
||||
U(θ) = Π_l [U_entangle × U_rotation(θ_l)]
|
||||
|
||||
U_rotation: R_y(θ₁) ⊗ R_y(θ₂) ⊗ ... ⊗ R_y(θ_V)
|
||||
U_entangle: CNOT cascade matching graph topology
|
||||
|
||||
Measurement: ⟨Z₁⟩ → occupancy classification
|
||||
|
||||
Training: Minimize L = Σ (y - ⟨Z₁⟩)² via parameter-shift rule
|
||||
|
||||
For V=16: Requires 16 qubits + ~100 variational parameters
|
||||
→ Within reach of current NISQ devices (IBM Eagle: 127 qubits)
|
||||
```
|
||||
|
||||
### 9.2 Quantum Kernel Methods
|
||||
|
||||
```
|
||||
Quantum kernel for CSI feature space:
|
||||
|
||||
Encode CSI vector x into quantum state: |φ(x)⟩ = U(x)|0⟩
|
||||
|
||||
Kernel: K(x, x') = |⟨φ(x)|φ(x')⟩|²
|
||||
|
||||
Properties:
|
||||
- Maps to exponentially large Hilbert space
|
||||
- Can capture correlations classical kernels miss
|
||||
- Computed on quantum hardware, used in classical SVM/GP
|
||||
|
||||
For edge classification (stable/unstable/transitioning):
|
||||
- Encode temporal CSI window as quantum state
|
||||
- Quantum kernel captures phase correlations
|
||||
- Classical SVM classifies using quantum kernel values
|
||||
```
|
||||
|
||||
### 9.3 Quantum Reservoir Computing
|
||||
|
||||
```
|
||||
Quantum Reservoir for Temporal RF Patterns:
|
||||
|
||||
RF Signal → Quantum System → Measurement → Classical Readout
|
||||
|
||||
Reservoir: N coupled qubits with natural dynamics
|
||||
H_res = Σ_i h_i σ_i^z + Σ_ij J_ij σ_i^z σ_j^z + Σ_i Ω_i σ_i^x
|
||||
|
||||
Input: CSI values modulate h_i (local fields)
|
||||
Dynamics: ρ(t+1) = U × ρ(t) × U† + noise
|
||||
Output: Measure ⟨σ_i^z⟩ for all qubits → feature vector
|
||||
|
||||
Advantages for temporal RF sensing:
|
||||
- Natural temporal memory (quantum coherence)
|
||||
- No training of reservoir (only readout layer)
|
||||
- Captures non-linear temporal correlations
|
||||
- Matches temporal graph evolution naturally
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Near-Term NISQ Applications
|
||||
|
||||
### 10.1 Quantum Annealing for Graph Cuts (D-Wave)
|
||||
|
||||
```
|
||||
Min-cut as QUBO on D-Wave:
|
||||
|
||||
Variables: x_i ∈ {0,1} (node partition assignment)
|
||||
|
||||
Objective: minimize Σ_ij w_ij × x_i × (1-x_j)
|
||||
|
||||
QUBO matrix:
|
||||
Q_ij = -w_ij (off-diagonal)
|
||||
Q_ii = Σ_j w_ij (diagonal)
|
||||
|
||||
D-Wave Advantage2: 7,000+ qubits
|
||||
→ Can handle graphs up to ~3,500 nodes
|
||||
→ Our V=16 graph trivially fits
|
||||
|
||||
Practical consideration:
|
||||
- Cloud API access: ~$2K/month
|
||||
- Annealing time: ~20 µs per sample
|
||||
- 1000 samples for statistics: ~20 ms
|
||||
- Compatible with 20 Hz update rate
|
||||
|
||||
Multi-cut extension (k-way):
|
||||
Use k binary variables per node
|
||||
→ 16 × k = 48 qubits for 3-person detection
|
||||
```
|
||||
|
||||
### 10.2 VQE for Spectral Graph Analysis
|
||||
|
||||
```
|
||||
Variational Quantum Eigensolver for Laplacian spectrum:
|
||||
|
||||
Goal: Find smallest eigenvalues of L = D - A
|
||||
|
||||
Ansatz: |ψ(θ)⟩ = U(θ)|0⟩^⊗n
|
||||
|
||||
Cost: E(θ) = ⟨ψ(θ)|L|ψ(θ)⟩
|
||||
|
||||
Optimization: θ* = argmin E(θ) via classical optimizer
|
||||
|
||||
For Fiedler value (λ₂):
|
||||
1. Find ground state |v₁⟩ (constant vector, known)
|
||||
2. Constrain ⟨v₁|ψ⟩ = 0
|
||||
3. Minimize in orthogonal subspace → λ₂
|
||||
|
||||
Application: Track λ₂ over time
|
||||
- λ₂ large → graph well-connected → no obstruction
|
||||
- λ₂ drops → graph nearly disconnected → boundary detected
|
||||
- Rate of λ₂ change → speed of perturbation
|
||||
```
|
||||
|
||||
### 10.3 QAOA for Balanced Partitioning
|
||||
|
||||
```
|
||||
Quantum Approximate Optimization Algorithm:
|
||||
|
||||
Cost Hamiltonian: H_C = Σ_ij w_ij (1 - Z_i Z_j) / 2
|
||||
Mixer Hamiltonian: H_M = Σ_i X_i
|
||||
|
||||
p-layer circuit:
|
||||
|ψ(γ,β)⟩ = Π_l [e^{-iβ_l H_M} × e^{-iγ_l H_C}] |+⟩^⊗n
|
||||
|
||||
For p=1: Guaranteed approximation ratio r ≥ 0.6924 for MaxCut
|
||||
For p=3-5: Near-optimal for small graphs
|
||||
|
||||
Our V=16 graph: 16 qubits, p=3 → 96 parameters
|
||||
→ Trainable on current hardware
|
||||
→ Could provide better-than-classical cuts in some cases
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. Integration with RuVector and Mincut
|
||||
|
||||
### 11.1 Quantum-Classical Data Flow
|
||||
|
||||
```
|
||||
Integration Pipeline:
|
||||
|
||||
ESP32 Mesh Quantum Sensors
|
||||
┌──────────┐ ┌──────────┐
|
||||
│ CSI Data │ │ QSensor │
|
||||
│ 120 edges│ │ 4 nodes │
|
||||
│ 20 Hz │ │ 100 Hz │
|
||||
└────┬─────┘ └────┬─────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────────────────┐
|
||||
│ Edge Weight Fusion │
|
||||
│ │
|
||||
│ w_ij = fuse( │
|
||||
│ classical_coherence, │
|
||||
│ magnetic_perturbation, │
|
||||
│ quantum_phase_ref │
|
||||
│ ) │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────┐
|
||||
│ RfGraph Construction │
|
||||
│ G = (V_classical ∪ V_quantum, E_fused)
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────┐
|
||||
│ Hybrid Mincut │
|
||||
│ - Classical: Stoer-Wagner │
|
||||
│ - Or quantum: D-Wave QUBO │
|
||||
│ - Select based on graph size│
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────┐
|
||||
│ RuVector Temporal Store │
|
||||
│ - Graph evolution history │
|
||||
│ - Quantum measurement log │
|
||||
│ - Attention-weighted fusion │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
### 11.2 Rust Module Design
|
||||
|
||||
```rust
|
||||
/// Quantum sensor integration for RF topological sensing
|
||||
pub trait QuantumSensor: Send + Sync {
|
||||
/// Get current measurement with uncertainty
|
||||
fn measure(&self) -> QuantumMeasurement;
|
||||
|
||||
/// Sensor sensitivity in appropriate units
|
||||
fn sensitivity(&self) -> f64;
|
||||
|
||||
/// Decoherence time (characterizes environment)
|
||||
fn coherence_time(&self) -> Duration;
|
||||
}
|
||||
|
||||
pub struct QuantumMeasurement {
|
||||
pub value: f64,
|
||||
pub uncertainty: f64, // Quantum uncertainty
|
||||
pub fisher_information: f64, // QFI for this measurement
|
||||
pub timestamp: Instant,
|
||||
pub sensor_type: QuantumSensorType,
|
||||
}
|
||||
|
||||
pub enum QuantumSensorType {
|
||||
NVDiamond { t2_star: Duration },
|
||||
Rydberg { principal_n: u32, transition_freq: f64 },
|
||||
SQUID { flux_quantum: f64 },
|
||||
SERF { vapor_temp: f64 },
|
||||
}
|
||||
|
||||
/// Fuse classical and quantum edge weights
|
||||
pub trait HybridEdgeWeightFusion {
|
||||
fn fuse(
|
||||
&self,
|
||||
classical: &ClassicalEdgeWeight,
|
||||
quantum: Option<&QuantumMeasurement>,
|
||||
) -> FusedEdgeWeight;
|
||||
}
|
||||
|
||||
pub struct FusedEdgeWeight {
|
||||
pub weight: f64,
|
||||
pub confidence: f64, // Higher with quantum data
|
||||
pub classical_contribution: f64,
|
||||
pub quantum_contribution: f64,
|
||||
pub fisher_bound: f64, // QCRB on precision
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 12. Hardware Roadmap
|
||||
|
||||
### 12.1 Technology Readiness Levels
|
||||
|
||||
| Technology | Current TRL | Field-Ready | Clinical | Notes |
|
||||
|-----------|-------------|-------------|----------|-------|
|
||||
| NV Diamond magnetometer | TRL 5-6 | 2026-2028 | 2030+ | Room temp, most practical |
|
||||
| Chip-scale NV | TRL 3-4 | 2028-2030 | 2032+ | Integration with CMOS |
|
||||
| Rydberg RF receiver | TRL 4-5 | 2027-2029 | N/A | Military interest high |
|
||||
| Miniature SQUID | TRL 7-8 | Available | Available | Requires cryogenics |
|
||||
| SERF magnetometer | TRL 5-6 | 2026-2028 | 2029+ | Needs shielding |
|
||||
| Quantum annealer (D-Wave) | TRL 8-9 | Available | N/A | Cloud access now |
|
||||
| NISQ processor (IBM/Google) | TRL 6-7 | 2026+ | N/A | 1000+ qubits by 2026 |
|
||||
|
||||
### 12.2 Size, Weight, Power (SWaP) Analysis
|
||||
|
||||
```
|
||||
Current vs Projected SWaP:
|
||||
|
||||
NV Diamond Sensor (2025):
|
||||
Size: 15 × 10 × 10 cm
|
||||
Weight: 2 kg
|
||||
Power: 5 W (laser + electronics)
|
||||
|
||||
NV Diamond Sensor (2028 projected):
|
||||
Size: 5 × 3 × 3 cm
|
||||
Weight: 200 g
|
||||
Power: 1 W
|
||||
|
||||
Rydberg Vapor Cell (2025):
|
||||
Size: 20 × 15 × 15 cm
|
||||
Weight: 3 kg
|
||||
Power: 10 W (two lasers + control)
|
||||
|
||||
Chip-Scale Rydberg (2030 projected):
|
||||
Size: 3 × 3 × 1 cm
|
||||
Weight: 50 g
|
||||
Power: 0.5 W
|
||||
|
||||
Compare ESP32:
|
||||
Size: 5 × 3 × 0.5 cm
|
||||
Weight: 10 g
|
||||
Power: 0.44 W
|
||||
```
|
||||
|
||||
### 12.3 Deployment Timeline
|
||||
|
||||
```
|
||||
Phase 1 (2026): Classical-only RF topology
|
||||
- 16 ESP32 nodes
|
||||
- Stoer-Wagner mincut
|
||||
- Proof of concept
|
||||
|
||||
Phase 2 (2027-2028): Quantum-enhanced
|
||||
- 16 ESP32 + 2-4 NV diamond nodes
|
||||
- Hybrid edge weights
|
||||
- Sub-threshold detection (breathing)
|
||||
|
||||
Phase 3 (2029-2030): Full quantum integration
|
||||
- 16 ESP32 + 4 NV + 1 Rydberg
|
||||
- Quantum-classical graph fusion
|
||||
- D-Wave cloud for multi-cut optimization
|
||||
|
||||
Phase 4 (2031+): Quantum-native
|
||||
- Chip-scale quantum sensors at every node
|
||||
- On-device quantum processing
|
||||
- Room-scale coherence imaging
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 13. Open Questions and Future Directions
|
||||
|
||||
### 13.1 Fundamental Questions
|
||||
|
||||
1. **Quantum advantage threshold**: At what graph size does quantum mincut outperform
|
||||
classical? Preliminary analysis suggests V > 100, but constant factors matter.
|
||||
|
||||
2. **Decoherence as feature**: Can quantum decoherence rates serve as edge weights
|
||||
directly, bypassing classical CSI entirely?
|
||||
|
||||
3. **Entanglement distribution**: Can entangled sensor pairs provide correlated
|
||||
edge weights with fundamentally lower uncertainty?
|
||||
|
||||
4. **Quantum memory for temporal graphs**: Can quantum memory store graph evolution
|
||||
states more efficiently than classical RuVector?
|
||||
|
||||
### 13.2 Engineering Questions
|
||||
|
||||
5. **Noise budget**: In a real room with WiFi, Bluetooth, and power line interference,
|
||||
what is the practical quantum advantage?
|
||||
|
||||
6. **Calibration**: How often do quantum sensors need recalibration in field deployment?
|
||||
|
||||
7. **Cost trajectory**: When will quantum sensor nodes reach $100/unit for mass deployment?
|
||||
|
||||
8. **Hybrid optimization**: What is the optimal ratio of classical to quantum nodes
|
||||
for a given room size and detection requirement?
|
||||
|
||||
### 13.3 Application Questions
|
||||
|
||||
9. **Resolution limits**: Does quantum sensing fundamentally change the 30-60 cm
|
||||
resolution bound, or only improve SNR within the same Fresnel-limited resolution?
|
||||
|
||||
10. **Multi-room scaling**: Can quantum entanglement between rooms provide correlated
|
||||
sensing that classical links cannot?
|
||||
|
||||
11. **Adversarial robustness**: Are quantum-enhanced edge weights more robust against
|
||||
deliberate spoofing or jamming?
|
||||
|
||||
---
|
||||
|
||||
## 14. References
|
||||
|
||||
1. Degen, C.L., Reinhard, F., Cappellaro, P. (2017). "Quantum sensing." Rev. Mod. Phys. 89, 035002.
|
||||
2. Sedlacek, J.A., et al. (2012). "Microwave electrometry with Rydberg atoms in a vapour cell." Nature Physics 8, 819.
|
||||
3. Holloway, C.L., et al. (2014). "Broadband Rydberg atom-based electric-field probe." IEEE Trans. Antentic. Propag. 62, 6169.
|
||||
4. Lloyd, S. (2008). "Enhanced sensitivity of photodetection via quantum illumination." Science 321, 1463.
|
||||
5. Tan, S.H., et al. (2008). "Quantum illumination with Gaussian states." Phys. Rev. Lett. 101, 253601.
|
||||
6. Childs, A.M. (2010). "On the relationship between continuous- and discrete-time quantum walk." Commun. Math. Phys. 294, 581.
|
||||
7. Farhi, E., Goldstone, J., Gutmann, S. (2014). "A quantum approximate optimization algorithm." arXiv:1411.4028.
|
||||
8. Peruzzo, A., et al. (2014). "A variational eigenvalue solver on a photonic quantum processor." Nature Communications 5, 4213.
|
||||
9. Taylor, J.M., et al. (2008). "High-sensitivity diamond magnetometer with nanoscale resolution." Nature Physics 4, 810.
|
||||
10. Boto, E., et al. (2018). "Moving magnetoencephalography towards real-world applications with a wearable system." Nature 555, 657.
|
||||
11. Schuld, M., Killoran, N. (2019). "Quantum machine learning in feature Hilbert spaces." Phys. Rev. Lett. 122, 040504.
|
||||
|
||||
---
|
||||
|
||||
## 15. Summary
|
||||
|
||||
Quantum sensing represents a paradigm shift for RF topological sensing. While the classical
|
||||
ESP32 mesh provides adequate sensitivity for person-scale detection, quantum sensors enable:
|
||||
|
||||
1. **100-1000× sensitivity improvement** for subtle perturbations
|
||||
2. **New sensing modalities** (magnetic fields, electric fields) complementing RF
|
||||
3. **Self-calibrated measurements** via Rydberg atom standards
|
||||
4. **Quantum-accelerated graph algorithms** for larger meshes
|
||||
5. **Decoherence-based environmental sensing** as a fundamentally new edge weight source
|
||||
|
||||
The most practical near-term integration path uses NV diamond sensors (room temperature,
|
||||
pT sensitivity) as enhancement nodes within the classical ESP32 mesh, with Rydberg sensors
|
||||
providing calibration references. Quantum computing (D-Wave, NISQ) offers immediate
|
||||
value for graph cut optimization at scale.
|
||||
|
||||
The long-term vision is a quantum-native sensing mesh where every node performs quantum
|
||||
measurements, edge weights encode quantum coherence between nodes, and graph algorithms
|
||||
run on quantum hardware — a true quantum radio nervous system.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,790 @@
|
||||
# NV Diamond Magnetometers for Neural Current Detection
|
||||
|
||||
## SOTA Research Document — RF Topological Sensing Series (13/22)
|
||||
|
||||
**Date**: 2026-03-09
|
||||
**Domain**: Nitrogen-Vacancy Quantum Sensing × Neural Magnetometry × Graph Topology
|
||||
**Status**: Research Survey
|
||||
|
||||
---
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
Neurons communicate through ionic currents. Those currents generate magnetic fields — tiny
|
||||
ones, measured in femtotesla (10⁻¹⁵ T). For context, Earth's magnetic field is approximately
|
||||
50 μT, roughly 10¹⁰ times stronger than the magnetic signature of a single cortical column.
|
||||
|
||||
Detecting these fields has historically required SQUID magnetometers operating at 4 Kelvin
|
||||
inside massive liquid helium dewars. This technology, while sensitive (3–5 fT/√Hz), is
|
||||
expensive ($2–5M per system), immobile, and impractical for wearable or portable applications.
|
||||
|
||||
Nitrogen-vacancy (NV) centers in diamond offer a fundamentally different approach. These
|
||||
atomic-scale defects in diamond crystal lattice can detect magnetic fields at femtotesla
|
||||
sensitivity while operating at room temperature. They can be miniaturized to chip scale,
|
||||
fabricated in dense arrays, and integrated with standard electronics.
|
||||
|
||||
For the RuVector + dynamic mincut brain analysis architecture, NV diamond magnetometers
|
||||
represent the medium-term sensor technology that could enable portable, affordable,
|
||||
high-spatial-resolution neural topology measurement.
|
||||
|
||||
---
|
||||
|
||||
## 2. NV Center Physics
|
||||
|
||||
### 2.1 Crystal Structure and Defect Properties
|
||||
|
||||
Diamond has a face-centered cubic crystal lattice of carbon atoms. An NV center forms when:
|
||||
1. A nitrogen atom substitutes for one carbon atom
|
||||
2. An adjacent lattice site is vacant (missing carbon)
|
||||
|
||||
The resulting NV⁻ (negatively charged) defect has remarkable quantum properties:
|
||||
- Electronic spin triplet ground state (³A₂) with S = 1
|
||||
- Spin sublevels: mₛ = 0 and mₛ = ±1, split by 2.87 GHz at zero field
|
||||
- Optically addressable: 532 nm green laser excites, red fluorescence (637–800 nm) reads out
|
||||
- Spin-dependent fluorescence: mₛ = 0 is brighter than mₛ = ±1
|
||||
|
||||
This spin-dependent fluorescence is the key to magnetometry: magnetic fields shift the
|
||||
energy of the mₛ = ±1 states (Zeeman effect), which is detected as a change in
|
||||
fluorescence intensity when microwaves are swept through resonance.
|
||||
|
||||
### 2.2 Optically Detected Magnetic Resonance (ODMR)
|
||||
|
||||
The measurement protocol:
|
||||
|
||||
1. **Optical initialization**: Green laser (532 nm) pumps NV into mₛ = 0 ground state
|
||||
2. **Microwave interrogation**: Sweep microwave frequency around 2.87 GHz
|
||||
3. **Optical readout**: Monitor red fluorescence intensity
|
||||
4. **Resonance detection**: Fluorescence dips at frequencies corresponding to mₛ = ±1
|
||||
|
||||
The resonance frequency shifts with external magnetic field B:
|
||||
|
||||
```
|
||||
f± = D ± γₑB
|
||||
```
|
||||
|
||||
Where:
|
||||
- D = 2.87 GHz (zero-field splitting)
|
||||
- γₑ = 28 GHz/T (electron gyromagnetic ratio)
|
||||
- B = external magnetic field component along NV axis
|
||||
|
||||
For a 1 fT field: Δf = 28 × 10⁻¹⁵ GHz = 28 μHz — extraordinarily small, requiring
|
||||
long integration times or ensemble measurements.
|
||||
|
||||
### 2.3 Sensitivity Fundamentals
|
||||
|
||||
**Single NV center**: Limited by photon shot noise
|
||||
```
|
||||
η_single ≈ (ℏ/gₑμ_B) × (1/√(C² × R × T₂*))
|
||||
```
|
||||
Where C is ODMR contrast (~0.03), R is photon count rate (~10⁵/s), T₂* is inhomogeneous
|
||||
dephasing time (~1 μs in bulk diamond).
|
||||
|
||||
Typical single NV sensitivity: ~1 μT/√Hz — insufficient for neural signals.
|
||||
|
||||
**NV ensemble**: N centers improve sensitivity by √N
|
||||
```
|
||||
η_ensemble = η_single / √N
|
||||
```
|
||||
|
||||
For N = 10¹² NV centers in a 100 μm × 100 μm × 10 μm sensing volume:
|
||||
η_ensemble ≈ 1 pT/√Hz
|
||||
|
||||
**State of the art (2025–2026)**: Laboratory demonstrations have achieved:
|
||||
- 1–10 fT/√Hz using large diamond chips with optimized NV density
|
||||
- Sub-pT/√Hz using advanced dynamical decoupling sequences
|
||||
- ~100 aT/√Hz projected with quantum-enhanced protocols (squeezed states)
|
||||
|
||||
### 2.4 Dynamical Decoupling for Neural Frequency Bands
|
||||
|
||||
Neural signals occupy specific frequency bands. Pulsed measurement protocols can be tuned
|
||||
to these bands:
|
||||
|
||||
| Protocol | Sensitivity Band | Application |
|
||||
|----------|-----------------|-------------|
|
||||
| Ramsey interferometry | DC–10 Hz | Infraslow oscillations |
|
||||
| Hahn echo | 10–100 Hz | Alpha, beta rhythms |
|
||||
| CPMG (N pulses) | f = N/(2τ) | Tunable narrowband |
|
||||
| XY-8 sequence | Narrowband, robust | Specific frequency targeting |
|
||||
| KDD (Knill DD) | Broadband | General neural activity |
|
||||
|
||||
**CPMG for alpha rhythm detection (10 Hz)**:
|
||||
- Set interpulse spacing τ = 1/(2 × 10 Hz) = 50 ms
|
||||
- N = 100 pulses → total sensing time = 5 s
|
||||
- Achieved sensitivity: ~10 fT/√Hz in laboratory conditions
|
||||
|
||||
### 2.5 T₁ and T₂ Relaxation Times
|
||||
|
||||
| Parameter | Bulk Diamond | Thin Film | Nanodiamonds |
|
||||
|-----------|-------------|-----------|--------------|
|
||||
| T₁ (spin-lattice) | ~6 ms | ~1 ms | ~10 μs |
|
||||
| T₂ (spin-spin) | ~1.8 ms | ~100 μs | ~1 μs |
|
||||
| T₂* (inhomogeneous) | ~10 μs | ~1 μs | ~100 ns |
|
||||
|
||||
Longer T₂ enables better sensitivity. Electronic-grade CVD diamond with low nitrogen
|
||||
concentration ([N] < 1 ppb) achieves the best T₂ values.
|
||||
|
||||
---
|
||||
|
||||
## 3. Neural Magnetic Field Sources
|
||||
|
||||
### 3.1 Origins of Neural Magnetic Fields
|
||||
|
||||
Neurons generate magnetic fields through two mechanisms:
|
||||
|
||||
1. **Intracellular currents**: Ionic flow (Na⁺, K⁺, Ca²⁺) along axons and dendrites during
|
||||
action potentials and synaptic activity. These are the primary sources measured by MEG.
|
||||
|
||||
2. **Transmembrane currents**: Ionic currents crossing the cell membrane during depolarization
|
||||
and repolarization. Generate weaker, more localized fields.
|
||||
|
||||
The magnetic field from a current dipole at distance r:
|
||||
|
||||
```
|
||||
B(r) = (μ₀/4π) × (Q × r̂)/(r²)
|
||||
```
|
||||
|
||||
Where Q is the current dipole moment (A·m) and μ₀ = 4π × 10⁻⁷ T·m/A.
|
||||
|
||||
### 3.2 Signal Magnitudes
|
||||
|
||||
| Source | Current Dipole | Field at Scalp | Field at 6mm |
|
||||
|--------|---------------|----------------|--------------|
|
||||
| Single neuron | ~0.02 pA·m | ~0.01 fT | ~0.1 fT |
|
||||
| Cortical column (~10⁴ neurons) | ~10 nA·m | ~10–100 fT | ~50–500 fT |
|
||||
| Evoked response (~10⁶ neurons) | ~10 μA·m | ~50–200 fT | ~200–1000 fT |
|
||||
| Epileptic spike | ~100 μA·m | ~500–5000 fT | ~2000–20000 fT |
|
||||
| Alpha rhythm | ~20 μA·m | ~50–200 fT | ~200–800 fT |
|
||||
|
||||
**Key insight for NV sensors**: At 6mm standoff (close proximity, like OPM), signals are
|
||||
3–5× stronger than at scalp surface measurements typical of SQUID MEG (20–30mm gap).
|
||||
NV arrays mounted directly on the scalp benefit from this proximity gain.
|
||||
|
||||
### 3.3 Frequency Bands
|
||||
|
||||
| Band | Frequency | Typical Amplitude (scalp) | Neural Correlate |
|
||||
|------|-----------|--------------------------|------------------|
|
||||
| Delta | 1–4 Hz | 50–200 fT | Deep sleep, pathology |
|
||||
| Theta | 4–8 Hz | 30–100 fT | Memory, navigation |
|
||||
| Alpha | 8–13 Hz | 50–200 fT | Inhibition, idling |
|
||||
| Beta | 13–30 Hz | 20–80 fT | Motor planning, attention |
|
||||
| Gamma | 30–100 Hz | 10–50 fT | Perception, binding |
|
||||
| High-gamma | >100 Hz | 5–20 fT | Local cortical processing |
|
||||
|
||||
**Sensitivity requirement**: To detect all bands, the sensor needs ~5–10 fT/√Hz sensitivity
|
||||
in the 1–200 Hz range. Current NV ensembles are approaching this in laboratory conditions.
|
||||
|
||||
### 3.4 Why Magnetic Fields Are Better Than Electric Fields for Topology
|
||||
|
||||
EEG measures electric potentials at the scalp. The skull acts as a volume conductor that
|
||||
severely smears the spatial distribution, limiting source localization to ~10–20 mm.
|
||||
|
||||
Magnetic fields pass through the skull nearly unattenuated (skull has permeability μ ≈ μ₀).
|
||||
This preserves spatial information, enabling source localization to ~2–5 mm with dense
|
||||
sensor arrays.
|
||||
|
||||
For brain network topology analysis, this spatial resolution difference is critical:
|
||||
- At 20 mm resolution (EEG): can distinguish ~20 brain regions
|
||||
- At 3–5 mm resolution (NV/OPM): can distinguish ~100–400 brain regions
|
||||
- More regions = more detailed connectivity graph = more precise mincut analysis
|
||||
|
||||
---
|
||||
|
||||
## 4. Sensor Architecture for Neural Imaging
|
||||
|
||||
### 4.1 Single NV vs Ensemble NV
|
||||
|
||||
| Configuration | Sensitivity | Spatial Resolution | Use Case |
|
||||
|--------------|-------------|-------------------|----------|
|
||||
| Single NV | ~1 μT/√Hz | ~10 nm | Nanoscale imaging (not neural) |
|
||||
| Small ensemble (10⁶) | ~1 nT/√Hz | ~1 μm | Cellular-scale |
|
||||
| Large ensemble (10¹²) | ~1 pT/√Hz | ~100 μm | Neural macroscale |
|
||||
| Optimized ensemble | ~1–10 fT/√Hz | ~1 mm | Neural imaging (target) |
|
||||
|
||||
For brain topology analysis, large ensemble sensors with ~1 mm spatial resolution are the
|
||||
correct target. Single-NV experiments are scientifically interesting but irrelevant for
|
||||
whole-brain network monitoring.
|
||||
|
||||
### 4.2 Diamond Chip Fabrication
|
||||
|
||||
**CVD (Chemical Vapor Deposition) Growth**:
|
||||
1. Start with high-purity diamond substrate (Element Six, Applied Diamond)
|
||||
2. Grow epitaxial diamond layer with controlled nitrogen incorporation
|
||||
3. Target NV density: 10¹⁶–10¹⁷ cm⁻³ (balance sensitivity vs T₂)
|
||||
4. Irradiate with electrons or protons to create vacancies
|
||||
5. Anneal at 800–1200°C to mobilize vacancies to nitrogen sites
|
||||
6. Surface treatment to stabilize NV⁻ charge state
|
||||
|
||||
**Chip dimensions**: Typical sensing element: 2×2×0.5 mm diamond chip
|
||||
**Array fabrication**: Multiple chips mounted on flexible PCB for conformal sensor arrays
|
||||
|
||||
### 4.3 Optical Readout System
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ Green Laser (532 nm, 100 mW) │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ Diamond Chip │ │
|
||||
│ │ (NV ensemble) │──── Microwave│
|
||||
│ └────────┬────────┘ Drive │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ Dichroic Filter │ │
|
||||
│ │ (pass >637 nm) │ │
|
||||
│ └────────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ Photodetector │ │
|
||||
│ │ (Si APD/PIN) │ │
|
||||
│ └────────┬────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────▼────────┐ │
|
||||
│ │ Lock-in / ADC │ │
|
||||
│ └─────────────────┘ │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Power budget per sensor**: Laser ~100 mW, microwave ~10 mW, electronics ~50 mW
|
||||
**Total**: ~160 mW per sensing element
|
||||
|
||||
### 4.4 Gradiometer Configurations
|
||||
|
||||
Environmental magnetic noise (urban: ~100 nT fluctuations) is 10⁸× larger than neural
|
||||
signals. Noise rejection is essential.
|
||||
|
||||
**First-order gradiometer**: Two NV sensors separated by ~5 cm
|
||||
```
|
||||
Signal = Sensor_near - Sensor_far
|
||||
```
|
||||
Rejects uniform background fields. Retains neural signals (which have steep spatial gradient).
|
||||
|
||||
**Second-order gradiometer**: Three sensors in line
|
||||
```
|
||||
Signal = Sensor_near - 2×Sensor_mid + Sensor_far
|
||||
```
|
||||
Rejects uniform fields AND linear gradients.
|
||||
|
||||
**Synthetic gradiometry**: Software-based, using reference sensors away from the head.
|
||||
More flexible than hardware gradiometers.
|
||||
|
||||
### 4.5 Array Configurations
|
||||
|
||||
**Linear array**: 8–16 sensors along a line. Good for slice imaging.
|
||||
**2D planar array**: 8×8 = 64 sensors on flat surface. Good for one brain region.
|
||||
**Helmet conformal**: 64–256 sensors on 3D-printed helmet. Full-head coverage.
|
||||
|
||||
For topology analysis, helmet conformal arrays are required to simultaneously measure
|
||||
all brain regions.
|
||||
|
||||
---
|
||||
|
||||
## 5. Comparison with Traditional SQUID MEG
|
||||
|
||||
### 5.1 Head-to-Head Comparison
|
||||
|
||||
| Parameter | SQUID MEG | NV Diamond (Current) | NV Diamond (Projected 2028) |
|
||||
|-----------|-----------|---------------------|---------------------------|
|
||||
| Sensitivity | 3–5 fT/√Hz | 10–100 fT/√Hz | 1–10 fT/√Hz |
|
||||
| Bandwidth | DC–1000 Hz | DC–1000 Hz | DC–1000 Hz |
|
||||
| Operating temp | 4 K (liquid He) | 300 K (room temp) | 300 K |
|
||||
| Cryogenics | Required ($50K/year He) | None | None |
|
||||
| Sensor-scalp gap | 20–30 mm | ~3–6 mm | ~3–6 mm |
|
||||
| Spatial resolution | 3–5 mm | 1–3 mm (projected) | 1–3 mm |
|
||||
| Channels | 275–306 | 4–64 (current) | 128–256 |
|
||||
| System cost | $2–5M | $50–200K (projected) | $20–100K |
|
||||
| Portability | Fixed installation | Potentially wearable | Wearable |
|
||||
| Maintenance | High (cryogen refills) | Low | Low |
|
||||
| Setup time | 30–60 min | <5 min (projected) | <5 min |
|
||||
|
||||
### 5.2 Proximity Advantage
|
||||
|
||||
The most significant practical advantage of NV sensors: they can be placed directly on the
|
||||
scalp. SQUID sensors sit inside a dewar with a ~20–30 mm gap between sensor and scalp.
|
||||
|
||||
Magnetic field from a dipole falls as 1/r³. Moving from 25 mm to 6 mm standoff:
|
||||
```
|
||||
Signal gain = (25/6)³ ≈ 72×
|
||||
```
|
||||
|
||||
This 72× proximity gain partially compensates for NV's lower intrinsic sensitivity.
|
||||
Effective comparison:
|
||||
- SQUID at 25 mm: 5 fT/√Hz sensitivity, signal attenuated by distance
|
||||
- NV at 6 mm: 50 fT/√Hz sensitivity, but 72× stronger signal
|
||||
|
||||
Net SNR comparison: roughly comparable for cortical sources.
|
||||
|
||||
### 5.3 Cost Trajectory
|
||||
|
||||
| Year | SQUID MEG System | NV Array System (est.) |
|
||||
|------|-----------------|----------------------|
|
||||
| 2020 | $3M | N/A (lab only) |
|
||||
| 2024 | $3.5M | $500K (research prototype) |
|
||||
| 2026 | $4M | $200K (multi-channel) |
|
||||
| 2028 | $4M+ | $50–100K (clinical prototype) |
|
||||
| 2030 | $4M+ | $20–50K (production) |
|
||||
|
||||
The cost crossover point is approaching. NV systems will likely be 10–100× cheaper than
|
||||
SQUID MEG within 5 years.
|
||||
|
||||
---
|
||||
|
||||
## 6. Signal Processing Pipeline
|
||||
|
||||
### 6.1 Raw ODMR Signal to Magnetic Field
|
||||
|
||||
1. **Continuous-wave ODMR**: Sweep microwave frequency, measure fluorescence
|
||||
- Simple but limited bandwidth (~100 Hz)
|
||||
- Sensitivity: ~100 pT/√Hz
|
||||
|
||||
2. **Pulsed ODMR (Ramsey)**: Initialize → free precession → readout
|
||||
- Better sensitivity, tunable bandwidth
|
||||
- Sensitivity: ~1 pT/√Hz
|
||||
|
||||
3. **Dynamical decoupling (CPMG/XY-8)**: Multiple π-pulses during precession
|
||||
- Narrowband, highest sensitivity
|
||||
- Sensitivity: ~10 fT/√Hz (demonstrated)
|
||||
- Tunable to specific neural frequency bands
|
||||
|
||||
### 6.2 Multi-Channel Processing
|
||||
|
||||
For a 128-channel NV array:
|
||||
- Each channel: continuous magnetic field time series at 1–10 kHz sampling
|
||||
- Data rate: 128 × 10 kHz × 32 bit = ~5 MB/s
|
||||
- Real-time processing: band-pass filtering, artifact rejection, source localization
|
||||
|
||||
### 6.3 Beamforming with NV Arrays
|
||||
|
||||
Dense NV arrays enable beamforming (spatial filtering):
|
||||
|
||||
```
|
||||
Virtual sensor output = Σᵢ wᵢ × sensorᵢ(t)
|
||||
```
|
||||
|
||||
Where weights wᵢ are computed to maximize sensitivity to a specific brain location while
|
||||
suppressing signals from other locations.
|
||||
|
||||
**LCMV (Linearly Constrained Minimum Variance) beamformer**:
|
||||
```
|
||||
w = (C⁻¹ × L) / (L^T × C⁻¹ × L)
|
||||
```
|
||||
Where C is the data covariance matrix and L is the lead field vector for the target location.
|
||||
|
||||
NV's high spatial density enables better beamformer performance than sparse SQUID arrays.
|
||||
|
||||
### 6.4 Source Localization
|
||||
|
||||
From sensor-space measurements to brain-space current estimates:
|
||||
|
||||
1. **Forward model**: Given brain anatomy (from MRI), compute expected sensor measurements
|
||||
for a unit current at each brain location. Stored as lead field matrix L.
|
||||
|
||||
2. **Inverse solution**: Given sensor measurements B, estimate brain currents J:
|
||||
```
|
||||
J = L^T(LL^T + λI)⁻¹B (minimum-norm estimate)
|
||||
```
|
||||
|
||||
3. **Parcellation**: Map continuous source space to discrete brain regions (68–400 parcels)
|
||||
|
||||
4. **Connectivity**: Compute coupling between parcels → graph edges → mincut analysis
|
||||
|
||||
---
|
||||
|
||||
## 7. Integration with RuVector Architecture
|
||||
|
||||
### 7.1 Data Flow: NV Sensor → Brain Topology Graph
|
||||
|
||||
```
|
||||
NV Array (128 ch, 1 kHz)
|
||||
│
|
||||
▼
|
||||
Preprocessing (filter, artifact rejection)
|
||||
│
|
||||
▼
|
||||
Source Localization (128 sensors → 86 parcels)
|
||||
│
|
||||
▼
|
||||
Connectivity Estimation (PLV, coherence per parcel pair)
|
||||
│
|
||||
▼
|
||||
Brain Graph G(t) = (V=86 parcels, E=weighted connections)
|
||||
│
|
||||
▼
|
||||
RuVector Embedding (graph → 256-d vector)
|
||||
│
|
||||
▼
|
||||
Dynamic Mincut Analysis (partition detection)
|
||||
│
|
||||
▼
|
||||
State Classification / Anomaly Detection
|
||||
```
|
||||
|
||||
### 7.2 Mapping to Existing RuVector Modules
|
||||
|
||||
| RuVector Module | Neural Application |
|
||||
|----------------|-------------------|
|
||||
| `ruvector-temporal-tensor` | Store sequential brain graph snapshots |
|
||||
| `ruvector-mincut` | Compute brain network minimum cut |
|
||||
| `ruvector-attn-mincut` | Attention-weighted brain region importance |
|
||||
| `ruvector-attention` | Spatial attention across sensor array |
|
||||
| `ruvector-solver` | Sparse interpolation for source reconstruction |
|
||||
|
||||
### 7.3 Real-Time Processing Budget
|
||||
|
||||
| Stage | Latency | Computation |
|
||||
|-------|---------|-------------|
|
||||
| Sensor readout | 1 ms | Hardware |
|
||||
| Preprocessing | 2 ms | FIR filtering (SIMD) |
|
||||
| Source localization | 5 ms | Matrix multiply (86×128) |
|
||||
| Connectivity (1 band) | 10 ms | Pairwise coherence (86²/2 pairs) |
|
||||
| Graph embedding | 3 ms | GNN forward pass |
|
||||
| Mincut | 2 ms | Stoer-Wagner on 86 nodes |
|
||||
| **Total** | **~23 ms** | **Real-time capable** |
|
||||
|
||||
### 7.4 Hybrid WiFi CSI + NV Magnetic Sensing
|
||||
|
||||
WiFi CSI provides macro-level body pose and room-scale activity detection.
|
||||
NV magnetometers provide neural state information.
|
||||
|
||||
**Temporal alignment**: Neural signals (mincut topology changes) precede motor output
|
||||
by 200–500 ms. WiFi CSI detects the actual movement. Combining both:
|
||||
|
||||
```
|
||||
t = -300 ms: NV detects motor cortex network reorganization (mincut change)
|
||||
t = -100 ms: NV detects motor command formation (further topology shift)
|
||||
t = 0 ms: WiFi CSI detects actual body movement
|
||||
```
|
||||
|
||||
This enables **predictive** body tracking: RuView knows the person will move before
|
||||
the movement physically occurs.
|
||||
|
||||
---
|
||||
|
||||
## 8. Real-Time Neural Current Flow Mapping
|
||||
|
||||
### 8.1 Current Density Imaging
|
||||
|
||||
From magnetic field measurements, reconstruct current density in the brain:
|
||||
|
||||
```
|
||||
J(r) = -σ∇V(r) + J_p(r)
|
||||
```
|
||||
|
||||
Where J_p is the primary (neural) current and σ∇V is the volume current.
|
||||
|
||||
Minimum-norm current estimation provides a smooth current density map that can be
|
||||
updated at each time point, creating a movie of current flow.
|
||||
|
||||
### 8.2 Connectivity Graph Construction from Current Flow
|
||||
|
||||
For each pair of brain parcels (i, j), compute:
|
||||
|
||||
1. **Phase Locking Value**: PLV(i,j) = |⟨exp(jΔφᵢⱼ(t))⟩|
|
||||
2. **Coherence**: Coh(i,j,f) = |Sᵢⱼ(f)|² / (Sᵢᵢ(f) × Sⱼⱼ(f))
|
||||
3. **Granger causality**: GC(i→j) = ln(var(jₜ|j_past) / var(jₜ|j_past, i_past))
|
||||
|
||||
Each metric produces edge weights for the brain connectivity graph.
|
||||
|
||||
### 8.3 Temporal Resolution Advantage
|
||||
|
||||
| Technology | Time Resolution | Network Changes Visible |
|
||||
|-----------|----------------|------------------------|
|
||||
| fMRI | 2 seconds | Slow state transitions |
|
||||
| EEG | 1 ms | Fast dynamics (poor spatial) |
|
||||
| SQUID MEG | 1 ms | Fast dynamics (fixed position) |
|
||||
| OPM | 5 ms | Fast dynamics (wearable) |
|
||||
| NV Diamond | 1 ms | Fast dynamics (dense array, wearable) |
|
||||
|
||||
NV's combination of high temporal resolution AND dense spatial sampling is unique.
|
||||
|
||||
---
|
||||
|
||||
## 9. State of the Art (2024–2026)
|
||||
|
||||
### 9.1 Leading Research Groups
|
||||
|
||||
**MIT/Harvard**: Walsworth group — pioneered NV magnetometry, demonstrated cellular-scale
|
||||
magnetic imaging, working on macroscale neural sensing arrays.
|
||||
|
||||
**University of Stuttgart**: Wrachtrup group — single NV defect spectroscopy, advanced
|
||||
dynamical decoupling protocols for NV magnetometry.
|
||||
|
||||
**University of Melbourne**: Hollenberg group — NV-based quantum sensing for biological
|
||||
applications, diamond fabrication optimization.
|
||||
|
||||
**NIST Boulder**: NV ensemble magnetometry with optimized readout, approaching fT sensitivity.
|
||||
|
||||
**UC Berkeley**: Budker group — NV magnetometry for fundamental physics and biomedical
|
||||
applications.
|
||||
|
||||
### 9.2 Commercial NV Sensor Companies
|
||||
|
||||
| Company | Product | Sensitivity | Price Range |
|
||||
|---------|---------|-------------|-------------|
|
||||
| Qnami | ProteusQ (scanning) | ~1 μT/√Hz | $200K+ |
|
||||
| QZabre | NV microscope | ~100 nT/√Hz | $150K+ |
|
||||
| Element Six | Electronic-grade diamond | Material supplier | $1K–10K/chip |
|
||||
| QDTI | Quantum diamond devices | ~10 nT/√Hz | Custom |
|
||||
| NVision | NV-enhanced NMR | ~1 nT/√Hz | Custom |
|
||||
|
||||
**Note**: No company currently sells a neural-grade NV magnetometer (fT sensitivity).
|
||||
This is a gap in the market and an opportunity.
|
||||
|
||||
### 9.3 Recent Key Publications
|
||||
|
||||
- Demonstration of NV ensemble sensitivity reaching 10 fT/√Hz in laboratory conditions
|
||||
(multiple groups, 2024–2025)
|
||||
- NV diamond arrays for magnetic microscopy of biological samples
|
||||
- Theoretical proposals for NV-based MEG replacement systems
|
||||
- Integration of NV sensors with CMOS readout electronics
|
||||
|
||||
### 9.4 Remaining Challenges
|
||||
|
||||
| Challenge | Current Status | Required | Timeline |
|
||||
|-----------|---------------|----------|----------|
|
||||
| Sensitivity | 10–100 fT/√Hz | 1–10 fT/√Hz | 2–3 years |
|
||||
| Channel count | 1–4 | 64–256 | 3–5 years |
|
||||
| Laser power near head | ~100 mW/sensor | Thermal safety validated | 1–2 years |
|
||||
| Diamond quality at scale | Research-grade | Reproducible production | 2–3 years |
|
||||
| Real-time processing | Offline analysis | <50 ms end-to-end | 1–2 years |
|
||||
|
||||
---
|
||||
|
||||
## 10. Portable MEG-Style Brain Imaging
|
||||
|
||||
### 10.1 Form Factor Target
|
||||
|
||||
**Helmet design**: 3D-printed shell conforming to head shape
|
||||
- NV diamond chips mounted in helmet surface
|
||||
- Optical fibers deliver green laser light to each chip
|
||||
- Red fluorescence collected via fibers to centralized photodetectors
|
||||
- Microwave drive via printed striplines in helmet
|
||||
|
||||
**Weight budget**:
|
||||
| Component | Weight |
|
||||
|-----------|--------|
|
||||
| Diamond chips (128) | ~10 g |
|
||||
| Optical fibers | ~100 g |
|
||||
| Helmet shell | ~300 g |
|
||||
| Electronics PCBs | ~200 g |
|
||||
| **Total helmet** | **~610 g** |
|
||||
| Processing unit (backpack) | ~2 kg |
|
||||
|
||||
### 10.2 Power Requirements
|
||||
|
||||
| Component | Power |
|
||||
|-----------|-------|
|
||||
| Laser source (shared, split to 128 channels) | 5 W |
|
||||
| Microwave generation (shared) | 2 W |
|
||||
| Photodetectors + amplifiers | 3 W |
|
||||
| FPGA/processor | 5 W |
|
||||
| **Total** | **~15 W** |
|
||||
|
||||
Battery operation: 15 W × 2 hours = 30 Wh → ~200g lithium battery. Feasible for
|
||||
portable operation.
|
||||
|
||||
### 10.3 Projected Timeline
|
||||
|
||||
| Year | Milestone |
|
||||
|------|-----------|
|
||||
| 2026 | 8-channel NV bench prototype, fT sensitivity demonstrated |
|
||||
| 2027 | 32-channel NV array in shielded room |
|
||||
| 2028 | 64-channel NV helmet prototype |
|
||||
| 2029 | First wearable NV-MEG with active shielding |
|
||||
| 2030 | Clinical-grade NV-MEG system |
|
||||
|
||||
---
|
||||
|
||||
## 11. Detection of Subtle Connectivity Changes
|
||||
|
||||
### 11.1 Neuroplasticity Tracking
|
||||
|
||||
Learning physically changes brain connectivity. NV arrays with sufficient sensitivity
|
||||
could track these changes:
|
||||
|
||||
- **Motor learning**: Strengthening of motor-cerebellar connections over practice sessions
|
||||
- **Language learning**: Reorganization of language network topology
|
||||
- **Skill acquisition**: Transition from effortful (distributed) to automated (focal) processing
|
||||
|
||||
Mincut signature: as a skill is learned, the task-relevant network becomes more tightly
|
||||
integrated (lower internal mincut) and more separated from task-irrelevant networks
|
||||
(higher cross-network mincut).
|
||||
|
||||
### 11.2 Pathological Connectivity Changes
|
||||
|
||||
Early connectivity disruption before clinical symptoms:
|
||||
|
||||
| Disease | Connectivity Change | Mincut Signature | Detection Window |
|
||||
|---------|-------------------|------------------|-----------------|
|
||||
| Alzheimer's | DMN fragmentation | Increasing mc(DMN) | 5–10 years before symptoms |
|
||||
| Parkinson's | Motor loop disruption | mc(motor) asymmetry | 3–5 years before symptoms |
|
||||
| Epilepsy | Local hypersynchrony | Decreasing mc(focus) | Minutes to hours before seizure |
|
||||
| Depression | DMN over-integration | Decreasing mc(DMN) | During episode |
|
||||
| Schizophrenia | Global disorganization | Abnormal mc variance | During active phase |
|
||||
|
||||
### 11.3 Sensitivity Requirements for Clinical Detection
|
||||
|
||||
To detect a 10% change in connectivity (clinically meaningful threshold):
|
||||
- Need to resolve edge weight changes of ~10% of baseline
|
||||
- Baseline PLV typically 0.2–0.8 between connected regions
|
||||
- 10% change: ΔPLV ≈ 0.02–0.08
|
||||
- Required sensor SNR: >10 dB in the relevant frequency band
|
||||
- Translates to: ~5–10 fT/√Hz sensor sensitivity for cortical sources
|
||||
|
||||
This is achievable with projected NV technology within 2–3 years.
|
||||
|
||||
---
|
||||
|
||||
## 12. Technical Challenges
|
||||
|
||||
### 12.1 Standoff Distance
|
||||
|
||||
Diamond chips sit on the scalp surface, ~10–15 mm from cortex (scalp tissue + skull).
|
||||
Deep brain structures (hippocampus, thalamus, basal ganglia) are 50–80 mm away.
|
||||
|
||||
Signal at these distances:
|
||||
- Cortex (10 mm): ~50–200 fT → detectable
|
||||
- Hippocampus (60 mm): ~0.1–1 fT → at noise floor
|
||||
- Brainstem (80 mm): ~0.01–0.1 fT → below detection
|
||||
|
||||
**Implication**: NV sensors are primarily cortical topology monitors. Deep structure
|
||||
topology requires either invasive sensing or indirect inference from cortical measurements.
|
||||
|
||||
### 12.2 Diamond Quality and Reproducibility
|
||||
|
||||
NV magnetometry performance depends critically on diamond quality:
|
||||
- Nitrogen concentration: needs [N] < 1 ppb for long T₂
|
||||
- NV density: balance between signal strength and T₂ degradation
|
||||
- Crystal strain: inhomogeneous strain broadens ODMR linewidth
|
||||
- Surface termination: affects NV⁻ charge stability
|
||||
|
||||
Current production variability: ~2× variation in T₂ between nominally identical chips.
|
||||
This needs to improve for standardized multi-channel systems.
|
||||
|
||||
### 12.3 Laser Heating
|
||||
|
||||
100 mW of green laser per sensor × 128 sensors = 12.8 W total optical power near the head.
|
||||
Even with fiber delivery, some heating occurs:
|
||||
|
||||
- Fiber-coupled: minimal heating at head (<1°C)
|
||||
- Free-space illumination: potentially dangerous without thermal management
|
||||
- Safety standard: IEC 62471 limits for skin exposure
|
||||
|
||||
**Solution**: Fiber-coupled laser delivery with reflective diamond chip mounting to direct
|
||||
waste heat away from scalp.
|
||||
|
||||
### 12.4 Bandwidth vs Sensitivity Tradeoff
|
||||
|
||||
Dynamical decoupling achieves best sensitivity in narrow frequency bands. Neural signals
|
||||
span 1–200 Hz. Options:
|
||||
|
||||
1. **Multiplexed measurement**: Rapidly switch between DD sequences tuned to different bands.
|
||||
Reduces effective sensitivity per band by √N_bands.
|
||||
|
||||
2. **Broadband measurement**: Use less aggressive DD (shorter sequences). Lower peak
|
||||
sensitivity but covers all bands simultaneously.
|
||||
|
||||
3. **Parallel sensors**: Dedicate different sensor subsets to different frequency bands.
|
||||
Requires more sensors but maintains sensitivity in each band.
|
||||
|
||||
Option 3 is most compatible with dense NV arrays and neural topology analysis (which
|
||||
benefits from simultaneous multi-band measurement).
|
||||
|
||||
---
|
||||
|
||||
## 13. Roadmap for NV Neural Magnetometry
|
||||
|
||||
### Phase 1: Characterization (2026–2027)
|
||||
- Build 8-channel NV array
|
||||
- Demonstrate fT-level sensitivity on bench
|
||||
- Validate with known magnetic phantom sources
|
||||
- Characterize noise sources and rejection methods
|
||||
- Cost: ~$100K
|
||||
|
||||
### Phase 2: Neural Validation (2027–2028)
|
||||
- 32-channel NV array in magnetically shielded room
|
||||
- Record alpha rhythm from human subject
|
||||
- Compare with simultaneous SQUID-MEG or OPM recording
|
||||
- Demonstrate source localization accuracy
|
||||
- Cost: ~$300K
|
||||
|
||||
### Phase 3: Prototype System (2028–2029)
|
||||
- 64-channel NV helmet with active shielding
|
||||
- Real-time connectivity graph construction
|
||||
- Demonstrate mincut-based cognitive state detection
|
||||
- First integration with RuVector pipeline
|
||||
- Cost: ~$500K
|
||||
|
||||
### Phase 4: Clinical Prototype (2029–2030)
|
||||
- 128-channel NV-MEG helmet
|
||||
- Portable form factor (helmet + backpack)
|
||||
- Validated against clinical SQUID-MEG
|
||||
- First clinical topology biomarker studies
|
||||
- Regulatory consultation
|
||||
- Cost: ~$1M
|
||||
|
||||
### Phase 5: Production System (2030+)
|
||||
- Manufactured NV arrays (cost target: <$500/chip)
|
||||
- Clinical-grade software pipeline
|
||||
- Normative topology database
|
||||
- Regulatory submission
|
||||
- Commercial deployment
|
||||
- Target system cost: $20–50K
|
||||
|
||||
---
|
||||
|
||||
## 14. Ethical and Safety Framework
|
||||
|
||||
### 14.1 Non-Invasive Nature
|
||||
|
||||
NV magnetometry is completely non-invasive:
|
||||
- No ionizing radiation
|
||||
- No strong magnetic fields (unlike MRI)
|
||||
- No electrical stimulation
|
||||
- Laser power is fiber-coupled, not directly incident on tissue
|
||||
- No known biological effects from measurement process
|
||||
|
||||
### 14.2 Privacy Considerations
|
||||
|
||||
**What NV neural sensors CAN detect**: brain network topology states (focused, relaxed,
|
||||
stressed, fatigued), pathological patterns, cognitive load level.
|
||||
|
||||
**What they CANNOT detect**: specific thoughts, memories, intentions, private mental content.
|
||||
|
||||
The topology-based approach is inherently privacy-preserving: it measures HOW the brain
|
||||
is organized, not WHAT it is computing. This is analogous to measuring traffic patterns
|
||||
in a city without reading anyone's mail.
|
||||
|
||||
### 14.3 Regulatory Classification
|
||||
|
||||
- FDA: likely Class II medical device (diagnostic aid) for clinical applications
|
||||
- No surgical risk, non-invasive, non-ionizing
|
||||
- 510(k) pathway with SQUID-MEG as predicate device
|
||||
- Additional pathway for wellness/consumer applications (lower regulatory burden)
|
||||
|
||||
---
|
||||
|
||||
## 15. Conclusion
|
||||
|
||||
NV diamond magnetometers represent the most promising medium-term technology for portable,
|
||||
affordable, high-resolution neural magnetic field measurement. While current sensitivity
|
||||
(10–100 fT/√Hz) is not yet sufficient for all neural applications, the trajectory toward
|
||||
1–10 fT/√Hz within 2–3 years makes NV a credible path to clinical-grade brain topology
|
||||
monitoring.
|
||||
|
||||
For the RuVector + dynamic mincut architecture, NV sensors offer:
|
||||
1. **Dense arrays** enabling detailed connectivity graph construction
|
||||
2. **Room-temperature operation** for wearable/portable form factors
|
||||
3. **Cost trajectory** enabling wide deployment
|
||||
4. **Spatial resolution** sufficient for 100+ brain parcel connectivity analysis
|
||||
5. **Temporal resolution** sufficient for real-time topology tracking
|
||||
|
||||
The combination of NV sensor arrays with RuVector graph memory and dynamic mincut analysis
|
||||
could create the first portable brain network topology observatory — measuring how cognition
|
||||
organizes itself in real time, without requiring the $3M SQUID MEG systems that currently
|
||||
dominate neuroimaging.
|
||||
|
||||
---
|
||||
|
||||
*This document is part of the RF Topological Sensing research series. It surveys
|
||||
nitrogen-vacancy diamond magnetometry technology and its application to neural current
|
||||
detection for brain network topology analysis.*
|
||||
@@ -0,0 +1,106 @@
|
||||
# RF Topological Sensing — Research Index
|
||||
|
||||
## SOTA Research Compendium
|
||||
|
||||
**Generated**: 2026-03-08
|
||||
**Total Documents**: 12
|
||||
**Total Lines**: 14,322
|
||||
**Branch**: `claude/rf-mincut-sensing-uHnQX`
|
||||
|
||||
---
|
||||
|
||||
## Core Concept
|
||||
|
||||
RF Topological Sensing treats a room as a dynamic signal graph where ESP32 nodes
|
||||
are vertices and TX-RX links are edges weighted by CSI coherence. Instead of
|
||||
estimating position, minimum cut detects where the RF field topology changes —
|
||||
revealing physical boundaries corresponding to objects, people, and environmental
|
||||
shifts. This creates a "radio nervous system" that is structurally aware of space.
|
||||
|
||||
---
|
||||
|
||||
## Document Index
|
||||
|
||||
### Foundations (Documents 1-2)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 01 | [RF Graph Theory & Mincut Foundations](01-rf-graph-theory-foundations.md) | 1,112 | Max-flow/min-cut theorem, Stoer-Wagner/Karger algorithms, Fiedler vector, Cheeger inequality, spectral graph theory, comparison to classical RF sensing |
|
||||
| 02 | [CSI Edge Weight Computation](02-csi-edge-weight-computation.md) | 1,059 | CSI feature extraction, coherence metrics, MUSIC/ESPRIT multipath decomposition, Kalman filtering of edges, noise robustness, normalization |
|
||||
|
||||
### Machine Learning (Documents 3-4)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 03 | [Attention Mechanisms for RF Sensing](03-attention-mechanisms-rf-sensing.md) | 1,110 | GAT for RF graphs, self-attention for CSI, cross-attention fusion, differentiable mincut, antenna-level attention, efficient attention variants |
|
||||
| 04 | [Transformer Architectures for Graph Sensing](04-transformer-architectures-graph-sensing.md) | 896 | Graphormer/SAN/GPS, temporal graph transformers, ViT for spectrograms, transformer-based mincut prediction, foundation models for RF, edge deployment |
|
||||
|
||||
### Algorithms (Document 5)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 05 | [Sublinear Mincut Algorithms](05-sublinear-mincut-algorithms.md) | 1,170 | Sublinear approximation, dynamic mincut, streaming algorithms, Benczúr-Karger sparsification, local partitioning, Rust implementation |
|
||||
|
||||
### Hardware & Systems (Documents 6, 10)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 06 | [ESP32 Mesh Hardware Constraints](06-esp32-mesh-hardware-constraints.md) | 1,122 | ESP32 CSI capabilities, 16-node topology, TDM synchronization, computational budget, channel hopping, power analysis, firmware architecture |
|
||||
| 10 | [System Architecture & Prototype Design](10-system-architecture-prototype.md) | 1,625 | End-to-end pipeline, crate integration, DDD module design, 100ms latency budget, 3-phase prototype, benchmark design, ADR-044, Rust traits |
|
||||
|
||||
### Learning & Temporal (Documents 7-8)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 07 | [Contrastive Learning for RF Coherence](07-contrastive-learning-rf-coherence.md) | 1,226 | SimCLR/MoCo for CSI, AETHER-Topo extension, delta-driven updates, self-supervised pre-training, triplet edge classification, MERIDIAN transfer |
|
||||
| 08 | [Temporal Graph Evolution & RuVector](08-temporal-graph-evolution-ruvector.md) | 1,528 | TGN/TGAT/DyRep, RuVector graph memory, cut trajectory tracking, event detection, compressed storage, cross-room transitions, drift detection |
|
||||
|
||||
### Analysis (Document 9)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 09 | [Resolution & Spatial Granularity](09-resolution-spatial-granularity.md) | 1,383 | Fresnel zone analysis, node density vs resolution, Cramér-Rao bounds, graph cut resolution theory, multi-frequency enhancement, scaling laws |
|
||||
|
||||
### Quantum Sensing (Documents 11-12)
|
||||
|
||||
| # | Document | Lines | Key Topics |
|
||||
|---|----------|-------|------------|
|
||||
| 11 | [Quantum-Level Sensors](11-quantum-level-sensors.md) | 934 | NV centers, Rydberg atoms, SQUIDs, quantum illumination, quantum graph algorithms, hybrid architecture, quantum ML, NISQ applications |
|
||||
| 12 | [Quantum Biomedical Sensing](12-quantum-biomedical-sensing.md) | 1,157 | Biomagnetic mapping, neural field imaging, circulation sensing, coherence diagnostics, non-contact vitals, ambient health monitoring, BCI |
|
||||
|
||||
---
|
||||
|
||||
## Key Findings
|
||||
|
||||
### Resolution
|
||||
- 16 ESP32 nodes at 1m spacing → **30-60 cm** spatial granularity
|
||||
- Dual-band (2.4 + 5 GHz) → **6 cm** theoretical coherent limit
|
||||
- Information-theoretic limit: **8.8 cm** for dense deployment
|
||||
|
||||
### Computational Feasibility
|
||||
- Stoer-Wagner on 16-node graph: **~2,000 operations** per sweep
|
||||
- At 20 Hz: **0.07%** of one ESP32 core
|
||||
- Full pipeline CSI → mincut: **< 100 ms** latency budget
|
||||
|
||||
### Quantum Enhancement
|
||||
- NV diamond: 100-1000× sensitivity improvement at room temperature
|
||||
- Rydberg atoms: self-calibrated, SI-traceable RF field measurement
|
||||
- D-Wave quantum annealing: native QUBO solver for graph cuts
|
||||
|
||||
### Biomedical Extension
|
||||
- Non-contact cardiac monitoring at 1-3m with quantum sensors
|
||||
- Coherence-based diagnostics: disease as topological change in body's EM graph
|
||||
- Same graph algorithms (mincut, spectral) apply to both room sensing and medical
|
||||
|
||||
---
|
||||
|
||||
## Proposed ADRs
|
||||
- **ADR-044**: RF Topological Sensing (Document 10)
|
||||
- **ADR-045**: Quantum Biomedical Sensing Extension (Document 12)
|
||||
|
||||
## Implementation Phases
|
||||
1. **Phase 1** (4 weeks): 4-node POC — detect person in room
|
||||
2. **Phase 2** (8 weeks): 16-node room — track movement boundaries < 50 cm
|
||||
3. **Phase 3** (16 weeks): Multi-room mesh — cross-room transition detection
|
||||
4. **Phase 4** (2027-2028): Quantum-enhanced — NV diamond + ESP32 hybrid
|
||||
5. **Phase 5** (2029+): Biomedical — coherence diagnostics, ambient health
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user