Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e9bb4faf53 | |||
| 14902e6b4e | |||
| 086b0e690f | |||
| e0fe10b3dc | |||
| 915943cef4 | |||
| 66392cb4e2 | |||
| 9f1fca5513 | |||
| 36b0d27474 | |||
| 113011e704 | |||
| c193cd4299 | |||
| 7e8568a8e5 | |||
| 51140f599f | |||
| 47d0640c49 | |||
| 6959668e21 | |||
| 6a408b30e8 | |||
| 64dae5b1c1 | |||
| 8e487c54ea | |||
| 135d7d3d8c | |||
| 9dd61bdbfa | |||
| 8166d8d822 | |||
| fdc7142dfa | |||
| 02192b0232 | |||
| 8a46fff6b0 | |||
| 67f1fc162e | |||
| 4e925dba50 | |||
| 46d718d62f | |||
| 88d39e2639 | |||
| 7c2e7e2b27 | |||
| 0aab555821 | |||
| df394019cc | |||
| 47861de821 | |||
| 779bf8ff43 | |||
| fbd7d837c7 |
@@ -0,0 +1,305 @@
|
||||
name: Firmware CI/CD
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop, 'feature/*', 'feat/*', 'hotfix/*' ]
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IDF_VERSION: v5.2
|
||||
IDF_TARGET: esp32s3
|
||||
FIRMWARE_DIR: firmware/esp32-csi-node
|
||||
BINARY_PATH: firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
# 900 KB in bytes = 921600
|
||||
BINARY_SIZE_LIMIT: 921600
|
||||
|
||||
jobs:
|
||||
# ── Build ────────────────────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build Firmware (ESP-IDF ${{ env.IDF_VERSION }})
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: espressif/idf:v5.2
|
||||
options: --user root
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build firmware
|
||||
working-directory: ${{ env.FIRMWARE_DIR }}
|
||||
shell: bash
|
||||
run: |
|
||||
. /opt/esp/idf/export.sh
|
||||
idf.py set-target ${{ env.IDF_TARGET }}
|
||||
idf.py build
|
||||
|
||||
- name: Capture build size summary
|
||||
working-directory: ${{ env.FIRMWARE_DIR }}
|
||||
shell: bash
|
||||
run: |
|
||||
. /opt/esp/idf/export.sh
|
||||
idf.py size 2>&1 | tee build-size.txt
|
||||
|
||||
- name: Upload firmware artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ github.sha }}
|
||||
retention-days: 30
|
||||
path: |
|
||||
${{ env.FIRMWARE_DIR }}/build/esp32-csi-node.bin
|
||||
${{ env.FIRMWARE_DIR }}/build/bootloader/bootloader.bin
|
||||
${{ env.FIRMWARE_DIR }}/build/partition_table/partition-table.bin
|
||||
${{ env.FIRMWARE_DIR }}/build/flasher_args.json
|
||||
${{ env.FIRMWARE_DIR }}/build/flash_args
|
||||
${{ env.FIRMWARE_DIR }}/build-size.txt
|
||||
|
||||
# ── Binary size gate ─────────────────────────────────────────────────────────
|
||||
binary-size-check:
|
||||
name: Binary Size Check (<= 900 KB)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
|
||||
steps:
|
||||
- name: Download firmware artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ github.sha }}
|
||||
path: artifacts
|
||||
|
||||
- name: Check binary size
|
||||
run: |
|
||||
BINARY="artifacts/firmware/esp32-csi-node/build/esp32-csi-node.bin"
|
||||
# Fallback: search for the binary if the path differs
|
||||
if [ ! -f "$BINARY" ]; then
|
||||
BINARY=$(find artifacts -name 'esp32-csi-node.bin' | head -n 1)
|
||||
fi
|
||||
|
||||
if [ ! -f "$BINARY" ]; then
|
||||
echo "ERROR: esp32-csi-node.bin not found in artifacts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SIZE=$(stat -c%s "$BINARY")
|
||||
LIMIT=${{ env.BINARY_SIZE_LIMIT }}
|
||||
|
||||
echo "Binary: $BINARY"
|
||||
echo "Size: $SIZE bytes ($(( SIZE / 1024 )) KB)"
|
||||
echo "Limit: $LIMIT bytes ($(( LIMIT / 1024 )) KB, 90% of 1 MB partition)"
|
||||
|
||||
if [ "$SIZE" -gt "$LIMIT" ]; then
|
||||
echo "FAIL: binary exceeds 900 KB limit by $(( SIZE - LIMIT )) bytes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PCT=$(( SIZE * 100 / LIMIT ))
|
||||
echo "PASS: binary is ${PCT}% of the 900 KB budget"
|
||||
|
||||
# ── Credential leak scan ─────────────────────────────────────────────────────
|
||||
credential-scan:
|
||||
name: Credential Leak Check
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
|
||||
steps:
|
||||
- name: Download firmware artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ github.sha }}
|
||||
path: artifacts
|
||||
|
||||
- name: Scan binary for credential patterns
|
||||
run: |
|
||||
BINARY=$(find artifacts -name 'esp32-csi-node.bin' | head -n 1)
|
||||
|
||||
if [ ! -f "$BINARY" ]; then
|
||||
echo "ERROR: esp32-csi-node.bin not found in artifacts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Scanning $BINARY for credential patterns..."
|
||||
|
||||
# Patterns to search for (case-insensitive strings embedded in the binary)
|
||||
PATTERNS=(
|
||||
"password"
|
||||
"passwd"
|
||||
"secret"
|
||||
"api_key"
|
||||
"apikey"
|
||||
"private_key"
|
||||
"access_token"
|
||||
"auth_token"
|
||||
"credentials"
|
||||
"BEGIN RSA PRIVATE"
|
||||
"BEGIN EC PRIVATE"
|
||||
"BEGIN OPENSSH PRIVATE"
|
||||
"AKIA"
|
||||
)
|
||||
|
||||
FOUND=0
|
||||
for PATTERN in "${PATTERNS[@]}"; do
|
||||
# Use strings to extract printable text from the binary, then grep
|
||||
MATCHES=$(strings "$BINARY" | grep -i "$PATTERN" | grep -v "^nvs_config\|^csi_cfg\|override: password=\*\*\*\|NVS override" || true)
|
||||
if [ -n "$MATCHES" ]; then
|
||||
echo "WARNING: pattern '$PATTERN' found in binary:"
|
||||
echo "$MATCHES"
|
||||
FOUND=$(( FOUND + 1 ))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FOUND" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "FAIL: $FOUND credential pattern(s) detected in firmware binary."
|
||||
echo "Review the matches above. Legitimate log-format strings (e.g."
|
||||
echo "'NVS override: password=***') are excluded automatically."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: no credential patterns detected in firmware binary"
|
||||
|
||||
# ── QEMU smoke test ──────────────────────────────────────────────────────────
|
||||
# NOTE: QEMU in espressif/idf:v5.2 only supports -machine esp32 (LX6),
|
||||
# not esp32s3 (LX7). This test verifies the flash image can be created
|
||||
# and QEMU can be invoked, but boot verification is best-effort.
|
||||
qemu-smoke-test:
|
||||
name: QEMU Smoke Test (flash image creation)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
container:
|
||||
image: espressif/idf:v5.2
|
||||
options: --user root
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download firmware artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ github.sha }}
|
||||
path: artifacts
|
||||
|
||||
- name: Locate firmware binaries
|
||||
id: locate
|
||||
run: |
|
||||
APP=$(find artifacts -name 'esp32-csi-node.bin' | head -n 1)
|
||||
BOOT=$(find artifacts -name 'bootloader.bin' | head -n 1)
|
||||
PART=$(find artifacts -name 'partition-table.bin' | head -n 1)
|
||||
|
||||
echo "app=$APP" >> "$GITHUB_OUTPUT"
|
||||
echo "boot=$BOOT" >> "$GITHUB_OUTPUT"
|
||||
echo "part=$PART" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "Application: $APP"
|
||||
echo "Bootloader: $BOOT"
|
||||
echo "Partitions: $PART"
|
||||
|
||||
for f in "$APP" "$BOOT" "$PART"; do
|
||||
if [ ! -f "$f" ]; then
|
||||
echo "ERROR: missing binary: $f"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Create merged flash image
|
||||
run: |
|
||||
. /opt/esp/idf/export.sh
|
||||
|
||||
APP="${{ steps.locate.outputs.app }}"
|
||||
BOOT="${{ steps.locate.outputs.boot }}"
|
||||
PART="${{ steps.locate.outputs.part }}"
|
||||
|
||||
# Merge bootloader + partition table + app into a single 4 MB flash image
|
||||
esptool.py --chip esp32s3 merge_bin \
|
||||
--fill-flash-size 4MB \
|
||||
-o /tmp/flash_image.bin \
|
||||
0x0000 "$BOOT" \
|
||||
0x8000 "$PART" \
|
||||
0x10000 "$APP"
|
||||
|
||||
ls -lh /tmp/flash_image.bin
|
||||
echo "PASS: flash image created successfully (ready for esptool.py write_flash)"
|
||||
|
||||
- name: Verify flash image structure
|
||||
run: |
|
||||
# Verify the merged image has the expected components at correct offsets
|
||||
FLASH=/tmp/flash_image.bin
|
||||
SIZE=$(stat -c%s "$FLASH")
|
||||
echo "Flash image size: $SIZE bytes ($(( SIZE / 1024 )) KB)"
|
||||
|
||||
# Check for ESP32-S3 bootloader magic at offset 0
|
||||
MAGIC=$(xxd -p -l 1 -s 0 "$FLASH")
|
||||
echo "Bootloader first byte: 0x$MAGIC"
|
||||
|
||||
# Check for partition table magic at offset 0x8000
|
||||
PT_MAGIC=$(xxd -p -l 2 -s 0x8000 "$FLASH")
|
||||
echo "Partition table magic: 0x$PT_MAGIC"
|
||||
|
||||
# Check for app binary at offset 0x10000 (ESP image magic = 0xE9)
|
||||
APP_MAGIC=$(xxd -p -l 1 -s 0x10000 "$FLASH")
|
||||
echo "App image magic: 0x$APP_MAGIC"
|
||||
if [ "$APP_MAGIC" = "e9" ]; then
|
||||
echo "PASS: ESP application image detected at 0x10000"
|
||||
else
|
||||
echo "WARN: unexpected app magic byte (expected 0xe9, got 0x$APP_MAGIC)"
|
||||
fi
|
||||
|
||||
# ── Release artifact ─────────────────────────────────────────────────────────
|
||||
release-artifacts:
|
||||
name: Attach Firmware to Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: [binary-size-check, credential-scan, qemu-smoke-test]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
|
||||
steps:
|
||||
- name: Download firmware artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ github.sha }}
|
||||
path: artifacts
|
||||
|
||||
- name: Bundle release assets
|
||||
run: |
|
||||
mkdir -p release
|
||||
find artifacts -name '*.bin' -exec cp {} release/ \;
|
||||
find artifacts -name 'flasher_args.json' -exec cp {} release/ \;
|
||||
find artifacts -name 'flash_args' -exec cp {} release/ \;
|
||||
ls -lh release/
|
||||
|
||||
- name: Upload release assets
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: firmware-release-${{ github.run_number }}
|
||||
retention-days: 90
|
||||
path: release/
|
||||
|
||||
- name: Create GitHub Release (on tag)
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
name: Firmware ${{ github.ref_name }}
|
||||
body: |
|
||||
ESP32-S3 CSI Node firmware — built from ${{ github.sha }}
|
||||
|
||||
**Build details:**
|
||||
- ESP-IDF version: ${{ env.IDF_VERSION }}
|
||||
- Target: ${{ env.IDF_TARGET }}
|
||||
- Commit: ${{ github.sha }}
|
||||
|
||||
**Flashing:**
|
||||
```
|
||||
esptool.py --chip esp32s3 --baud 460800 write_flash @flash_args
|
||||
```
|
||||
files: release/*
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -1,3 +1,6 @@
|
||||
# Local machine configuration (not shared)
|
||||
CLAUDE.local.md
|
||||
|
||||
# ESP32 firmware build artifacts and local config (contains WiFi credentials)
|
||||
firmware/esp32-csi-node/build/
|
||||
firmware/esp32-csi-node/sdkconfig
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# WiFi DensePose
|
||||
# π RuView: WiFi DensePose:
|
||||
|
||||
**See through walls with WiFi.** No cameras. No wearables. Just radio waves.
|
||||
|
||||
@@ -12,6 +12,7 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
[](#esp32-s3-hardware-pipeline)
|
||||
[](https://crates.io/crates/wifi-densepose-ruvector)
|
||||
|
||||
|
||||
> | What | How | Speed |
|
||||
> |------|-----|-------|
|
||||
> | **Pose estimation** | CSI subcarrier amplitude/phase → DensePose UV maps | 54K fps (Rust) |
|
||||
@@ -54,6 +55,12 @@ docker run -p 3000:3000 ruvnet/wifi-densepose:latest
|
||||
|
||||
---
|
||||
|
||||
|
||||
<img src="assets/screen.png" alt="WiFi DensePose — Live pose detection with setup guide" width="800">
|
||||
<br>
|
||||
<em>Real-time pose skeleton from WiFi CSI signals — no cameras, no wearables</em>
|
||||
|
||||
|
||||
## 🚀 Key Features
|
||||
|
||||
### Sensing
|
||||
@@ -80,7 +87,8 @@ The system learns on its own and gets smarter over time — no hand-tuning, no l
|
||||
| 🎯 | **AI Signal Processing** | Attention networks, graph algorithms, and smart compression replace hand-tuned thresholds — adapts to each room automatically ([RuVector](https://github.com/ruvnet/ruvector)) |
|
||||
| 🌍 | **Works Everywhere** | Train once, deploy in any room — adversarial domain generalization strips environment bias so models transfer across rooms, buildings, and hardware ([ADR-027](docs/adr/ADR-027-cross-environment-domain-generalization.md)) |
|
||||
| 👁️ | **Cross-Viewpoint Fusion** | Learned attention fuses multiple viewpoints with geometric bias — reduces body occlusion and depth ambiguity that physics prevents any single sensor from solving ([ADR-031](docs/adr/ADR-031-ruview-sensing-first-rf-mode.md)) |
|
||||
| 🔮 | **Signal-Line Protocol** | `ruvector-crv` 6-stage CRV pipeline maps CSI sensing to Poincare ball embeddings, GNN topology, SNN temporal encoding, and MinCut partitioning | -- |
|
||||
| 🔮 | **Signal-Line Protocol** | `ruvector-crv` 6-stage CRV pipeline maps CSI sensing to Poincare ball embeddings, GNN topology, SNN temporal encoding, and MinCut partitioning ([ADR-033](docs/adr/ADR-033-crv-signal-line-sensing-integration.md)) |
|
||||
| 🔒 | **QUIC Mesh Security** | `midstreamer-quic` TLS 1.3 AEAD transport with HMAC-authenticated beacons, SipHash frame integrity, replay protection, and connection migration ([ADR-032](docs/adr/ADR-032-multistatic-mesh-security-hardening.md)) |
|
||||
|
||||
### Performance & Deployment
|
||||
|
||||
@@ -823,16 +831,16 @@ ESP32-S3 (STA + promiscuous) UDP/5005 Rust aggregator
|
||||
|
||||
```bash
|
||||
# Pre-built binaries — no toolchain required
|
||||
# https://github.com/ruvnet/wifi-densepose/releases/tag/v0.1.0-esp32
|
||||
# https://github.com/ruvnet/wifi-densepose/releases/tag/v0.2.0-esp32
|
||||
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write-flash --flash-mode dio --flash-size 4MB \
|
||||
0x0 bootloader.bin 0x8000 partition-table.bin 0x10000 esp32-csi-node.bin
|
||||
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourWiFi" --password "secret" --target-ip 192.168.1.20
|
||||
|
||||
cargo run -p wifi-densepose-hardware --bin aggregator -- --bind 0.0.0.0:5005 --verbose
|
||||
cargo run -p wifi-densepose-sensing-server -- --http-port 3000 --source esp32
|
||||
```
|
||||
|
||||
See [firmware/esp32-csi-node/README.md](firmware/esp32-csi-node/README.md) and [Tutorial #34](https://github.com/ruvnet/wifi-densepose/issues/34).
|
||||
@@ -1585,9 +1593,13 @@ Multistatic sensing, persistent field model, and cross-viewpoint fusion — the
|
||||
- **Channel-Hopping Firmware** — ESP32 firmware extended with hop table, timer-driven channel switching, NDP injection stub; NVS config for all TDM parameters; fully backward-compatible
|
||||
- **DDD Domain Model** — 6 bounded contexts, ubiquitous language, aggregate roots, domain events, full event bus specification
|
||||
- **`ruvector-crv` 6-stage CRV signal-line integration (ADR-033)** — Maps Coordinate Remote Viewing methodology to WiFi CSI: gestalt classification, sensory encoding, GNN topology, SNN coherence gating, differentiable search, MinCut partitioning; cross-session convergence for multi-room identity continuity
|
||||
- **ADR-032 multistatic mesh security hardening** — Bounded calibration buffers, atomic counters, division-by-zero guards, NaN-safe normalization across all multistatic modules
|
||||
- **ADR-032 multistatic mesh security hardening** — HMAC-SHA256 beacon auth, SipHash-2-4 frame integrity, NDP rate limiter, coherence gate timeout, bounded buffers, NVS credential zeroing, atomic firmware state
|
||||
- **ADR-032a QUIC transport layer** — `midstreamer-quic` TLS 1.3 AEAD for aggregator nodes, dual-mode security (ManualCrypto/QuicTransport), QUIC stream mapping, connection migration, congestion control
|
||||
- **ADR-033 CRV signal-line sensing integration** — Architecture decision record for the 6-stage CRV pipeline mapping to ruvector components
|
||||
- **9,000+ lines of new Rust code** across 17 modules with 300+ tests
|
||||
- **Temporal gesture matching** — `midstreamer-temporal-compare` DTW/LCS/edit-distance gesture classification with quantized feature comparison
|
||||
- **Attractor drift analysis** — `midstreamer-attractor` Takens' theorem phase-space embedding with Lyapunov exponent regime detection (Stable/Periodic/Chaotic)
|
||||
- **v0.3.0 published** — All 15 workspace crates published to [crates.io](https://crates.io/crates/wifi-densepose-core) with updated dependencies
|
||||
- **28,000+ lines of new Rust code** across 26 modules with 400+ tests
|
||||
- **Security hardened** — Bounded buffers, NaN guards, no panics in public APIs, input validation at all boundaries
|
||||
|
||||
### v3.0.0 — 2026-03-01
|
||||
|
||||
|
After Width: | Height: | Size: 270 KiB |
@@ -1,369 +0,0 @@
|
||||
# Claude Code Configuration — WiFi-DensePose + Claude Flow V3
|
||||
|
||||
## Project: wifi-densepose
|
||||
|
||||
WiFi-based human pose estimation using Channel State Information (CSI).
|
||||
Dual codebase: Python v1 (`v1/`) and Rust port (`rust-port/wifi-densepose-rs/`).
|
||||
### Key Rust Crates
|
||||
| Crate | Description |
|
||||
|-------|-------------|
|
||||
| `wifi-densepose-core` | Core types, traits, error types, CSI frame primitives |
|
||||
| `wifi-densepose-signal` | SOTA signal processing + RuvSense multistatic sensing (14 modules) |
|
||||
| `wifi-densepose-nn` | Neural network inference (ONNX, PyTorch, Candle backends) |
|
||||
| `wifi-densepose-train` | Training pipeline with ruvector integration + ruview_metrics |
|
||||
| `wifi-densepose-mat` | Mass Casualty Assessment Tool — disaster survivor detection |
|
||||
| `wifi-densepose-hardware` | ESP32 aggregator, TDM protocol, channel hopping firmware |
|
||||
| `wifi-densepose-ruvector` | RuVector v2.0.4 integration + cross-viewpoint fusion (5 modules) |
|
||||
| `wifi-densepose-api` | REST API (Axum) |
|
||||
| `wifi-densepose-db` | Database layer (Postgres, SQLite, Redis) |
|
||||
| `wifi-densepose-config` | Configuration management |
|
||||
| `wifi-densepose-wasm` | WebAssembly bindings for browser deployment |
|
||||
| `wifi-densepose-cli` | CLI tool (`wifi-densepose` binary) |
|
||||
| `wifi-densepose-sensing-server` | Lightweight Axum server for WiFi sensing UI |
|
||||
| `wifi-densepose-wifiscan` | Multi-BSSID WiFi scanning (ADR-022) |
|
||||
| `wifi-densepose-vitals` | ESP32 CSI-grade vital sign extraction (ADR-021) |
|
||||
|
||||
### RuvSense Modules (`signal/src/ruvsense/`)
|
||||
| Module | Purpose |
|
||||
|--------|---------|
|
||||
| `multiband.rs` | Multi-band CSI frame fusion, cross-channel coherence |
|
||||
| `phase_align.rs` | Iterative LO phase offset estimation, circular mean |
|
||||
| `multistatic.rs` | Attention-weighted fusion, geometric diversity |
|
||||
| `coherence.rs` | Z-score coherence scoring, DriftProfile |
|
||||
| `coherence_gate.rs` | Accept/PredictOnly/Reject/Recalibrate gate decisions |
|
||||
| `pose_tracker.rs` | 17-keypoint Kalman tracker with AETHER re-ID embeddings |
|
||||
| `field_model.rs` | SVD room eigenstructure, perturbation extraction |
|
||||
| `tomography.rs` | RF tomography, ISTA L1 solver, voxel grid |
|
||||
| `longitudinal.rs` | Welford stats, biomechanics drift detection |
|
||||
| `intention.rs` | Pre-movement lead signals (200-500ms) |
|
||||
| `cross_room.rs` | Environment fingerprinting, transition graph |
|
||||
| `gesture.rs` | DTW template matching gesture classifier |
|
||||
| `adversarial.rs` | Physically impossible signal detection, multi-link consistency |
|
||||
|
||||
### Cross-Viewpoint Fusion (`ruvector/src/viewpoint/`)
|
||||
| Module | Purpose |
|
||||
|--------|---------|
|
||||
| `attention.rs` | CrossViewpointAttention, GeometricBias, softmax with G_bias |
|
||||
| `geometry.rs` | GeometricDiversityIndex, Cramer-Rao bounds, Fisher Information |
|
||||
| `coherence.rs` | Phase phasor coherence, hysteresis gate |
|
||||
| `fusion.rs` | MultistaticArray aggregate root, domain events |
|
||||
|
||||
### RuVector v2.0.4 Integration (ADR-016 complete, ADR-017 proposed)
|
||||
All 5 ruvector crates integrated in workspace:
|
||||
- `ruvector-mincut` → `metrics.rs` (DynamicPersonMatcher) + `subcarrier_selection.rs`
|
||||
- `ruvector-attn-mincut` → `model.rs` (apply_antenna_attention) + `spectrogram.rs`
|
||||
- `ruvector-temporal-tensor` → `dataset.rs` (CompressedCsiBuffer) + `breathing.rs`
|
||||
- `ruvector-solver` → `subcarrier.rs` (sparse interpolation 114→56) + `triangulation.rs`
|
||||
- `ruvector-attention` → `model.rs` (apply_spatial_attention) + `bvp.rs`
|
||||
|
||||
### Architecture Decisions
|
||||
32 ADRs in `docs/adr/` (ADR-001 through ADR-032). Key ones:
|
||||
- ADR-014: SOTA signal processing (Accepted)
|
||||
- ADR-015: MM-Fi + Wi-Pose training datasets (Accepted)
|
||||
- ADR-016: RuVector training pipeline integration (Accepted — complete)
|
||||
- ADR-017: RuVector signal + MAT integration (Proposed — next target)
|
||||
- ADR-024: Contrastive CSI embedding / AETHER (Accepted)
|
||||
- ADR-027: Cross-environment domain generalization / MERIDIAN (Accepted)
|
||||
- ADR-028: ESP32 capability audit + witness verification (Accepted)
|
||||
- ADR-029: RuvSense multistatic sensing mode (Proposed)
|
||||
- ADR-030: RuvSense persistent field model (Proposed)
|
||||
- ADR-031: RuView sensing-first RF mode (Proposed)
|
||||
- ADR-032: Multistatic mesh security hardening (Proposed)
|
||||
|
||||
### Build & Test Commands (this repo)
|
||||
```bash
|
||||
# Rust — full workspace tests (1,031+ tests, ~2 min)
|
||||
cd rust-port/wifi-densepose-rs
|
||||
cargo test --workspace --no-default-features
|
||||
|
||||
# Rust — single crate check (no GPU needed)
|
||||
cargo check -p wifi-densepose-train --no-default-features
|
||||
|
||||
# Rust — publish crates (dependency order)
|
||||
cargo publish -p wifi-densepose-core --no-default-features
|
||||
cargo publish -p wifi-densepose-signal --no-default-features
|
||||
# ... see crate publishing order below
|
||||
|
||||
# Python — deterministic proof verification (SHA-256)
|
||||
python v1/data/proof/verify.py
|
||||
|
||||
# Python — test suite
|
||||
cd v1 && python -m pytest tests/ -x -q
|
||||
```
|
||||
|
||||
### Crate Publishing Order
|
||||
Crates must be published in dependency order:
|
||||
1. `wifi-densepose-core` (no internal deps)
|
||||
2. `wifi-densepose-vitals` (no internal deps)
|
||||
3. `wifi-densepose-wifiscan` (no internal deps)
|
||||
4. `wifi-densepose-hardware` (no internal deps)
|
||||
5. `wifi-densepose-config` (no internal deps)
|
||||
6. `wifi-densepose-db` (no internal deps)
|
||||
7. `wifi-densepose-signal` (depends on core)
|
||||
8. `wifi-densepose-nn` (no internal deps, workspace only)
|
||||
9. `wifi-densepose-ruvector` (no internal deps, workspace only)
|
||||
10. `wifi-densepose-train` (depends on signal, nn)
|
||||
11. `wifi-densepose-mat` (depends on core, signal, nn)
|
||||
12. `wifi-densepose-api` (no internal deps)
|
||||
13. `wifi-densepose-wasm` (depends on mat)
|
||||
14. `wifi-densepose-sensing-server` (depends on wifiscan)
|
||||
15. `wifi-densepose-cli` (depends on mat)
|
||||
|
||||
### Validation & Witness Verification (ADR-028)
|
||||
|
||||
**After any significant code change, run the full validation:**
|
||||
|
||||
```bash
|
||||
# 1. Rust tests — must be 1,031+ passed, 0 failed
|
||||
cd rust-port/wifi-densepose-rs
|
||||
cargo test --workspace --no-default-features
|
||||
|
||||
# 2. Python proof — must print VERDICT: PASS
|
||||
cd ../..
|
||||
python v1/data/proof/verify.py
|
||||
|
||||
# 3. Generate witness bundle (includes both above + firmware hashes)
|
||||
bash scripts/generate-witness-bundle.sh
|
||||
|
||||
# 4. Self-verify the bundle — must be 7/7 PASS
|
||||
cd dist/witness-bundle-ADR028-*/
|
||||
bash VERIFY.sh
|
||||
```
|
||||
|
||||
**If the Python proof hash changes** (e.g., numpy/scipy version update):
|
||||
```bash
|
||||
# Regenerate the expected hash, then verify it passes
|
||||
python v1/data/proof/verify.py --generate-hash
|
||||
python v1/data/proof/verify.py
|
||||
```
|
||||
|
||||
**Witness bundle contents** (`dist/witness-bundle-ADR028-<sha>.tar.gz`):
|
||||
- `WITNESS-LOG-028.md` — 33-row attestation matrix with evidence per capability
|
||||
- `ADR-028-esp32-capability-audit.md` — Full audit findings
|
||||
- `proof/verify.py` + `expected_features.sha256` — Deterministic pipeline proof
|
||||
- `test-results/rust-workspace-tests.log` — Full cargo test output
|
||||
- `firmware-manifest/source-hashes.txt` — SHA-256 of all 7 ESP32 firmware files
|
||||
- `crate-manifest/versions.txt` — All 15 crates with versions
|
||||
- `VERIFY.sh` — One-command self-verification for recipients
|
||||
|
||||
**Key proof artifacts:**
|
||||
- `v1/data/proof/verify.py` — Trust Kill Switch: feeds reference signal through production pipeline, hashes output
|
||||
- `v1/data/proof/expected_features.sha256` — Published expected hash
|
||||
- `v1/data/proof/sample_csi_data.json` — 1,000 synthetic CSI frames (seed=42)
|
||||
- `docs/WITNESS-LOG-028.md` — 11-step reproducible verification procedure
|
||||
- `docs/adr/ADR-028-esp32-capability-audit.md` — Complete audit record
|
||||
|
||||
### Branch
|
||||
Default branch: `main`
|
||||
Active feature branch: `ruvsense-full-implementation` (PR #77)
|
||||
|
||||
---
|
||||
|
||||
## Behavioral Rules (Always Enforced)
|
||||
|
||||
- Do what has been asked; nothing more, nothing less
|
||||
- NEVER create files unless they're absolutely necessary for achieving your goal
|
||||
- ALWAYS prefer editing an existing file to creating a new one
|
||||
- NEVER proactively create documentation files (*.md) or README files unless explicitly requested
|
||||
- NEVER save working files, text/mds, or tests to the root folder
|
||||
- Never continuously check status after spawning a swarm — wait for results
|
||||
- ALWAYS read a file before editing it
|
||||
- NEVER commit secrets, credentials, or .env files
|
||||
|
||||
## File Organization
|
||||
|
||||
- NEVER save to root folder — use the directories below
|
||||
- `docs/adr/` — Architecture Decision Records (32 ADRs)
|
||||
- `docs/ddd/` — Domain-Driven Design models
|
||||
- `rust-port/wifi-densepose-rs/crates/` — Rust workspace crates (15 crates)
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-signal/src/ruvsense/` — RuvSense multistatic modules (14 files)
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-ruvector/src/viewpoint/` — Cross-viewpoint fusion (5 files)
|
||||
- `rust-port/wifi-densepose-rs/crates/wifi-densepose-hardware/src/esp32/` — ESP32 TDM protocol
|
||||
- `firmware/esp32-csi-node/main/` — ESP32 C firmware (channel hopping, NVS config, TDM)
|
||||
- `v1/src/` — Python source (core, hardware, services, api)
|
||||
- `v1/data/proof/` — Deterministic CSI proof bundles
|
||||
- `.claude-flow/` — Claude Flow coordination state (committed for team sharing)
|
||||
- `.claude/` — Claude Code settings, agents, memory (committed for team sharing)
|
||||
|
||||
## Project Architecture
|
||||
|
||||
- Follow Domain-Driven Design with bounded contexts
|
||||
- Keep files under 500 lines
|
||||
- Use typed interfaces for all public APIs
|
||||
- Prefer TDD London School (mock-first) for new code
|
||||
- Use event sourcing for state changes
|
||||
- Ensure input validation at system boundaries
|
||||
|
||||
### Project Config
|
||||
|
||||
- **Topology**: hierarchical-mesh
|
||||
- **Max Agents**: 15
|
||||
- **Memory**: hybrid
|
||||
- **HNSW**: Enabled
|
||||
- **Neural**: Enabled
|
||||
|
||||
## Pre-Merge Checklist
|
||||
|
||||
Before merging any PR, verify each item applies and is addressed:
|
||||
|
||||
1. **Rust tests pass** — `cargo test --workspace --no-default-features` (1,031+ passed, 0 failed)
|
||||
2. **Python proof passes** — `python v1/data/proof/verify.py` (VERDICT: PASS)
|
||||
3. **README.md** — Update platform tables, crate descriptions, hardware tables, feature summaries if scope changed
|
||||
4. **CLAUDE.md** — Update crate table, ADR list, module tables, version if scope changed
|
||||
5. **CHANGELOG.md** — Add entry under `[Unreleased]` with what was added/fixed/changed
|
||||
6. **User guide** (`docs/user-guide.md`) — Update if new data sources, CLI flags, or setup steps were added
|
||||
7. **ADR index** — Update ADR count in README docs table if a new ADR was created
|
||||
8. **Witness bundle** — Regenerate if tests or proof hash changed: `bash scripts/generate-witness-bundle.sh`
|
||||
9. **Docker Hub image** — Only rebuild if Dockerfile, dependencies, or runtime behavior changed
|
||||
10. **Crate publishing** — Only needed if a crate is published to crates.io and its public API changed
|
||||
11. **`.gitignore`** — Add any new build artifacts or binaries
|
||||
12. **Security audit** — Run security review for new modules touching hardware/network boundaries
|
||||
|
||||
## Build & Test
|
||||
|
||||
```bash
|
||||
# Build
|
||||
npm run build
|
||||
|
||||
# Test
|
||||
npm test
|
||||
|
||||
# Lint
|
||||
npm run lint
|
||||
```
|
||||
|
||||
- ALWAYS run tests after making code changes
|
||||
- ALWAYS verify build succeeds before committing
|
||||
|
||||
## Security Rules
|
||||
|
||||
- NEVER hardcode API keys, secrets, or credentials in source files
|
||||
- NEVER commit .env files or any file containing secrets
|
||||
- Always validate user input at system boundaries
|
||||
- Always sanitize file paths to prevent directory traversal
|
||||
- Run `npx @claude-flow/cli@latest security scan` after security-related changes
|
||||
|
||||
## Concurrency: 1 MESSAGE = ALL RELATED OPERATIONS
|
||||
|
||||
- All operations MUST be concurrent/parallel in a single message
|
||||
- Use Claude Code's Task tool for spawning agents, not just MCP
|
||||
- ALWAYS batch ALL todos in ONE TodoWrite call (5-10+ minimum)
|
||||
- ALWAYS spawn ALL agents in ONE message with full instructions via Task tool
|
||||
- ALWAYS batch ALL file reads/writes/edits in ONE message
|
||||
- ALWAYS batch ALL Bash commands in ONE message
|
||||
|
||||
## Swarm Orchestration
|
||||
|
||||
- MUST initialize the swarm using CLI tools when starting complex tasks
|
||||
- MUST spawn concurrent agents using Claude Code's Task tool
|
||||
- Never use CLI tools alone for execution — Task tool agents do the actual work
|
||||
- MUST call CLI tools AND Task tool in ONE message for complex work
|
||||
|
||||
### 3-Tier Model Routing (ADR-026)
|
||||
|
||||
| Tier | Handler | Latency | Cost | Use Cases |
|
||||
|------|---------|---------|------|-----------|
|
||||
| **1** | Agent Booster (WASM) | <1ms | $0 | Simple transforms (var→const, add types) — Skip LLM |
|
||||
| **2** | Haiku | ~500ms | $0.0002 | Simple tasks, low complexity (<30%) |
|
||||
| **3** | Sonnet/Opus | 2-5s | $0.003-0.015 | Complex reasoning, architecture, security (>30%) |
|
||||
|
||||
- Always check for `[AGENT_BOOSTER_AVAILABLE]` or `[TASK_MODEL_RECOMMENDATION]` before spawning agents
|
||||
- Use Edit tool directly when `[AGENT_BOOSTER_AVAILABLE]`
|
||||
|
||||
## Swarm Configuration & Anti-Drift
|
||||
|
||||
- ALWAYS use hierarchical topology for coding swarms
|
||||
- Keep maxAgents at 6-8 for tight coordination
|
||||
- Use specialized strategy for clear role boundaries
|
||||
- Use `raft` consensus for hive-mind (leader maintains authoritative state)
|
||||
- Run frequent checkpoints via `post-task` hooks
|
||||
- Keep shared memory namespace for all agents
|
||||
|
||||
```bash
|
||||
npx @claude-flow/cli@latest swarm init --topology hierarchical --max-agents 8 --strategy specialized
|
||||
```
|
||||
|
||||
## Swarm Execution Rules
|
||||
|
||||
- ALWAYS use `run_in_background: true` for all agent Task calls
|
||||
- ALWAYS put ALL agent Task calls in ONE message for parallel execution
|
||||
- After spawning, STOP — do NOT add more tool calls or check status
|
||||
- Never poll TaskOutput or check swarm status — trust agents to return
|
||||
- When agent results arrive, review ALL results before proceeding
|
||||
|
||||
## V3 CLI Commands
|
||||
|
||||
### Core Commands
|
||||
|
||||
| Command | Subcommands | Description |
|
||||
|---------|-------------|-------------|
|
||||
| `init` | 4 | Project initialization |
|
||||
| `agent` | 8 | Agent lifecycle management |
|
||||
| `swarm` | 6 | Multi-agent swarm coordination |
|
||||
| `memory` | 11 | AgentDB memory with HNSW search |
|
||||
| `task` | 6 | Task creation and lifecycle |
|
||||
| `session` | 7 | Session state management |
|
||||
| `hooks` | 17 | Self-learning hooks + 12 workers |
|
||||
| `hive-mind` | 6 | Byzantine fault-tolerant consensus |
|
||||
|
||||
### Quick CLI Examples
|
||||
|
||||
```bash
|
||||
npx @claude-flow/cli@latest init --wizard
|
||||
npx @claude-flow/cli@latest agent spawn -t coder --name my-coder
|
||||
npx @claude-flow/cli@latest swarm init --v3-mode
|
||||
npx @claude-flow/cli@latest memory search --query "authentication patterns"
|
||||
npx @claude-flow/cli@latest doctor --fix
|
||||
```
|
||||
|
||||
## Available Agents (60+ Types)
|
||||
|
||||
### Core Development
|
||||
`coder`, `reviewer`, `tester`, `planner`, `researcher`
|
||||
|
||||
### Specialized
|
||||
`security-architect`, `security-auditor`, `memory-specialist`, `performance-engineer`
|
||||
|
||||
### Swarm Coordination
|
||||
`hierarchical-coordinator`, `mesh-coordinator`, `adaptive-coordinator`
|
||||
|
||||
### GitHub & Repository
|
||||
`pr-manager`, `code-review-swarm`, `issue-tracker`, `release-manager`
|
||||
|
||||
### SPARC Methodology
|
||||
`sparc-coord`, `sparc-coder`, `specification`, `pseudocode`, `architecture`
|
||||
|
||||
## Memory Commands Reference
|
||||
|
||||
```bash
|
||||
# Store (REQUIRED: --key, --value; OPTIONAL: --namespace, --ttl, --tags)
|
||||
npx @claude-flow/cli@latest memory store --key "pattern-auth" --value "JWT with refresh" --namespace patterns
|
||||
|
||||
# Search (REQUIRED: --query; OPTIONAL: --namespace, --limit, --threshold)
|
||||
npx @claude-flow/cli@latest memory search --query "authentication patterns"
|
||||
|
||||
# List (OPTIONAL: --namespace, --limit)
|
||||
npx @claude-flow/cli@latest memory list --namespace patterns --limit 10
|
||||
|
||||
# Retrieve (REQUIRED: --key; OPTIONAL: --namespace)
|
||||
npx @claude-flow/cli@latest memory retrieve --key "pattern-auth" --namespace patterns
|
||||
```
|
||||
|
||||
## Quick Setup
|
||||
|
||||
```bash
|
||||
claude mcp add claude-flow -- npx -y @claude-flow/cli@latest
|
||||
npx @claude-flow/cli@latest daemon start
|
||||
npx @claude-flow/cli@latest doctor --fix
|
||||
```
|
||||
|
||||
## Claude Code vs CLI Tools
|
||||
|
||||
- Claude Code's Task tool handles ALL execution: agents, file ops, code generation, git
|
||||
- CLI tools handle coordination via Bash: swarm init, memory, hooks, routing
|
||||
- NEVER use CLI tools as a substitute for Task tool agents
|
||||
|
||||
## Support
|
||||
|
||||
- Documentation: https://github.com/ruvnet/claude-flow
|
||||
- Issues: https://github.com/ruvnet/claude-flow/issues
|
||||
@@ -26,4 +26,9 @@ EXPOSE 8080
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
#Prevent Python from writing .pyc files and __pycache__ folders to disk
|
||||
#Make the runtime faster
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
CMD ["python", "-m", "v1.src.sensing.ws_server"]
|
||||
|
||||
@@ -42,5 +42,15 @@ EXPOSE 5005/udp
|
||||
|
||||
ENV RUST_LOG=info
|
||||
|
||||
ENTRYPOINT ["/app/sensing-server"]
|
||||
CMD ["--source", "simulated", "--tick-ms", "100", "--ui-path", "/app/ui", "--http-port", "3000", "--ws-port", "3001"]
|
||||
# CSI_SOURCE controls which data source the sensing server uses at startup.
|
||||
# auto — probe UDP port 5005 for an ESP32 first; fall back to simulation (default)
|
||||
# esp32 — receive real CSI frames from an ESP32 device over UDP port 5005
|
||||
# wifi — use host Wi-Fi RSSI/scan data (Windows netsh; not available in containers)
|
||||
# simulated — generate synthetic CSI frames (no hardware required)
|
||||
# Override at runtime: docker run -e CSI_SOURCE=esp32 ...
|
||||
ENV CSI_SOURCE=auto
|
||||
|
||||
ENTRYPOINT ["/bin/sh", "-c"]
|
||||
# Shell-form CMD allows $CSI_SOURCE to be substituted at container start.
|
||||
# The ENV default above (CSI_SOURCE=auto) applies when the variable is unset.
|
||||
CMD ["/app/sensing-server --source ${CSI_SOURCE} --tick-ms 100 --ui-path /app/ui --http-port 3000 --ws-port 3001"]
|
||||
|
||||
@@ -12,7 +12,14 @@ services:
|
||||
- "5005:5005/udp" # ESP32 UDP
|
||||
environment:
|
||||
- RUST_LOG=info
|
||||
command: ["--source", "simulated", "--tick-ms", "100", "--ui-path", "/app/ui", "--http-port", "3000", "--ws-port", "3001"]
|
||||
# CSI_SOURCE controls the data source for the sensing server.
|
||||
# Options: auto (default) — probe for ESP32 UDP then fall back to simulation
|
||||
# esp32 — receive real CSI frames from an ESP32 on UDP port 5005
|
||||
# wifi — use host Wi-Fi RSSI/scan data (Windows netsh)
|
||||
# simulated — generate synthetic CSI data (no hardware required)
|
||||
- CSI_SOURCE=${CSI_SOURCE:-auto}
|
||||
# command is passed as arguments to ENTRYPOINT (/bin/sh -c), so $CSI_SOURCE is expanded by the shell.
|
||||
command: ["/app/sensing-server --source ${CSI_SOURCE:-auto} --tick-ms 100 --ui-path /app/ui --http-port 3000 --ws-port 3001"]
|
||||
|
||||
python-sensing:
|
||||
build:
|
||||
|
||||
@@ -0,0 +1,688 @@
|
||||
# ADR-034: Expo React Native Mobile Application
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-03-02 |
|
||||
| **Deciders** | MaTriXy, rUv |
|
||||
| **Codename** | **FieldView** -- Mobile Companion for WiFi-DensePose Field Deployment |
|
||||
| **Relates to** | ADR-019 (Sensing-Only UI Mode), ADR-021 (Vital Sign Detection), ADR-026 (Survivor Track Lifecycle), ADR-029 (RuvSense Multistatic), ADR-031 (RuView Sensing-First RF), ADR-032 (Mesh Security) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
### 1.1 Need for a Mobile Companion
|
||||
|
||||
WiFi-DensePose is a WiFi-based human pose estimation system using Channel State Information (CSI) from ESP32 mesh nodes. The existing web UI (`ui/`) serves desktop browsers but is not optimized for mobile form factors. Three deployment scenarios demand a purpose-built mobile application:
|
||||
|
||||
1. **Disaster response (WiFi-MAT)**: First responders deploying ESP32 mesh nodes in collapsed structures need a portable device to visualize survivor detections, breathing/heart rate vitals, and zone maps in real time. A laptop is impractical in rubble fields.
|
||||
2. **Building security**: Security operators patrolling a facility need a handheld display showing occupancy by zone, movement alerts, and historical patterns. The phone in their pocket is the natural form factor.
|
||||
3. **Healthcare monitoring**: Clinical staff monitoring patients via CSI-based contactless vitals need a tablet view at the bedside or nurse station, with gauges for breathing rate and heart rate that update in real time.
|
||||
|
||||
In all three scenarios, the mobile device does not communicate with ESP32 nodes directly. Instead, a Rust sensing server (`wifi-densepose-sensing-server`, ADR-031) aggregates ESP32 UDP streams and exposes a WebSocket API. The mobile app connects to this server over local WiFi.
|
||||
|
||||
### 1.2 Technology Selection Rationale
|
||||
|
||||
| Requirement | Decision | Rationale |
|
||||
|-------------|----------|-----------|
|
||||
| Cross-platform (iOS + Android + Web) | Expo SDK 55 + React Native 0.83 | Single codebase, managed workflow, OTA updates |
|
||||
| Real-time streaming | WebSocket (ws://host:3001/ws/sensing) | Sub-100ms latency from CSI capture to mobile display |
|
||||
| 3D visualization | Three.js Gaussian splat via WebView | Reuses existing `ui/` Three.js splat renderer; avoids native OpenGL binding |
|
||||
| State management | Zustand | Minimal boilerplate, React-concurrent safe, selector-based re-renders |
|
||||
| Persistence | AsyncStorage | Built into Expo, sufficient for settings and small cached state |
|
||||
| Navigation | react-navigation v7 (bottom tabs) | Standard React Native navigation; 5-tab layout fits mobile ergonomics |
|
||||
| WiFi RSSI scanning | Platform-specific (Android: react-native-wifi-reborn, iOS: CoreWLAN stub, Web: synthetic) | No cross-platform WiFi scanning API exists; platform modules are required |
|
||||
| E2E testing | Maestro YAML specs | Declarative, no Detox native build dependency, runs on CI |
|
||||
| Design system | Dark theme (#0D1117 bg, #32B8C6 accent) | Matches existing `ui/` sensing dashboard aesthetic; reduces eye strain in field conditions |
|
||||
|
||||
### 1.3 Relationship to Existing UI
|
||||
|
||||
The desktop web UI (`ui/`) and the mobile app share no code at the component level, but they consume the same backend APIs:
|
||||
|
||||
- **WebSocket**: `ws://host:3001/ws/sensing` -- streaming SensingFrame JSON
|
||||
- **REST**: `http://host:3000/api/v1/...` -- configuration, history, health
|
||||
|
||||
The mobile app's Three.js Gaussian splat viewer (LiveScreen) loads the same splat HTML bundle used by the desktop UI, rendered inside a WebView (native) or iframe (web).
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
Build an Expo React Native mobile application at `ui/mobile/` that provides five primary screens for field operators, connected to the Rust sensing server via WebSocket streaming. The app automatically falls back to simulated data when the sensing server is unreachable, enabling demos and offline testing.
|
||||
|
||||
### 2.1 Screen Architecture
|
||||
|
||||
```
|
||||
+---------------------------------------------------------------+
|
||||
| MainTabs (Bottom Tab Navigator) |
|
||||
+---------------------------------------------------------------+
|
||||
| |
|
||||
| +----------+ +----------+ +----------+ +--------+ +-----+ |
|
||||
| | Live | | Vitals | | Zones | | MAT | | Cog | |
|
||||
| | (3D splat| |(breathing| |(floor | |(disaster| |(set-| |
|
||||
| | + HUD) | | + heart) | | plan SVG)| |response)| |tings| |
|
||||
| +----------+ +----------+ +----------+ +--------+ +-----+ |
|
||||
| |
|
||||
+---------------------------------------------------------------+
|
||||
| ConnectionBanner (Connected / Simulated / Disconnected) |
|
||||
+---------------------------------------------------------------+
|
||||
```
|
||||
|
||||
**Screen responsibilities:**
|
||||
|
||||
| Screen | Primary View | Data Source | Key Components |
|
||||
|--------|-------------|-------------|----------------|
|
||||
| **Live** | 3D Gaussian splat with 17 COCO keypoints + HUD overlay | `poseStore.latestFrame` | `GaussianSplatWebView`, `LiveHUD`, `HudOverlay` |
|
||||
| **Vitals** | Breathing BPM gauge, heart rate BPM gauge, sparkline history | `poseStore.latestFrame.vital_signs` | `BreathingGauge`, `HeartRateGauge`, `MetricCard`, `SparklineChart` |
|
||||
| **Zones** | Floor plan SVG with occupancy heat overlay, zone legend | `poseStore.latestFrame.persons` | `FloorPlanSvg`, `OccupancyGrid`, `ZoneLegend` |
|
||||
| **MAT** | Survivor counter, zone map WebView, alert list | `matStore.survivors`, `matStore.alerts` | `SurvivorCounter`, `MatWebView`, `AlertList`, `AlertCard` |
|
||||
| **Settings** | Server URL input, theme picker, RSSI toggle | `settingsStore` | `ServerUrlInput`, `ThemePicker`, `RssiToggle` |
|
||||
|
||||
### 2.2 State Architecture
|
||||
|
||||
Three Zustand stores separate concerns and prevent unnecessary re-renders:
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Zustand Stores |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
| poseStore |
|
||||
| +--------------------------------------------------------+ |
|
||||
| | connectionStatus: 'connected' | 'simulated' | 'error' | |
|
||||
| | latestFrame: SensingFrame | null | |
|
||||
| | frameHistory: RingBuffer<SensingFrame> | |
|
||||
| | features: FeatureVector | null | |
|
||||
| | persons: Person[] | |
|
||||
| | vitalSigns: VitalSigns | null | |
|
||||
| +--------------------------------------------------------+ |
|
||||
| |
|
||||
| matStore |
|
||||
| +--------------------------------------------------------+ |
|
||||
| | survivors: Survivor[] | |
|
||||
| | alerts: MatAlert[] | |
|
||||
| | events: MatEvent[] | |
|
||||
| | zoneMap: ZoneMap | null | |
|
||||
| +--------------------------------------------------------+ |
|
||||
| |
|
||||
| settingsStore (persisted via AsyncStorage) |
|
||||
| +--------------------------------------------------------+ |
|
||||
| | serverUrl: string (default: 'http://localhost:3000') | |
|
||||
| | wsUrl: string (default: 'ws://localhost:3001') | |
|
||||
| | theme: 'dark' | 'light' | |
|
||||
| | rssiEnabled: boolean | |
|
||||
| | simulationMode: boolean | |
|
||||
| +--------------------------------------------------------+ |
|
||||
| |
|
||||
+------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### 2.3 Service Layer
|
||||
|
||||
Four services encapsulate external communication and data generation:
|
||||
|
||||
| Service | File | Responsibility |
|
||||
|---------|------|----------------|
|
||||
| `ws.service` | `src/services/ws.service.ts` | WebSocket connection lifecycle, reconnection with exponential backoff, SensingFrame parsing, dispatches to `poseStore` |
|
||||
| `api.service` | `src/services/api.service.ts` | REST calls to sensing server (health check, configuration, history endpoints) |
|
||||
| `rssi.service` | `src/services/rssi.service.ts` (+ platform variants) | Platform-specific WiFi RSSI scanning. Android uses `react-native-wifi-reborn`, iOS provides a CoreWLAN stub, Web generates synthetic RSSI values |
|
||||
| `simulation.service` | `src/services/simulation.service.ts` | Generates synthetic SensingFrame data when the real server is unreachable. Produces realistic amplitude, phase, vital signs, and person data on a configurable tick interval |
|
||||
|
||||
**Platform-specific RSSI service files:**
|
||||
|
||||
| File | Platform | Implementation |
|
||||
|------|----------|----------------|
|
||||
| `rssi.service.android.ts` | Android | `react-native-wifi-reborn` native module, requires `ACCESS_FINE_LOCATION` permission |
|
||||
| `rssi.service.ios.ts` | iOS | CoreWLAN stub (returns empty scan results; Apple restricts WiFi scanning to system apps) |
|
||||
| `rssi.service.web.ts` | Web | Synthetic RSSI values generated from noise model |
|
||||
| `rssi.service.ts` | Default | Re-exports platform-appropriate module via React Native file resolution |
|
||||
|
||||
### 2.4 Data Flow
|
||||
|
||||
```
|
||||
ESP32 Mesh Nodes
|
||||
|
|
||||
| UDP CSI frames (ADR-029 TDM protocol)
|
||||
v
|
||||
+---------------------------+
|
||||
| Rust Sensing Server |
|
||||
| (wifi-densepose-sensing- |
|
||||
| server, ADR-031) |
|
||||
| |
|
||||
| Aggregates ESP32 streams |
|
||||
| Runs RuvSense pipeline |
|
||||
| Exposes WS + REST APIs |
|
||||
+---------------------------+
|
||||
| |
|
||||
| WebSocket | REST
|
||||
| ws://host:3001 | http://host:3000
|
||||
| /ws/sensing | /api/v1/...
|
||||
v v
|
||||
+---------------------------+
|
||||
| Expo Mobile App |
|
||||
| |
|
||||
| ws.service |
|
||||
| -> poseStore |
|
||||
| -> matStore |
|
||||
| |
|
||||
| Screens subscribe to |
|
||||
| stores via Zustand |
|
||||
| selectors |
|
||||
+---------------------------+
|
||||
```
|
||||
|
||||
**Connection lifecycle:**
|
||||
|
||||
1. App boots. `settingsStore` loads persisted server URL from AsyncStorage.
|
||||
2. `ws.service` opens WebSocket to `wsUrl/ws/sensing`.
|
||||
3. On each message, `ws.service` parses the `SensingFrame` JSON and dispatches to `poseStore`.
|
||||
4. If the WebSocket fails, `ws.service` retries with exponential backoff (1s, 2s, 4s, 8s, 16s max).
|
||||
5. After `MAX_RECONNECT_ATTEMPTS` (5) consecutive failures, `ws.service` switches to `simulation.service`, which generates synthetic frames at 10 Hz.
|
||||
6. `poseStore.connectionStatus` transitions: `connected` -> `error` -> `simulated`.
|
||||
7. `ConnectionBanner` component reflects the current status on all screens.
|
||||
8. If the server becomes reachable again, `ws.service` reconnects and resumes live data.
|
||||
|
||||
### 2.5 SensingFrame JSON Schema
|
||||
|
||||
The WebSocket stream delivers JSON frames matching the Rust `SensingFrame` struct:
|
||||
|
||||
```typescript
|
||||
interface SensingFrame {
|
||||
timestamp: number; // Unix epoch ms
|
||||
amplitude: number[]; // Per-subcarrier amplitude (52 or 114 values)
|
||||
phase: number[]; // Per-subcarrier phase (radians)
|
||||
features: {
|
||||
mean_amplitude: number;
|
||||
std_amplitude: number;
|
||||
phase_slope: number;
|
||||
doppler_shift: number;
|
||||
delay_spread: number;
|
||||
};
|
||||
classification: string; // "empty" | "single_person" | "multi_person" | "motion"
|
||||
confidence: number; // 0.0 - 1.0
|
||||
persons: Array<{
|
||||
id: number;
|
||||
keypoints: Array<[number, number, number]>; // 17 COCO keypoints [x, y, confidence]
|
||||
bbox: [number, number, number, number]; // [x, y, width, height]
|
||||
track_id: number;
|
||||
}>;
|
||||
vital_signs?: {
|
||||
breathing_rate_bpm: number;
|
||||
heart_rate_bpm: number;
|
||||
breathing_confidence: number;
|
||||
heart_confidence: number;
|
||||
};
|
||||
rssi?: number;
|
||||
node_id?: number;
|
||||
}
|
||||
```
|
||||
|
||||
### 2.6 Three.js Gaussian Splat Rendering
|
||||
|
||||
The LiveScreen uses a WebView (native) or iframe (web) to render a Three.js Gaussian splat scene. This avoids native OpenGL bindings while reusing the existing splat renderer from the desktop UI.
|
||||
|
||||
**Native path (iOS/Android):**
|
||||
- `GaussianSplatWebView.tsx` renders a `<WebView>` loading a bundled HTML page.
|
||||
- The HTML page initializes a Three.js scene with Gaussian splat shaders.
|
||||
- Communication between React Native and the WebView uses `postMessage` / `onMessage` bridge.
|
||||
- `useGaussianBridge.ts` hook manages the bridge, sending skeleton keypoint updates as JSON.
|
||||
|
||||
**Web path:**
|
||||
- `GaussianSplatWebView.web.tsx` (platform-specific file) renders an `<iframe>` with the same HTML bundle.
|
||||
- Communication uses `window.postMessage` with origin checks.
|
||||
|
||||
### 2.7 Design System
|
||||
|
||||
| Token | Value | Usage |
|
||||
|-------|-------|-------|
|
||||
| `colors.background` | `#0D1117` | Primary background (dark theme) |
|
||||
| `colors.surface` | `#161B22` | Card/panel backgrounds |
|
||||
| `colors.border` | `#30363D` | Borders, dividers |
|
||||
| `colors.accent` | `#32B8C6` | Primary accent, active tab, gauge fill |
|
||||
| `colors.danger` | `#F85149` | Alerts, errors, critical vitals |
|
||||
| `colors.warning` | `#D29922` | Warnings, degraded state |
|
||||
| `colors.success` | `#3FB950` | Connected status, normal vitals |
|
||||
| `colors.text` | `#E6EDF3` | Primary text |
|
||||
| `colors.textSecondary` | `#8B949E` | Secondary/muted text |
|
||||
| `typography.mono` | `Courier New` | Monospace for data values, HUD |
|
||||
| `spacing.xs` | `4` | Tight spacing |
|
||||
| `spacing.sm` | `8` | Small spacing |
|
||||
| `spacing.md` | `16` | Medium spacing |
|
||||
| `spacing.lg` | `24` | Large spacing |
|
||||
| `spacing.xl` | `32` | Extra-large spacing |
|
||||
|
||||
The dark theme is the default and primary design target, optimized for field conditions (low ambient light, glare reduction). A light theme variant is available via the Settings screen.
|
||||
|
||||
### 2.8 ESP32 Integration Model
|
||||
|
||||
The mobile app does not communicate with ESP32 nodes directly. The architecture is:
|
||||
|
||||
```
|
||||
ESP32 Node A ---\
|
||||
ESP32 Node B ----+---> Sensing Server (Raspberry Pi / Laptop) <---> Mobile App
|
||||
ESP32 Node C ---/ (local WiFi) (local WiFi)
|
||||
```
|
||||
|
||||
- **Field deployment**: The sensing server runs on a Raspberry Pi 4 or operator laptop. All devices (ESP32 nodes, server, mobile app) connect to the same local WiFi network or a portable router.
|
||||
- **Server URL**: Configurable in Settings screen. Default: `http://localhost:3000` (server) and `ws://localhost:3001/ws/sensing` (WebSocket). In field use, the operator sets this to the server's LAN IP (e.g., `http://192.168.1.100:3000`).
|
||||
- **No BLE/direct connection**: ESP32 nodes use UDP broadcast for CSI frames (ADR-029). The mobile app has no UDP listener; it consumes the server's processed output.
|
||||
|
||||
---
|
||||
|
||||
## 3. Directory Structure
|
||||
|
||||
```
|
||||
ui/mobile/
|
||||
|-- App.tsx # Root component, ThemeProvider + NavigationContainer
|
||||
|-- app.config.ts # Expo config (SDK 55, app name, icons, splash)
|
||||
|-- app.json # Expo static config
|
||||
|-- babel.config.js # Babel config (expo-router preset)
|
||||
|-- eas.json # EAS Build profiles (dev, preview, production)
|
||||
|-- index.ts # Entry point (registerRootComponent)
|
||||
|-- jest.config.js # Jest config for unit tests
|
||||
|-- jest.setup.ts # Jest setup (mock AsyncStorage, react-native modules)
|
||||
|-- metro.config.js # Metro bundler config
|
||||
|-- package.json # Dependencies and scripts
|
||||
|-- tsconfig.json # TypeScript config (strict mode)
|
||||
|
|
||||
|-- assets/
|
||||
| |-- android-icon-background.png # Android adaptive icon background
|
||||
| |-- android-icon-foreground.png # Android adaptive icon foreground
|
||||
| |-- android-icon-monochrome.png # Android monochrome icon
|
||||
| |-- favicon.png # Web favicon
|
||||
| |-- icon.png # App icon (1024x1024)
|
||||
| |-- splash-icon.png # Splash screen icon
|
||||
|
|
||||
|-- e2e/ # Maestro E2E test specs
|
||||
| |-- live_screen.yaml # LiveScreen: splat renders, HUD shows data
|
||||
| |-- vitals_screen.yaml # VitalsScreen: gauges animate, sparklines update
|
||||
| |-- zones_screen.yaml # ZonesScreen: floor plan renders, legend visible
|
||||
| |-- mat_screen.yaml # MATScreen: survivor count, alerts list
|
||||
| |-- settings_screen.yaml # SettingsScreen: URL input, theme toggle
|
||||
| |-- offline_fallback.yaml # Simulated mode activates on server disconnect
|
||||
|
|
||||
|-- src/
|
||||
| |-- components/ # Shared UI components (12 components)
|
||||
| | |-- ConnectionBanner.tsx # Status banner: Connected/Simulated/Disconnected
|
||||
| | |-- ErrorBoundary.tsx # React error boundary with fallback UI
|
||||
| | |-- GaugeArc.tsx # SVG arc gauge (used by vitals)
|
||||
| | |-- HudOverlay.tsx # Translucent HUD overlay for LiveScreen
|
||||
| | |-- LoadingSpinner.tsx # Animated loading indicator
|
||||
| | |-- ModeBadge.tsx # Badge showing current mode (Live/Sim)
|
||||
| | |-- OccupancyGrid.tsx # Grid overlay for zone occupancy
|
||||
| | |-- SignalBar.tsx # WiFi signal strength bar
|
||||
| | |-- SparklineChart.tsx # Inline sparkline chart (SVG)
|
||||
| | |-- StatusDot.tsx # Colored status dot indicator
|
||||
| | |-- ThemedText.tsx # Text component with theme support
|
||||
| | |-- ThemedView.tsx # View component with theme support
|
||||
| |
|
||||
| |-- constants/ # App-wide constants
|
||||
| | |-- api.ts # REST API endpoint paths, timeouts
|
||||
| | |-- simulation.ts # Simulation tick rate, data ranges
|
||||
| | |-- websocket.ts # WS reconnect config, max attempts
|
||||
| |
|
||||
| |-- hooks/ # Custom React hooks (5 hooks)
|
||||
| | |-- usePoseStream.ts # Subscribe to poseStore, manage WS lifecycle
|
||||
| | |-- useRssiScanner.ts # Platform RSSI scanning with permission handling
|
||||
| | |-- useServerReachability.ts # Periodic health check, reachability state
|
||||
| | |-- useTheme.ts # Theme context consumer
|
||||
| | |-- useWebViewBridge.ts # WebView <-> RN message bridge
|
||||
| |
|
||||
| |-- navigation/ # React Navigation setup
|
||||
| | |-- MainTabs.tsx # Bottom tab navigator (5 tabs)
|
||||
| | |-- RootNavigator.tsx # Root stack (splash -> MainTabs)
|
||||
| | |-- types.ts # Navigation type definitions
|
||||
| |
|
||||
| |-- screens/ # Screen modules (5 screens)
|
||||
| | |-- LiveScreen/
|
||||
| | | |-- index.tsx # LiveScreen container
|
||||
| | | |-- GaussianSplatWebView.tsx # Native: WebView 3D splat
|
||||
| | | |-- GaussianSplatWebView.web.tsx # Web: iframe 3D splat
|
||||
| | | |-- LiveHUD.tsx # Heads-up display overlay
|
||||
| | | |-- useGaussianBridge.ts # Bridge hook for splat WebView
|
||||
| | |
|
||||
| | |-- VitalsScreen/
|
||||
| | | |-- index.tsx # VitalsScreen container
|
||||
| | | |-- BreathingGauge.tsx # Breathing rate arc gauge
|
||||
| | | |-- HeartRateGauge.tsx # Heart rate arc gauge
|
||||
| | | |-- MetricCard.tsx # Metric display card
|
||||
| | |
|
||||
| | |-- ZonesScreen/
|
||||
| | | |-- index.tsx # ZonesScreen container
|
||||
| | | |-- FloorPlanSvg.tsx # SVG floor plan with occupancy overlay
|
||||
| | | |-- useOccupancyGrid.ts # Occupancy grid computation hook
|
||||
| | | |-- ZoneLegend.tsx # Zone color legend
|
||||
| | |
|
||||
| | |-- MATScreen/
|
||||
| | | |-- index.tsx # MATScreen container
|
||||
| | | |-- SurvivorCounter.tsx # Large survivor count display
|
||||
| | | |-- MatWebView.tsx # WebView for MAT zone map
|
||||
| | | |-- AlertList.tsx # Scrollable alert list
|
||||
| | | |-- AlertCard.tsx # Individual alert card
|
||||
| | | |-- useMatBridge.ts # Bridge hook for MAT WebView
|
||||
| | |
|
||||
| | |-- SettingsScreen/
|
||||
| | |-- index.tsx # SettingsScreen container
|
||||
| | |-- ServerUrlInput.tsx # Server URL text input with validation
|
||||
| | |-- ThemePicker.tsx # Dark/light theme toggle
|
||||
| | |-- RssiToggle.tsx # RSSI scanning enable/disable
|
||||
| |
|
||||
| |-- services/ # External communication services (4 services)
|
||||
| | |-- ws.service.ts # WebSocket client with reconnection
|
||||
| | |-- api.service.ts # REST API client (fetch-based)
|
||||
| | |-- rssi.service.ts # Default RSSI service (platform re-export)
|
||||
| | |-- rssi.service.android.ts # Android RSSI via react-native-wifi-reborn
|
||||
| | |-- rssi.service.ios.ts # iOS CoreWLAN stub
|
||||
| | |-- rssi.service.web.ts # Web synthetic RSSI
|
||||
| | |-- simulation.service.ts # Synthetic SensingFrame generator
|
||||
| |
|
||||
| |-- stores/ # Zustand state stores (3 stores)
|
||||
| | |-- poseStore.ts # Connection state, frames, features, persons
|
||||
| | |-- matStore.ts # Survivors, alerts, events, zone map
|
||||
| | |-- settingsStore.ts # Server URL, theme, RSSI toggle (persisted)
|
||||
| |
|
||||
| |-- theme/ # Design system tokens
|
||||
| | |-- index.ts # Theme re-exports
|
||||
| | |-- colors.ts # Color palette (dark + light)
|
||||
| | |-- spacing.ts # Spacing scale
|
||||
| | |-- typography.ts # Font families and sizes
|
||||
| | |-- ThemeContext.tsx # React context for theme
|
||||
| |
|
||||
| |-- types/ # TypeScript type definitions
|
||||
| | |-- api.ts # REST API response types
|
||||
| | |-- html.d.ts # HTML asset module declaration
|
||||
| | |-- mat.ts # MAT domain types (Survivor, Alert, Event)
|
||||
| | |-- navigation.ts # Navigation param list types
|
||||
| | |-- react-native-wifi-reborn.d.ts # Type stubs for wifi-reborn
|
||||
| | |-- sensing.ts # SensingFrame, Person, VitalSigns types
|
||||
| |
|
||||
| |-- utils/ # Utility functions
|
||||
| | |-- colorMap.ts # Value-to-color mapping for gauges
|
||||
| | |-- formatters.ts # Number/date formatting helpers
|
||||
| | |-- ringBuffer.ts # Fixed-size ring buffer for frame history
|
||||
| | |-- urlValidator.ts # Server URL validation
|
||||
| |
|
||||
| |-- __tests__/ # Unit tests (mirroring src/ structure)
|
||||
| |-- test-utils.tsx # Test utilities, render helpers, mocks
|
||||
| |-- components/ # Component unit tests (7 test files)
|
||||
| |-- hooks/ # Hook unit tests (3 test files)
|
||||
| |-- screens/ # Screen unit tests (5 test files)
|
||||
| |-- services/ # Service unit tests (4 test files)
|
||||
| |-- stores/ # Store unit tests (3 test files)
|
||||
| |-- utils/ # Utility unit tests (3 test files)
|
||||
```
|
||||
|
||||
**File count summary:**
|
||||
|
||||
| Category | Files |
|
||||
|----------|-------|
|
||||
| Source (components, screens, services, stores, hooks, utils, types, theme, navigation) | 63 `.ts`/`.tsx` files |
|
||||
| Unit tests | 25 test files |
|
||||
| E2E tests (Maestro) | 6 YAML specs |
|
||||
| Config (babel, metro, jest, tsconfig, eas, app) | 7 config files |
|
||||
| Assets | 6 image files |
|
||||
| **Total** | **107 files** |
|
||||
|
||||
---
|
||||
|
||||
## 4. Implementation Plan (File-Level)
|
||||
|
||||
### 4.1 Phase 1: Core Infrastructure
|
||||
|
||||
| File | Purpose | Priority |
|
||||
|------|---------|----------|
|
||||
| `App.tsx` | Root component with ThemeProvider and NavigationContainer | P0 |
|
||||
| `index.ts` | Expo entry point | P0 |
|
||||
| `app.config.ts` | Expo SDK 55 configuration | P0 |
|
||||
| `src/theme/colors.ts` | Dark and light color palettes | P0 |
|
||||
| `src/theme/spacing.ts` | Spacing scale | P0 |
|
||||
| `src/theme/typography.ts` | Font definitions | P0 |
|
||||
| `src/theme/ThemeContext.tsx` | React context provider for theme | P0 |
|
||||
| `src/navigation/MainTabs.tsx` | Bottom tab navigator with 5 tabs | P0 |
|
||||
| `src/navigation/RootNavigator.tsx` | Root stack navigator | P0 |
|
||||
| `src/types/sensing.ts` | SensingFrame, Person, VitalSigns type definitions | P0 |
|
||||
|
||||
### 4.2 Phase 2: State and Services
|
||||
|
||||
| File | Purpose | Priority |
|
||||
|------|---------|----------|
|
||||
| `src/stores/poseStore.ts` | Zustand store for connection state, frames, persons | P0 |
|
||||
| `src/stores/matStore.ts` | Zustand store for MAT survivors, alerts, events | P0 |
|
||||
| `src/stores/settingsStore.ts` | Zustand store with AsyncStorage persistence | P0 |
|
||||
| `src/services/ws.service.ts` | WebSocket client with reconnection and dispatch | P0 |
|
||||
| `src/services/api.service.ts` | REST API client | P1 |
|
||||
| `src/services/simulation.service.ts` | Synthetic SensingFrame generator for fallback | P0 |
|
||||
| `src/services/rssi.service.ts` | Platform RSSI re-export | P1 |
|
||||
| `src/services/rssi.service.android.ts` | Android react-native-wifi-reborn integration | P1 |
|
||||
| `src/services/rssi.service.ios.ts` | iOS CoreWLAN stub | P2 |
|
||||
| `src/services/rssi.service.web.ts` | Web synthetic RSSI | P1 |
|
||||
| `src/utils/ringBuffer.ts` | Fixed-size ring buffer for frame history | P0 |
|
||||
| `src/utils/urlValidator.ts` | Server URL validation | P1 |
|
||||
|
||||
### 4.3 Phase 3: Shared Components
|
||||
|
||||
| File | Purpose | Priority |
|
||||
|------|---------|----------|
|
||||
| `src/components/ConnectionBanner.tsx` | Status banner across all screens | P0 |
|
||||
| `src/components/GaugeArc.tsx` | SVG arc gauge for vitals | P0 |
|
||||
| `src/components/SparklineChart.tsx` | Inline sparkline for history | P0 |
|
||||
| `src/components/OccupancyGrid.tsx` | Grid overlay for zones | P1 |
|
||||
| `src/components/StatusDot.tsx` | Colored status indicator | P1 |
|
||||
| `src/components/SignalBar.tsx` | WiFi signal strength display | P1 |
|
||||
| `src/components/ModeBadge.tsx` | Live/Sim mode badge | P1 |
|
||||
| `src/components/ErrorBoundary.tsx` | React error boundary | P0 |
|
||||
| `src/components/LoadingSpinner.tsx` | Loading state indicator | P1 |
|
||||
| `src/components/ThemedText.tsx` | Themed text component | P0 |
|
||||
| `src/components/ThemedView.tsx` | Themed view component | P0 |
|
||||
| `src/components/HudOverlay.tsx` | Translucent HUD for Live screen | P1 |
|
||||
|
||||
### 4.4 Phase 4: Screens
|
||||
|
||||
| File | Purpose | Priority |
|
||||
|------|---------|----------|
|
||||
| `src/screens/LiveScreen/index.tsx` | Live 3D splat + HUD | P0 |
|
||||
| `src/screens/LiveScreen/GaussianSplatWebView.tsx` | Native WebView for splat | P0 |
|
||||
| `src/screens/LiveScreen/GaussianSplatWebView.web.tsx` | Web iframe for splat | P1 |
|
||||
| `src/screens/LiveScreen/LiveHUD.tsx` | HUD overlay with metrics | P1 |
|
||||
| `src/screens/LiveScreen/useGaussianBridge.ts` | WebView bridge hook | P0 |
|
||||
| `src/screens/VitalsScreen/index.tsx` | Vitals gauges and sparklines | P0 |
|
||||
| `src/screens/VitalsScreen/BreathingGauge.tsx` | Breathing rate gauge | P0 |
|
||||
| `src/screens/VitalsScreen/HeartRateGauge.tsx` | Heart rate gauge | P0 |
|
||||
| `src/screens/VitalsScreen/MetricCard.tsx` | Vitals metric card | P1 |
|
||||
| `src/screens/ZonesScreen/index.tsx` | Floor plan with occupancy | P1 |
|
||||
| `src/screens/ZonesScreen/FloorPlanSvg.tsx` | SVG floor plan renderer | P1 |
|
||||
| `src/screens/ZonesScreen/useOccupancyGrid.ts` | Occupancy computation | P1 |
|
||||
| `src/screens/ZonesScreen/ZoneLegend.tsx` | Zone legend | P2 |
|
||||
| `src/screens/MATScreen/index.tsx` | MAT dashboard | P1 |
|
||||
| `src/screens/MATScreen/SurvivorCounter.tsx` | Survivor count display | P1 |
|
||||
| `src/screens/MATScreen/MatWebView.tsx` | MAT zone map WebView | P1 |
|
||||
| `src/screens/MATScreen/AlertList.tsx` | Alert list | P1 |
|
||||
| `src/screens/MATScreen/AlertCard.tsx` | Alert card | P2 |
|
||||
| `src/screens/MATScreen/useMatBridge.ts` | MAT WebView bridge | P1 |
|
||||
| `src/screens/SettingsScreen/index.tsx` | Settings form | P0 |
|
||||
| `src/screens/SettingsScreen/ServerUrlInput.tsx` | Server URL input | P0 |
|
||||
| `src/screens/SettingsScreen/ThemePicker.tsx` | Theme toggle | P2 |
|
||||
| `src/screens/SettingsScreen/RssiToggle.tsx` | RSSI toggle | P2 |
|
||||
|
||||
### 4.5 Phase 5: Testing
|
||||
|
||||
| File | Purpose | Priority |
|
||||
|------|---------|----------|
|
||||
| `src/__tests__/stores/poseStore.test.ts` | Store state transitions, frame processing | P0 |
|
||||
| `src/__tests__/stores/matStore.test.ts` | MAT store state management | P1 |
|
||||
| `src/__tests__/stores/settingsStore.test.ts` | Persistence, defaults | P1 |
|
||||
| `src/__tests__/services/ws.service.test.ts` | WS connection, reconnection, fallback | P0 |
|
||||
| `src/__tests__/services/simulation.service.test.ts` | Synthetic frame generation | P1 |
|
||||
| `src/__tests__/services/api.service.test.ts` | REST client mocking | P1 |
|
||||
| `src/__tests__/services/rssi.service.test.ts` | Platform RSSI mocking | P2 |
|
||||
| `src/__tests__/components/*.test.tsx` | Component render tests (7 files) | P1 |
|
||||
| `src/__tests__/hooks/*.test.ts` | Hook behavior tests (3 files) | P1 |
|
||||
| `src/__tests__/screens/*.test.tsx` | Screen integration tests (5 files) | P1 |
|
||||
| `src/__tests__/utils/*.test.ts` | Utility function tests (3 files) | P1 |
|
||||
| `e2e/*.yaml` | Maestro E2E specs (6 files) | P2 |
|
||||
|
||||
---
|
||||
|
||||
## 5. Acceptance Criteria
|
||||
|
||||
### 5.1 Build and Platform Support
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| B-1 | App builds successfully with `npx expo start` for iOS, Android, and Web | CI build matrix: `expo start --ios`, `--android`, `--web` |
|
||||
| B-2 | App runs on iOS Simulator (iPhone 15 Pro, iOS 17+) | Manual verification on Simulator |
|
||||
| B-3 | App runs on Android Emulator (API 34+) | Manual verification on Emulator |
|
||||
| B-4 | App runs in web browser (Chrome 120+, Safari 17+, Firefox 120+) | Manual verification in browsers |
|
||||
| B-5 | TypeScript compiles with zero errors in strict mode | `npx tsc --noEmit` in CI |
|
||||
|
||||
### 5.2 WebSocket and Data Streaming
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| W-1 | WebSocket connects to sensing server and receives SensingFrame JSON | Integration test: start server, verify `poseStore.connectionStatus === 'connected'` |
|
||||
| W-2 | `poseStore.latestFrame` updates within 100ms of WebSocket message receipt | Unit test: mock WS, measure dispatch latency |
|
||||
| W-3 | WebSocket reconnects with exponential backoff after connection loss | Unit test: simulate WS close, verify retry intervals (1s, 2s, 4s, 8s, 16s) |
|
||||
| W-4 | Automatic fallback to simulated data within 5 seconds of connection failure | Unit test: fail WS 5 times, verify `connectionStatus === 'simulated'` within 5s |
|
||||
| W-5 | App recovers gracefully from sensing server restart (reconnects without crash) | Integration test: kill server, restart, verify reconnection and `connectionStatus === 'connected'` |
|
||||
|
||||
### 5.3 Screen Rendering
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| S-1 | All 5 screens render correctly with live data from sensing server | Integration test: connect to server, navigate all tabs, verify content |
|
||||
| S-2 | All 5 screens render correctly with simulated data | Unit test: set `connectionStatus = 'simulated'`, verify all screens render |
|
||||
| S-3 | Vital signs gauges animate smoothly (breathing BPM, heart rate BPM) | Visual inspection: gauges update at frame rate without jank |
|
||||
| S-4 | 3D Gaussian splat viewer shows skeleton with 17 COCO keypoints | Integration test: verify WebView loads, bridge sends keypoints, splat renders |
|
||||
| S-5 | Floor plan SVG updates with occupancy data when persons are detected | Unit test: inject 3 persons into poseStore, verify 3 markers on FloorPlanSvg |
|
||||
| S-6 | MAT dashboard shows survivor count, zone map, and alert list | Unit test: inject matStore data, verify SurvivorCounter and AlertList render |
|
||||
| S-7 | Connection banner shows correct status text and color for all 3 states | Unit test: cycle through `connected`/`simulated`/`error`, verify banner text and color |
|
||||
|
||||
### 5.4 Persistence and Settings
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| P-1 | Settings persist across app restarts (server URL, theme, RSSI toggle) | Integration test: set values, kill app, restart, verify values restored |
|
||||
| P-2 | Default server URL is `http://localhost:3000` when no persisted value exists | Unit test: clear AsyncStorage, verify default |
|
||||
| P-3 | Server URL input validates format before saving | Unit test: submit `not-a-url`, verify rejection; submit `http://192.168.1.1:3000`, verify acceptance |
|
||||
|
||||
### 5.5 Navigation and UX
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| N-1 | Bottom tab navigation works with correct icons for all 5 tabs | E2E: Maestro navigates all tabs, verifies active state |
|
||||
| N-2 | Dark theme renders correctly on all platforms (background #0D1117, accent #32B8C6) | Visual inspection on iOS, Android, Web |
|
||||
| N-3 | No infinite render loops or memory leaks in stores | Unit test: mount all screens, process 1000 frames, verify no memory growth beyond ring buffer size |
|
||||
| N-4 | ErrorBoundary catches and displays fallback UI for component errors | Unit test: throw in child component, verify fallback renders |
|
||||
|
||||
### 5.6 Platform-Specific Features
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| R-1 | RSSI scanning works on Android with react-native-wifi-reborn | Manual test on Android device with location permission granted |
|
||||
| R-2 | iOS RSSI service returns empty results without crashing | Unit test: call `scanNetworks()` on iOS, verify empty array returned |
|
||||
| R-3 | Web RSSI service generates synthetic RSSI values | Unit test: call `scanNetworks()` on web, verify synthetic data returned |
|
||||
|
||||
### 5.7 Testing
|
||||
|
||||
| ID | Criterion | Test Method |
|
||||
|----|-----------|-------------|
|
||||
| T-1 | All unit tests pass (`npm test` exits 0) | CI: `cd ui/mobile && npm test` |
|
||||
| T-2 | E2E Maestro tests pass for all 5 screens | CI: `maestro test e2e/` |
|
||||
| T-3 | E2E offline fallback test passes (simulated mode activates on disconnect) | CI: `maestro test e2e/offline_fallback.yaml` |
|
||||
| T-4 | No TypeScript type errors | CI: `npx tsc --noEmit` |
|
||||
|
||||
---
|
||||
|
||||
## 6. Consequences
|
||||
|
||||
### 6.1 Positive
|
||||
|
||||
- **Single codebase for three platforms**: Expo SDK 55 with React Native 0.83 builds iOS, Android, and Web from the same TypeScript source, reducing development and maintenance cost by approximately 60% compared to separate native apps.
|
||||
- **Instant field deployment**: Operators can install the app via Expo Go (development) or EAS Build (production) and connect to a local sensing server within minutes. No server-side mobile infrastructure required.
|
||||
- **Sub-100ms display latency**: WebSocket streaming from the Rust sensing server to the mobile app introduces less than 100ms additional latency beyond the CSI processing pipeline, providing near-real-time visualization.
|
||||
- **Offline-capable demos**: The simulation service generates realistic synthetic SensingFrame data, enabling demonstrations to stakeholders and testing without ESP32 hardware or a running sensing server.
|
||||
- **Operator-friendly UX**: Five purpose-built screens cover the primary use cases (live view, vitals, zones, MAT, settings) with a bottom-tab navigation pattern familiar to mobile users.
|
||||
- **Testable architecture**: Zustand stores with selector-based subscriptions, service-layer abstraction, and Maestro E2E specs provide a comprehensive testing strategy from unit to integration to end-to-end.
|
||||
- **Reuses existing infrastructure**: The app consumes the same WebSocket and REST APIs as the desktop UI, requiring no backend changes. The Three.js splat renderer is reused via WebView.
|
||||
|
||||
### 6.2 Negative
|
||||
|
||||
- **WebView-based 3D rendering has lower performance than native OpenGL**: The Gaussian splat viewer runs inside a WebView (native) or iframe (web), adding a JavaScript-to-native bridge hop and limiting frame rate to approximately 30 FPS on mid-range devices. Native OpenGL or Metal/Vulkan rendering would achieve 60 FPS but requires platform-specific code.
|
||||
- **react-native-wifi-reborn requires native module linking for Android RSSI**: This breaks the pure Expo managed workflow for Android builds. EAS Build with a custom development client is required. iOS RSSI scanning is not possible at all due to Apple restrictions.
|
||||
- **Expo managed workflow limits some native module access**: Certain native APIs (background location, Bluetooth LE, raw WiFi frames) are not available without ejecting to a bare workflow. This constrains future features like Bluetooth mesh fallback.
|
||||
- **WebView bridge latency**: Communication between React Native and the Three.js WebView via `postMessage` adds 5-15ms per message, reducing effective update rate for the 3D splat view. This is acceptable for 10-20 Hz sensing frame rates but would become a bottleneck at higher rates.
|
||||
- **AsyncStorage has no encryption**: Settings (including server URL) are stored in plaintext AsyncStorage. For security-sensitive deployments, expo-secure-store should replace AsyncStorage for credential storage.
|
||||
|
||||
### 6.3 Risks
|
||||
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|-------------|--------|------------|
|
||||
| Expo SDK 55 breaking changes in future updates | Medium | Build failures, API deprecations | Pin SDK version in `app.config.ts`; test upgrades in preview branch |
|
||||
| WebView memory pressure on low-end Android devices | Medium | OOM crash during Three.js splat rendering | Implement splat LOD (level of detail) fallback; monitor WebView memory via `onContentProcessDidTerminate` |
|
||||
| react-native-wifi-reborn unmaintained or incompatible with RN 0.83 | Low | Android RSSI scanning broken | Fork and patch if needed; RSSI scanning is a secondary feature |
|
||||
| Sensing server WebSocket protocol changes | Medium | Frame parsing errors, broken display | Version the WebSocket protocol; add `protocol_version` field to SensingFrame |
|
||||
| Battery drain from continuous WebSocket connection on mobile | Medium | Poor user experience in extended field use | Implement configurable update rate throttling in settings; pause WS when app is backgrounded |
|
||||
| Three.js Gaussian splat HTML bundle size exceeds WebView limits | Low | Slow initial load, white screen | Lazy-load splat bundle; show placeholder skeleton during load; cache bundle in AsyncStorage |
|
||||
|
||||
---
|
||||
|
||||
## 7. Future Work
|
||||
|
||||
### 7.1 Offline Model Inference
|
||||
|
||||
Run a quantized ONNX pose estimation model directly on the mobile device using `onnxruntime-react-native`. This would allow the app to process raw CSI data (received via a local UDP relay or Bluetooth) without a sensing server, enabling fully disconnected field operation.
|
||||
|
||||
**Prerequisites:** Export the trained WiFi-DensePose model (ADR-023) to ONNX format; quantize to INT8 for mobile; benchmark inference latency on iPhone 15 and Pixel 8.
|
||||
|
||||
### 7.2 Push Notifications for MAT Alerts
|
||||
|
||||
Integrate Firebase Cloud Messaging (Android) and APNs (iOS) to deliver push notifications when the sensing server detects new survivors or critical vital sign alerts. This allows operators to be alerted even when the app is backgrounded.
|
||||
|
||||
**Prerequisites:** Add a push notification endpoint to the Rust sensing server; implement Expo Notifications integration in the mobile app.
|
||||
|
||||
### 7.3 Apple Watch Companion
|
||||
|
||||
Build a watchOS companion app using Expo's experimental watch support or a native SwiftUI module. The watch would display a minimal vitals view (breathing rate, heart rate, alert count) on the operator's wrist, with haptic feedback for critical MAT alerts.
|
||||
|
||||
**Prerequisites:** Evaluate Expo watch support maturity; define minimal watch screen set; implement WatchConnectivity bridge.
|
||||
|
||||
### 7.4 Bluetooth Mesh Fallback
|
||||
|
||||
When WiFi is unavailable (collapsed building, power outage), use Bluetooth Low Energy (BLE) mesh to relay aggregated CSI summaries from ESP32 nodes to the mobile device. This requires ejecting from Expo managed workflow to bare workflow for BLE native module access.
|
||||
|
||||
**Prerequisites:** Implement BLE GATT service on ESP32 firmware (ADR-018); integrate `react-native-ble-plx` in bare Expo workflow; define BLE CSI summary protocol (compressed, lower bandwidth than WiFi).
|
||||
|
||||
### 7.5 Multi-Server Dashboard
|
||||
|
||||
Support connecting to multiple sensing servers simultaneously (e.g., one per floor or building wing). The app would aggregate data from all servers into a unified zone map and MAT dashboard with per-server status indicators.
|
||||
|
||||
**Prerequisites:** Extend `settingsStore` to support server list; modify `ws.service` to manage multiple WebSocket connections; merge `poseStore` frames from multiple sources with server-id tags.
|
||||
|
||||
---
|
||||
|
||||
## 8. Related ADRs
|
||||
|
||||
| ADR | Relationship |
|
||||
|-----|-------------|
|
||||
| ADR-019 (Sensing-Only UI Mode) | **Extended**: The mobile app is the field-optimized evolution of the sensing-only UI mode, adding native mobile capabilities (push, RSSI, offline) |
|
||||
| ADR-021 (Vital Sign Detection) | **Consumed**: VitalsScreen displays breathing_rate_bpm and heart_rate_bpm extracted by the ADR-021 pipeline |
|
||||
| ADR-026 (Survivor Track Lifecycle) | **Consumed**: MATScreen displays survivor tracks with lifecycle states (detected, confirmed, rescued, lost) from ADR-026 |
|
||||
| ADR-029 (RuvSense Multistatic) | **Consumed**: The sensing server aggregates ESP32 TDM frames (ADR-029) and streams processed results to the mobile app |
|
||||
| ADR-031 (RuView Sensing-First RF) | **Consumed**: The WebSocket and REST APIs exposed by `wifi-densepose-sensing-server` (ADR-031) are the mobile app's data source |
|
||||
| ADR-032 (Mesh Security) | **Consumed**: Authenticated CSI frames (ADR-032) ensure the mobile app displays trustworthy data, not spoofed sensor readings |
|
||||
|
||||
---
|
||||
|
||||
## 9. References
|
||||
|
||||
1. Expo SDK 55 Documentation. https://docs.expo.dev/
|
||||
2. React Native 0.83 Release Notes. https://reactnative.dev/
|
||||
3. Zustand v5. https://github.com/pmndrs/zustand
|
||||
4. React Navigation v7. https://reactnavigation.org/
|
||||
5. Maestro Mobile Testing Framework. https://maestro.mobile.dev/
|
||||
6. react-native-wifi-reborn. https://github.com/JuanSeBestworker/react-native-wifi-reborn
|
||||
7. Three.js Gaussian Splatting. https://github.com/mrdoob/three.js
|
||||
8. AsyncStorage. https://react-native-async-storage.github.io/async-storage/
|
||||
9. Geng, J. et al. (2023). "DensePose From WiFi." arXiv:2301.00250.
|
||||
10. ADR-019 through ADR-032 (internal).
|
||||
@@ -0,0 +1,98 @@
|
||||
# ADR-035: Live Sensing UI Accuracy & Data Source Transparency
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
## Date
|
||||
2026-03-02
|
||||
|
||||
## Context
|
||||
|
||||
Issue #86 reported that the live demo shows a static/barely-animated stick figure and the sensing page displays inaccurate data, despite a working ESP32 sending real CSI frames. Investigation revealed three root causes:
|
||||
|
||||
1. **Docker defaults to `--source simulated`** — even with a real ESP32 connected, the server generates synthetic sine-wave data instead of reading UDP frames.
|
||||
2. **Live demo pose is analytically computed** — `derive_pose_from_sensing()` generates keypoints using `sin(tick)` math unrelated to actual signal content. No trained `.rvf` model is loaded by default.
|
||||
3. **Sensing feature extraction is oversimplified** — the server uses single-frame thresholds for motion detection and has no temporal analysis (breathing FFT, sliding window variance, frame history).
|
||||
4. **No data source indicator** — users cannot tell whether they are seeing real or simulated data.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Docker: Auto-detect data source
|
||||
- Default `CSI_SOURCE` changed from `simulated` to `auto`.
|
||||
- `auto` probes UDP port 5005 for an ESP32; falls back to simulation if none found.
|
||||
- Users override via `CSI_SOURCE=esp32 docker-compose up`.
|
||||
|
||||
### 2. Signal-responsive pose derivation
|
||||
- `derive_pose_from_sensing()` now reads actual sensing features:
|
||||
- `motion_band_power` drives limb splay and walking gait detection (> 0.55).
|
||||
- `breathing_band_power` drives torso expansion/contraction phased to breathing rate.
|
||||
- `variance` seeds per-joint noise so the skeleton moves independently.
|
||||
- `dominant_freq_hz` drives lateral torso lean.
|
||||
- `change_points` add burst jitter to extremity keypoints.
|
||||
- Tick rate reduced from 500ms to 100ms (2 fps → 10 fps).
|
||||
- `pose_source` field (`signal_derived` | `model_inference`) added to every WebSocket frame.
|
||||
|
||||
### 3. Temporal feature extraction
|
||||
- 100-frame circular buffer (`VecDeque`) added to `AppStateInner`.
|
||||
- Per-subcarrier temporal variance via Welford-style accumulation.
|
||||
- Breathing rate estimation via 9-candidate Goertzel filter bank (0.1–0.5 Hz) with 3x SNR gate.
|
||||
- Frame-to-frame L2 motion score replaces single-frame amplitude thresholds.
|
||||
- Signal quality metric: SNR-based (RSSI − noise floor) blended with temporal stability.
|
||||
- Signal field driven by subcarrier variance spatial mapping instead of fixed animation.
|
||||
|
||||
### 4. Data source transparency in UI
|
||||
- **Sensing tab**: Banner showing "LIVE - ESP32" (green), "RECONNECTING..." (yellow), or "SIMULATED DATA" (red).
|
||||
- **Live Demo tab**: "Estimation Mode" badge showing "Signal-Derived" (green) or "Model Inference" (blue).
|
||||
- **Setup Guide** panel explaining what each ESP32 count provides (1x: presence/breathing, 3x: localization, 4x+: full pose with trained model).
|
||||
- Simulation fallback delayed from immediate to 5 failed reconnect attempts (~30s).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Users with real ESP32 hardware get real data by default (auto-detect).
|
||||
- Simulated data is clearly labeled — no more confusion about data authenticity.
|
||||
- Pose skeleton visually responds to actual signal changes (motion, breathing, variance).
|
||||
- Feature extraction produces physiologically meaningful metrics (breathing rate via Goertzel, temporal motion detection).
|
||||
- Setup guide manages expectations about what each hardware configuration provides.
|
||||
|
||||
### Negative
|
||||
- Signal-derived pose is still an approximation, not neural network inference. Per-limb tracking requires a trained `.rvf` model + 4+ ESP32 nodes.
|
||||
- Goertzel filter bank adds ~O(9×N) computation per frame (negligible at 100 frames).
|
||||
- Users with only 1 ESP32 may still be disappointed that arm tracking doesn't work — but the UI now explains why.
|
||||
|
||||
### 5. Dark mode consistency
|
||||
- Live Demo tab converted from light theme to dark mode matching the rest of the UI.
|
||||
- All sidebar panels, badges, buttons, dropdowns use dark backgrounds with muted text.
|
||||
|
||||
### 6. Render mode implementations
|
||||
All four render modes in the pose visualization dropdown now produce distinct visual output:
|
||||
|
||||
| Mode | Rendering |
|
||||
|------|-----------|
|
||||
| **Skeleton** | Green lines connecting joints + red keypoint dots |
|
||||
| **Keypoints** | Large colored dots with glow and labels, no connecting lines |
|
||||
| **Heatmap** | Gaussian radial blobs per keypoint (hue per person), faint skeleton overlay at 25% opacity |
|
||||
| **Dense** | Body region segmentation with colored filled polygons — head (red), torso (blue), left arm (green), right arm (orange), left leg (purple), right leg (yellow) |
|
||||
|
||||
Previously heatmap and dense were stubs that fell back to skeleton mode.
|
||||
|
||||
### 7. pose_source passthrough fix
|
||||
The `pose_source` field from the WebSocket message was being dropped in `convertZoneDataToRestFormat()` in `pose.service.js`. Now passed through so the Estimation Mode badge displays correctly.
|
||||
|
||||
## Files Changed
|
||||
- `docker/Dockerfile.rust` — `CSI_SOURCE=auto` env, shell entrypoint for variable expansion
|
||||
- `docker/docker-compose.yml` — `CSI_SOURCE=${CSI_SOURCE:-auto}`, shell command string
|
||||
- `wifi-densepose-sensing-server/src/main.rs` — frame history buffer, Goertzel breathing estimation, temporal motion score, signal-driven pose derivation, pose_source field, 100ms tick default
|
||||
- `ui/services/sensing.service.js` — `dataSource` state, delayed simulation fallback, `_simulated` marker
|
||||
- `ui/services/pose.service.js` — `pose_source` passthrough in data conversion
|
||||
- `ui/components/SensingTab.js` — data source banner, "About This Data" card
|
||||
- `ui/components/LiveDemoTab.js` — estimation mode badge, setup guide panel, dark mode theme
|
||||
- `ui/utils/pose-renderer.js` — heatmap (Gaussian blobs) and dense (body region segmentation) render modes
|
||||
- `ui/style.css` — banner, badge, guide panel, and about-text styles
|
||||
- `README.md` — live pose detection screenshot
|
||||
- `assets/screen.png` — screenshot asset
|
||||
|
||||
## References
|
||||
- Issue: https://github.com/ruvnet/wifi-densepose/issues/86
|
||||
- ADR-029: RuvSense multistatic sensing mode (proposed — full pipeline integration)
|
||||
- ADR-014: SOTA signal processing
|
||||
@@ -0,0 +1,228 @@
|
||||
# ADR-036: RVF Model Training Pipeline & UI Integration
|
||||
|
||||
## Status
|
||||
Proposed
|
||||
|
||||
## Date
|
||||
2026-03-02
|
||||
|
||||
## Context
|
||||
|
||||
The wifi-densepose system currently operates in **signal-derived** mode — `derive_pose_from_sensing()` maps aggregate CSI features (motion power, breathing rate, variance) to keypoint positions using deterministic math. This gives whole-body presence and gross motion but cannot track individual limbs.
|
||||
|
||||
The infrastructure for **model inference** mode exists but is disconnected:
|
||||
|
||||
1. **RVF container format** (`rvf_container.rs`, 1,102 lines) — a 64-byte-aligned binary format supporting model weights (`SEG_VEC`), metadata (`SEG_MANIFEST`), quantization (`SEG_QUANT`), LoRA profiles (`SEG_LORA`), contrastive embeddings (`SEG_EMBED`), and witness audit trails (`SEG_WITNESS`). Builder and reader are fully implemented with CRC32 integrity checks.
|
||||
|
||||
2. **Training crate** (`wifi-densepose-train`) — AdamW optimizer, PCK@0.2/OKS metrics, LR scheduling with warmup, early stopping, CSV logging, and checkpoint export. Supports `CsiDataset` trait with planned MM-Fi (114→56 subcarrier interpolation) and Wi-Pose (30→56 zero-pad) loaders per ADR-015.
|
||||
|
||||
3. **NN inference crate** (`wifi-densepose-nn`) — ONNX Runtime backend with CPU/GPU support, dynamic tensor shapes, thread-safe `OnnxBackend` wrapper, model info inspection, and warmup.
|
||||
|
||||
4. **Sensing server CLI** (`--model <path>`, `--train`, `--pretrain`, `--embed`) — flags exist for model loading, training mode, and embedding extraction, but the end-to-end path from raw CSI → trained `.rvf` → live inference is not wired together.
|
||||
|
||||
5. **UI gaps** — No model management, training progress visualization, LoRA profile switching, or embedding inspection. The Settings panel lacks model configuration. The Live Demo has no way to load a trained model or compare signal-derived vs model-inference output side-by-side.
|
||||
|
||||
### What users need
|
||||
|
||||
- A way to **collect labeled CSI data** from their own environment (self-supervised or teacher-student from camera).
|
||||
- A way to **train an .rvf model** from collected data without leaving the UI.
|
||||
- A way to **load and switch models** in the live demo, seeing the quality improvement.
|
||||
- Visibility into **training progress** (loss curves, validation PCK, early stopping).
|
||||
- **Environment adaptation** via LoRA profiles (office → home → warehouse) without full retraining.
|
||||
|
||||
## Decision
|
||||
|
||||
### Phase 1: Data Collection & Self-Supervised Pretraining
|
||||
|
||||
#### 1.1 CSI Recording API
|
||||
Add REST endpoints to the sensing server:
|
||||
```
|
||||
POST /api/v1/recording/start { duration_secs, label?, session_name }
|
||||
POST /api/v1/recording/stop
|
||||
GET /api/v1/recording/list
|
||||
GET /api/v1/recording/download/:id
|
||||
DELETE /api/v1/recording/:id
|
||||
```
|
||||
- Records raw CSI frames + extracted features to `.csi.jsonl` files.
|
||||
- Optional camera-based label overlay via teacher model (Detectron2/MediaPipe on client).
|
||||
- Each recording session tagged with environment metadata (room dimensions, node positions, AP count).
|
||||
|
||||
#### 1.2 Contrastive Pretraining (ADR-024 Phase 1)
|
||||
- Self-supervised NT-Xent loss learns a 128-dim CSI embedding without pose labels.
|
||||
- Positive pairs: adjacent frames from same person; negatives: different sessions/rooms.
|
||||
- VICReg regularization prevents embedding collapse.
|
||||
- Output: `.rvf` container with `SEG_EMBED` + `SEG_VEC` segments.
|
||||
- Training triggered via `POST /api/v1/train/pretrain { dataset_ids[], epochs, lr }`.
|
||||
|
||||
### Phase 2: Supervised Training Pipeline
|
||||
|
||||
#### 2.1 Dataset Integration
|
||||
- **MM-Fi loader**: Parse HDF5 files, 114→56 subcarrier interpolation via `ruvector-solver` sparse least-squares.
|
||||
- **Wi-Pose loader**: Parse .mat files, 30→56 zero-padding with Hann window smoothing.
|
||||
- **Self-collected**: `.csi.jsonl` from Phase 1 recording + camera-generated labels.
|
||||
- All datasets implement `CsiDataset` trait and produce `(amplitude[B,T*links,56], phase[B,T*links,56], keypoints[B,17,2], visibility[B,17])`.
|
||||
|
||||
#### 2.2 Training API
|
||||
```
|
||||
POST /api/v1/train/start {
|
||||
dataset_ids: string[],
|
||||
config: {
|
||||
epochs: 100,
|
||||
batch_size: 32,
|
||||
learning_rate: 3e-4,
|
||||
weight_decay: 1e-4,
|
||||
early_stopping_patience: 15,
|
||||
warmup_epochs: 5,
|
||||
pretrained_rvf?: string, // Base model for fine-tuning
|
||||
lora_profile?: string, // Environment-specific LoRA
|
||||
}
|
||||
}
|
||||
POST /api/v1/train/stop
|
||||
GET /api/v1/train/status // { epoch, train_loss, val_pck, val_oks, lr, eta_secs }
|
||||
WS /ws/train/progress // Real-time streaming of training metrics
|
||||
```
|
||||
|
||||
#### 2.3 RVF Export
|
||||
On training completion:
|
||||
- Best checkpoint exported as `.rvf` with `SEG_VEC` (weights), `SEG_MANIFEST` (metadata), `SEG_WITNESS` (training hash + final metrics), and optional `SEG_QUANT` (INT8 quantization).
|
||||
- Stored in `data/models/` directory, indexed by model ID.
|
||||
- `GET /api/v1/models` lists available models; `POST /api/v1/models/load { model_id }` hot-loads into inference.
|
||||
|
||||
### Phase 3: LoRA Environment Adaptation
|
||||
|
||||
#### 3.1 LoRA Fine-Tuning
|
||||
- Given a base `.rvf` model, fine-tune only LoRA adapter weights (rank 4-16) on environment-specific recordings.
|
||||
- 5-10 minutes of labeled data from new environment suffices.
|
||||
- New LoRA profile appended to existing `.rvf` via `SEG_LORA` segment.
|
||||
- `POST /api/v1/train/lora { base_model_id, dataset_ids[], profile_name, rank: 8, epochs: 20 }`.
|
||||
|
||||
#### 3.2 Profile Switching
|
||||
- `POST /api/v1/models/lora/activate { model_id, profile_name }` — hot-swap LoRA weights without reloading base model.
|
||||
- UI dropdown lists available profiles per loaded model.
|
||||
|
||||
### Phase 4: UI Integration
|
||||
|
||||
#### 4.1 Model Management Panel (new: `ui/components/ModelPanel.js`)
|
||||
- **Model Library**: List loaded and available `.rvf` models with metadata (version, dataset, PCK score, size, created date).
|
||||
- **Model Inspector**: Show RVF segment breakdown — weight count, quantization type, LoRA profiles, embedding config, witness hash.
|
||||
- **Load/Unload**: One-click model loading with progress bar.
|
||||
- **Compare**: Side-by-side signal-derived vs model-inference toggle in Live Demo.
|
||||
|
||||
#### 4.2 Training Dashboard (new: `ui/components/TrainingPanel.js`)
|
||||
- **Recording Controls**: Start/stop CSI recording, session list with duration and frame counts.
|
||||
- **Training Progress**: Real-time loss curve (train loss, val loss) and metric charts (PCK@0.2, OKS) via WebSocket streaming.
|
||||
- **Epoch Table**: Scrollable table of per-epoch metrics with best-epoch highlighting.
|
||||
- **Early Stopping Indicator**: Visual countdown of patience remaining.
|
||||
- **Export Button**: Download trained `.rvf` from browser.
|
||||
|
||||
#### 4.3 Live Demo Enhancements
|
||||
- **Model Selector**: Dropdown in toolbar to switch between signal-derived and loaded `.rvf` models.
|
||||
- **LoRA Profile Selector**: Sub-dropdown showing environment profiles for the active model.
|
||||
- **Confidence Heatmap Overlay**: Per-keypoint confidence visualization when model is loaded (toggle in render mode dropdown).
|
||||
- **Pose Trail**: Ghosted keypoint history showing last N frames of motion trajectory.
|
||||
- **A/B Split View**: Left half signal-derived, right half model-inference for quality comparison.
|
||||
|
||||
#### 4.4 Settings Panel Extensions
|
||||
- **Model section**: Default model path, auto-load on startup, GPU/CPU toggle, inference threads.
|
||||
- **Training section**: Default hyperparameters, checkpoint directory, auto-export on completion.
|
||||
- **Recording section**: Default recording directory, max duration, auto-label with camera.
|
||||
|
||||
#### 4.5 Dark Mode
|
||||
All new panels follow the dark mode established in ADR-035 (`#0d1117` backgrounds, `#e0e0e0` text, translucent dark panels with colored accents).
|
||||
|
||||
### Phase 5: Inference Pipeline Wiring
|
||||
|
||||
#### 5.1 Model-Inference Pose Path
|
||||
When a `.rvf` model is loaded:
|
||||
1. CSI frame arrives (UDP or simulated).
|
||||
2. Extract amplitude + phase tensors from subcarrier data.
|
||||
3. Feed through ONNX session: `input[1, T*links, 56]` → `output[1, 17, 4]` (x, y, z, conf).
|
||||
4. Apply Kalman smoothing from `pose_tracker.rs`.
|
||||
5. Broadcast via WebSocket with `pose_source: "model_inference"`.
|
||||
6. UI Estimation Mode badge switches from green "SIGNAL-DERIVED" to blue "MODEL INFERENCE".
|
||||
|
||||
#### 5.2 Progressive Loading (ADR-031 Layer A/B/C)
|
||||
- **Layer A** (instant): Signal-derived pose starts immediately.
|
||||
- **Layer B** (5-10s): Contrastive embeddings loaded, HNSW index warm.
|
||||
- **Layer C** (30-60s): Full pose model loaded, inference active.
|
||||
- Transitions seamlessly; UI badge updates automatically.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Users can train a model on **their own environment** without external tools or Python dependencies.
|
||||
- LoRA profiles mean a single base model adapts to multiple rooms in minutes, not hours.
|
||||
- Training progress is visible in real-time — no black-box waiting.
|
||||
- A/B comparison lets users see the quality jump from signal-derived to model-inference.
|
||||
- RVF container bundles everything (weights, metadata, LoRA, witness) in one portable file.
|
||||
- Self-supervised pretraining requires no labels — just leave ESP32s running.
|
||||
- Progressive loading means the UI is never "loading..." — signal-derived kicks in immediately.
|
||||
|
||||
### Negative
|
||||
- Training requires significant compute: GPU recommended for supervised training (CPU possible but 10-50x slower).
|
||||
- MM-Fi and Wi-Pose datasets must be downloaded separately (10-50 GB each) — cannot be bundled.
|
||||
- LoRA rank must be tuned per environment; too low loses expressiveness, too high overfits.
|
||||
- ONNX Runtime adds ~50 MB to the binary size when GPU support is enabled.
|
||||
- Real-time inference at 10 FPS requires ~10ms per frame — tight budget on CPU.
|
||||
- Teacher-student labeling (camera → pose labels → CSI training) requires camera access, which may conflict with the privacy-first premise.
|
||||
|
||||
### Mitigations
|
||||
- Provide pre-trained base `.rvf` model downloadable from releases (trained on MM-Fi + Wi-Pose).
|
||||
- INT8 quantization (`SEG_QUANT`) reduces model size 4x and speeds inference ~2x on CPU.
|
||||
- Camera-based labeling is **optional** — self-supervised pretraining works without camera.
|
||||
- Training API validates VRAM availability before starting GPU training; falls back to CPU with warning.
|
||||
|
||||
## Implementation Order
|
||||
|
||||
| Phase | Effort | Dependencies | Priority |
|
||||
|-------|--------|-------------|----------|
|
||||
| 1.1 CSI Recording API | 2-3 days | sensing server | High |
|
||||
| 1.2 Contrastive Pretraining | 3-5 days | ADR-024, recording API | High |
|
||||
| 2.1 Dataset Integration | 3-5 days | ADR-015, CsiDataset trait | High |
|
||||
| 2.2 Training API | 2-3 days | training crate, dataset loaders | High |
|
||||
| 2.3 RVF Export | 1-2 days | RvfBuilder | Medium |
|
||||
| 3.1 LoRA Fine-Tuning | 3-5 days | base trained model | Medium |
|
||||
| 3.2 Profile Switching | 1 day | LoRA in RVF | Medium |
|
||||
| 4.1 Model Panel UI | 2-3 days | models API | High |
|
||||
| 4.2 Training Dashboard UI | 3-4 days | training API + WS | High |
|
||||
| 4.3 Live Demo Enhancements | 2-3 days | model loading | Medium |
|
||||
| 4.4 Settings Extensions | 1 day | model/training APIs | Low |
|
||||
| 4.5 Dark Mode | 0.5 days | new panels | Low |
|
||||
| 5.1 Inference Wiring | 3-5 days | ONNX backend, pose tracker | High |
|
||||
| 5.2 Progressive Loading | 2-3 days | ADR-031 | Medium |
|
||||
|
||||
**Total estimate: 4-6 weeks** (phases can overlap; 1+2 parallel with 4).
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Files
|
||||
- `ui/components/ModelPanel.js` — Model library, inspector, load/unload controls
|
||||
- `ui/components/TrainingPanel.js` — Recording controls, training progress, metric charts
|
||||
- `rust-port/.../sensing-server/src/recording.rs` — CSI recording API handlers
|
||||
- `rust-port/.../sensing-server/src/training_api.rs` — Training API handlers + WS progress stream
|
||||
- `rust-port/.../sensing-server/src/model_manager.rs` — Model loading, hot-swap, 32LoRA activation
|
||||
- `data/models/` — Default model storage directory
|
||||
|
||||
### Modified Files
|
||||
- `rust-port/.../sensing-server/src/main.rs` — Wire recording, training, and model APIs
|
||||
- `rust-port/.../train/src/trainer.rs` — Add WebSocket progress callback, LoRA training mode
|
||||
- `rust-port/.../train/src/dataset.rs` — MM-Fi and Wi-Pose dataset loaders
|
||||
- `rust-port/.../nn/src/onnx.rs` — LoRA weight injection, INT8 quantization support
|
||||
- `ui/components/LiveDemoTab.js` — Model selector, LoRA dropdown, A/B spsplit view
|
||||
- `ui/components/SettingsPanel.js` — Model and training configuration sections
|
||||
- `ui/components/PoseDetectionCanvas.js` — Pose trail rendering, confidence heatmap overlay
|
||||
- `ui/services/pose.service.js` — Model-inference keypoint processing
|
||||
- `ui/index.html` — Add Training tabhee
|
||||
- `ui/style.css` — Styles for new panels
|
||||
|
||||
## References
|
||||
- ADR-015: MM-Fi + Wi-Pose training datasets
|
||||
- ADR-016: RuVector training pipeline integration
|
||||
- ADR-024: Project AETHER — contrastive CSI embedding model
|
||||
- ADR-029: RuvSense multistatic sensing mode
|
||||
- ADR-031: RuView sensing-first RF mode (progressive loading)
|
||||
- ADR-035: Live sensing UI accuracy & data source transparency
|
||||
- Issue: https://github.com/ruvnet/wifi-densepose/issues/92
|
||||
- RVF format: `crates/wifi-densepose-sensing-server/src/rvf_container.rs`
|
||||
- Training crate: `crates/wifi-densepose-train/src/trainer.rs`
|
||||
- NN inference: `crates/wifi-densepose-nn/src/onnx.rs`
|
||||
@@ -0,0 +1,121 @@
|
||||
# ADR-037: Multi-Person Pose Detection from Single ESP32 CSI Stream
|
||||
|
||||
- **Status**: Proposed
|
||||
- **Date**: 2026-03-02
|
||||
- **Issue**: [#97](https://github.com/ruvnet/wifi-densepose/issues/97)
|
||||
- **Deciders**: @ruvnet
|
||||
- **Supersedes**: None
|
||||
- **Related**: ADR-014 (SOTA signal processing), ADR-024 (AETHER re-ID), ADR-029 (multistatic sensing), ADR-036 (RVF training pipeline)
|
||||
|
||||
## Context
|
||||
|
||||
The current signal-derived pose estimation pipeline (`derive_pose_from_sensing()` in the sensing server) generates at most one skeleton per frame from aggregate CSI features. When multiple people are present, only a single blended skeleton is produced. Live testing with ESP32 hardware confirmed: 2 people in the room yields 1 detected person.
|
||||
|
||||
A single ESP32 node provides 1 TX × 1 RX × 56 subcarriers of CSI data per frame. While this is limited spatial resolution compared to camera-based systems, the signal contains composite reflections from all scatterers in the environment. The challenge is decomposing these composite signals into per-person contributions.
|
||||
|
||||
## Decision
|
||||
|
||||
Implement multi-person pose detection in four phases, progressively improving accuracy from heuristic to neural approaches.
|
||||
|
||||
### Phase 1: Person Count Estimation
|
||||
|
||||
Estimate occupancy count from CSI signal statistics without decomposition.
|
||||
|
||||
**Approach**: Eigenvalue analysis of the CSI covariance matrix across subcarriers.
|
||||
|
||||
- Compute the 56×56 covariance matrix of CSI amplitudes over a sliding window (e.g., 50 frames / 5 seconds)
|
||||
- Count eigenvalues above a noise threshold — each significant eigenvalue corresponds to an independent scatterer (person or static object)
|
||||
- Subtract the static environment baseline (estimated during calibration or from the field model's SVD eigenstructure)
|
||||
- The residual significant eigenvalue count estimates person count
|
||||
|
||||
**Accuracy target**: > 80% for 0-3 people with single ESP32 node.
|
||||
|
||||
**Integration point**: `signal/src/ruvsense/field_model.rs` already computes SVD eigenstructure. Extend with a `estimate_occupancy()` method.
|
||||
|
||||
### Phase 2: Signal Decomposition
|
||||
|
||||
Separate per-person signal contributions using blind source separation.
|
||||
|
||||
**Approach**: Non-negative Matrix Factorization (NMF) on the CSI spectrogram.
|
||||
|
||||
- Construct a time-frequency matrix from CSI amplitudes: rows = subcarriers (56), columns = time frames
|
||||
- Apply NMF with k components (k = estimated person count from Phase 1)
|
||||
- Each component's frequency profile maps to a person's motion pattern
|
||||
- NMF is preferred over ICA because CSI amplitudes are non-negative
|
||||
|
||||
**Alternative**: Independent Component Analysis (ICA) on complex CSI (amplitude + phase). More powerful but requires phase calibration (see `ruvsense/phase_align.rs`).
|
||||
|
||||
**Integration point**: New module `signal/src/ruvsense/separation.rs`.
|
||||
|
||||
### Phase 3: Multi-Skeleton Generation
|
||||
|
||||
Generate distinct pose skeletons per decomposed component.
|
||||
|
||||
**Approach**: Per-component feature extraction → per-person skeleton synthesis.
|
||||
|
||||
- Extract motion features (dominant frequency, energy, spectral centroid) per NMF component
|
||||
- Map each component to a spatial position using subcarrier phase gradient (Fresnel zone model)
|
||||
- Generate 17-keypoint COCO skeleton per person with position offset
|
||||
- Assign person IDs using the existing Kalman tracker (`ruvsense/pose_tracker.rs`) with AETHER re-ID embeddings (ADR-024)
|
||||
|
||||
**Integration point**: Modify `derive_pose_from_sensing()` in `sensing-server/src/main.rs` to return `Vec<Person>` with length > 1.
|
||||
|
||||
### Phase 4: Neural Multi-Person Model
|
||||
|
||||
Train a dedicated multi-person model using the RVF pipeline (ADR-036).
|
||||
|
||||
- Use MM-Fi dataset (ADR-015) multi-person scenarios for training data
|
||||
- Architecture: shared CSI encoder → person count head + per-person pose heads
|
||||
- LoRA fine-tuning profile for multi-person specialization
|
||||
- Inference via the model manager in the sensing server
|
||||
|
||||
**Accuracy target**: PCK@0.2 > 60% for 2-person scenarios.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Enables room occupancy counting (Phase 1 alone is useful)
|
||||
- Distinct pose tracking per person enables activity recognition per individual
|
||||
- Progressive approach — each phase delivers incremental value
|
||||
- Reuses existing infrastructure (field model SVD, Kalman tracker, AETHER, RVF pipeline)
|
||||
|
||||
### Negative
|
||||
|
||||
- Single ESP32 node has fundamental spatial resolution limits — separating 2 people standing close together (< 0.5m) will be unreliable
|
||||
- NMF decomposition adds ~5-10ms latency per frame
|
||||
- Person count estimation will have false positives from large moving objects (pets, fans)
|
||||
- Phase 4 neural model requires multi-person training data collection
|
||||
|
||||
### Neutral
|
||||
|
||||
- Multi-node multistatic mesh (ADR-029) dramatically improves multi-person separation but is a separate effort
|
||||
- UI already supports multi-person rendering — no frontend changes needed for the `persons[]` array
|
||||
|
||||
## Affected Components
|
||||
|
||||
| Component | Phase | Change |
|
||||
|-----------|-------|--------|
|
||||
| `signal/src/ruvsense/field_model.rs` | 1 | Add `estimate_occupancy()` |
|
||||
| `signal/src/ruvsense/separation.rs` | 2 | New module: NMF decomposition |
|
||||
| `sensing-server/src/main.rs` | 3 | `derive_pose_from_sensing()` multi-person output |
|
||||
| `signal/src/ruvsense/pose_tracker.rs` | 3 | Multi-target tracking |
|
||||
| `nn/` | 4 | Multi-person inference head |
|
||||
| `train/` | 4 | Multi-person training pipeline |
|
||||
|
||||
## Performance Budget
|
||||
|
||||
| Operation | Budget | Phase |
|
||||
|-----------|--------|-------|
|
||||
| Person count estimation | < 2ms | 1 |
|
||||
| NMF decomposition (k=3) | < 10ms | 2 |
|
||||
| Multi-skeleton synthesis | < 3ms | 3 |
|
||||
| Neural inference (multi-person) | < 50ms | 4 |
|
||||
| **Total pipeline** | **< 65ms** (15 FPS) | All |
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **Camera fusion**: Use a camera for person detection and WiFi for pose — rejected because the project goal is camera-free sensing.
|
||||
2. **Multiple single-person models**: Run N independent pose estimators — rejected because they would produce correlated outputs from the same CSI data.
|
||||
3. **Spatial filtering (beamforming)**: Use antenna array beamforming to isolate directions — rejected because single ESP32 has only 1 antenna; viable with multistatic mesh (ADR-029).
|
||||
4. **Skip signal-derived, go straight to neural**: Train an end-to-end multi-person model — rejected because signal-derived provides faster iteration and interpretability for the early phases.
|
||||
@@ -0,0 +1,546 @@
|
||||
# ADR-038: Sublinear Goal-Oriented Action Planning (GOAP) for Project Roadmap Optimization
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-03-02 |
|
||||
| **Deciders** | ruv |
|
||||
| **Relates to** | All 37 prior ADRs; ADR-014 (SOTA Signal Processing), ADR-016 (RuVector Integration), ADR-024 (AETHER Embeddings), ADR-027 (MERIDIAN Generalization), ADR-029 (RuvSense Multistatic), ADR-037 (Multi-Person Detection) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
### 1.1 The Planning Problem
|
||||
|
||||
WiFi-DensePose has 37 Architecture Decision Records. Of these, 14 are Accepted/Complete, 4 are Partially Implemented, 19 are Proposed, and 1 is Superseded. The proposed ADRs span diverse capabilities: vital sign detection (ADR-021), multi-BSSID scanning (ADR-022), contrastive embeddings (ADR-024), cross-environment generalization (ADR-027), multistatic mesh sensing (ADR-029), persistent field models (ADR-030), multi-person pose detection (ADR-037), and more.
|
||||
|
||||
A single developer (or a small team aided by AI agents) must decide **what to build next** given:
|
||||
|
||||
- **Dense dependency graph**: ADR-037 (multi-person) depends on ADR-014 (signal processing), ADR-024 (AETHER), and ADR-029 (multistatic). ADR-029 depends on ADR-012 (ESP32 mesh), ADR-014, ADR-016, and ADR-018. Many ADRs share prerequisites.
|
||||
- **Hardware variability**: Some ADRs require ESP32 hardware (ADR-021 vital signs, ADR-029 multistatic mesh), while others are software-only (ADR-024 AETHER, ADR-027 MERIDIAN). The available hardware changes session to session.
|
||||
- **Shifting goals**: One session the user wants accuracy improvement; the next session they want multi-person support; the next they want WebAssembly deployment.
|
||||
- **Resource constraints**: Limited compute budget, single-developer throughput, CI pipeline capacity.
|
||||
|
||||
Manually navigating this decision space is error-prone. The developer must hold the full dependency graph in working memory, re-evaluate priorities when goals shift, and avoid dead-end plans that block on unavailable hardware.
|
||||
|
||||
### 1.2 Why GOAP
|
||||
|
||||
Goal-Oriented Action Planning (GOAP), originally developed for game AI by Jeff Orkin (2003), models the world as a set of boolean/numeric state properties and defines actions with typed preconditions and effects. A planner searches from the current world state to a goal state, producing an optimal action sequence. GOAP is a natural fit for this problem because:
|
||||
|
||||
1. **ADR implementations are actions** with clear preconditions (which other ADRs/hardware must exist) and effects (which capabilities are unlocked).
|
||||
2. **The world state is observable** -- we can query cargo test results, check hardware connections, read crate manifests, and measure accuracy metrics.
|
||||
3. **Goals are declarative** -- "I want multi-person tracking at 20 Hz" translates to `{multi_person_tracking: true, update_rate_hz: 20}`.
|
||||
4. **Replanning is cheap** -- when hardware becomes available or a user changes goals, the planner re-runs in milliseconds.
|
||||
|
||||
### 1.3 Why Sublinear
|
||||
|
||||
The naive GOAP planner uses A* search over the full action-state graph. With 37 ADRs, each potentially having multiple phases (ADR-037 has 4 phases, ADR-029 has 9 actions), the raw action count exceeds 80. The full state space is `2^N` for N boolean properties. Exhaustive search is wasteful because:
|
||||
|
||||
- Most actions are irrelevant to any given goal (the user asking for vital signs does not need WebAssembly deployment actions in the search).
|
||||
- The dependency graph is sparse -- most actions depend on 1-3 prerequisites, not all other actions.
|
||||
- Many state properties are independent (vital sign detection does not interact with WebAssembly compilation).
|
||||
|
||||
A sublinear approach avoids exploring the full state space by exploiting this sparsity.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
Implement a GOAP planning system as a coordinator module within the claude-flow swarm framework. The planner takes a user goal, the current project state, and available hardware as input, and produces an ordered action plan that is dispatched to specialized agents for execution.
|
||||
|
||||
### 2.1 World State Model
|
||||
|
||||
The world state is a flat map of typed properties representing the current project capabilities.
|
||||
|
||||
#### 2.1.1 Feature Implementation Flags (Boolean)
|
||||
|
||||
| Property | Source of Truth | Description |
|
||||
|----------|----------------|-------------|
|
||||
| `sota_signal_processing` | `cargo test -p wifi-densepose-signal` passes | ADR-014 SOTA algorithms implemented |
|
||||
| `ruvector_training_integrated` | `train/` crate builds with ruvector deps | ADR-016 RuVector training pipeline |
|
||||
| `ruvector_signal_integrated` | `signal/src/ruvsense/` module exists | ADR-017 RuVector signal integration |
|
||||
| `esp32_firmware_base` | `firmware/esp32-csi-node/` compiles | ADR-018 ESP32 base firmware |
|
||||
| `esp32_channel_hopping` | Firmware supports multi-channel | ADR-029 Phase 1 |
|
||||
| `multi_band_fusion` | `ruvsense/multiband.rs` passes tests | ADR-029 Phase 2 |
|
||||
| `multistatic_mesh` | Multi-node fusion operational | ADR-029 Phase 3 |
|
||||
| `coherence_gating` | `ruvsense/coherence_gate.rs` passes tests | ADR-029 Phase 6-7 |
|
||||
| `pose_tracker_17kp` | `ruvsense/pose_tracker.rs` passes tests | ADR-029 Phase 4 |
|
||||
| `vital_signs_extraction` | `vitals/` crate passes tests | ADR-021 |
|
||||
| `vital_signs_esp32_validated` | ESP32 breathing detection verified | ADR-021 Phase 2 |
|
||||
| `multi_bssid_scan` | `wifiscan/` crate passes tests | ADR-022 Phase 1 |
|
||||
| `multi_bssid_concurrent` | Concurrent BSSID scanning | ADR-022 Phase 2 |
|
||||
| `aether_embeddings` | Contrastive CSI encoder trained | ADR-024 |
|
||||
| `aether_reid` | Person re-identification via embeddings | ADR-024 Phase 3 |
|
||||
| `meridian_generalization` | Cross-environment transfer working | ADR-027 |
|
||||
| `persistent_field_model` | Field model serializes/deserializes | ADR-030 |
|
||||
| `person_count_estimation` | Eigenvalue occupancy estimator | ADR-037 Phase 1 |
|
||||
| `signal_decomposition` | NMF per-person separation | ADR-037 Phase 2 |
|
||||
| `multi_skeleton_generation` | Multiple skeletons per frame | ADR-037 Phase 3 |
|
||||
| `multi_person_neural` | Neural multi-person model | ADR-037 Phase 4 |
|
||||
| `wasm_deployment` | WebAssembly build functional | ADR-025 |
|
||||
| `mat_survivor_detection` | MAT disaster detection operational | ADR-011/ADR-026 |
|
||||
| `ruview_sensing_ui` | Sensing-first RF UI mode | ADR-031 |
|
||||
| `mesh_security_hardened` | Multistatic mesh security layer | ADR-032 |
|
||||
|
||||
#### 2.1.2 Hardware Availability Flags (Boolean)
|
||||
|
||||
| Property | Detection Method | Description |
|
||||
|----------|-----------------|-------------|
|
||||
| `esp32_connected` | USB serial probe (`/dev/ttyUSB*` or `COM*`) | At least one ESP32 on USB |
|
||||
| `esp32_count` | Count USB serial devices with ESP32 VID/PID | Number of ESP32 nodes |
|
||||
| `esp32_multistatic_ready` | `esp32_count >= 2` | Sufficient for multistatic |
|
||||
| `gpu_available` | `nvidia-smi` or CUDA probe | GPU for neural training |
|
||||
| `wifi_adapter_present` | OS WiFi interface enumeration | Host WiFi for multi-BSSID |
|
||||
|
||||
#### 2.1.3 Quality Metrics (Numeric)
|
||||
|
||||
| Property | Source | Description |
|
||||
|----------|--------|-------------|
|
||||
| `pose_accuracy_pck02` | Benchmark suite output | PCK@0.2 accuracy (0.0-1.0) |
|
||||
| `update_rate_hz` | Pipeline timing measurement | Effective output frame rate |
|
||||
| `max_persons_tracked` | Multi-person test result | Maximum simultaneous persons |
|
||||
| `breathing_snr_db` | Vital signs test output | Breathing detection SNR |
|
||||
| `torso_jitter_mm` | Tracking benchmark | RMS torso keypoint jitter |
|
||||
| `rust_test_count` | `cargo test --workspace` output | Total passing Rust tests |
|
||||
|
||||
### 2.2 Action Definitions
|
||||
|
||||
Each action maps to an ADR implementation phase. Actions are defined as structs with preconditions, effects, cost, and metadata.
|
||||
|
||||
```rust
|
||||
pub struct GoapAction {
|
||||
/// Unique identifier (e.g., "adr029_phase1_channel_hopping")
|
||||
pub id: String,
|
||||
/// Human-readable name
|
||||
pub name: String,
|
||||
/// ADR reference (e.g., "ADR-029")
|
||||
pub adr: String,
|
||||
/// Phase within the ADR (e.g., "Phase 1")
|
||||
pub phase: Option<String>,
|
||||
/// Preconditions: state properties that must be true/meet threshold
|
||||
pub preconditions: Vec<Condition>,
|
||||
/// Effects: state properties set after successful execution
|
||||
pub effects: Vec<Effect>,
|
||||
/// Estimated effort in developer-days
|
||||
pub cost_days: f32,
|
||||
/// Whether this action requires hardware
|
||||
pub requires_hardware: Vec<String>,
|
||||
/// Agent types needed to execute this action
|
||||
pub agent_types: Vec<String>,
|
||||
/// Affected crates/files
|
||||
pub affected_components: Vec<String>,
|
||||
}
|
||||
|
||||
pub enum Condition {
|
||||
BoolTrue(String), // property must be true
|
||||
BoolFalse(String), // property must be false
|
||||
NumericGte(String, f64), // property >= threshold
|
||||
NumericLte(String, f64), // property <= threshold
|
||||
}
|
||||
|
||||
pub enum Effect {
|
||||
SetBool(String, bool), // set boolean property
|
||||
SetNumeric(String, f64), // set numeric property
|
||||
IncrementNumeric(String, f64), // add to numeric property
|
||||
}
|
||||
```
|
||||
|
||||
#### 2.2.1 Action Catalog (Key ADR Actions)
|
||||
|
||||
| Action ID | ADR | Cost (days) | Preconditions | Effects | Hardware |
|
||||
|-----------|-----|-------------|---------------|---------|----------|
|
||||
| `adr037_p1_person_count` | 037 | 3 | `sota_signal_processing` | `person_count_estimation = true` | None |
|
||||
| `adr037_p2_nmf_decomp` | 037 | 5 | `person_count_estimation` | `signal_decomposition = true` | None |
|
||||
| `adr037_p3_multi_skel` | 037 | 4 | `signal_decomposition`, `pose_tracker_17kp` | `multi_skeleton_generation = true`, `max_persons_tracked += 2` | None |
|
||||
| `adr037_p4_neural_multi` | 037 | 10 | `signal_decomposition`, `aether_embeddings`, `gpu_available` | `multi_person_neural = true`, `pose_accuracy_pck02 = 0.6` | GPU |
|
||||
| `adr021_vital_core` | 021 | 3 | `sota_signal_processing` | `vital_signs_extraction = true` | None |
|
||||
| `adr021_vital_esp32` | 021 | 5 | `vital_signs_extraction`, `esp32_connected` | `vital_signs_esp32_validated = true`, `breathing_snr_db = 10.0` | ESP32 |
|
||||
| `adr030_persist_field` | 030 | 2 | `ruvector_signal_integrated` | `persistent_field_model = true` | None |
|
||||
| `adr022_p2_concurrent` | 022 | 4 | `multi_bssid_scan`, `wifi_adapter_present` | `multi_bssid_concurrent = true` | WiFi adapter |
|
||||
| `adr029_p1_ch_hop` | 029 | 5 | `esp32_firmware_base`, `esp32_connected` | `esp32_channel_hopping = true` | ESP32 |
|
||||
| `adr029_p2_multiband` | 029 | 5 | `esp32_channel_hopping` | `multi_band_fusion = true` | ESP32 |
|
||||
| `adr029_p3_multistatic` | 029 | 5 | `multi_band_fusion`, `esp32_multistatic_ready` | `multistatic_mesh = true` | 2+ ESP32 |
|
||||
| `adr029_p67_coherence` | 029 | 3 | `multi_band_fusion` | `coherence_gating = true` | None |
|
||||
| `adr029_p4_tracker` | 029 | 3 | `multistatic_mesh`, `coherence_gating` | `pose_tracker_17kp = true`, `torso_jitter_mm = 30.0` | None |
|
||||
| `adr024_aether_train` | 024 | 8 | `sota_signal_processing`, `gpu_available` | `aether_embeddings = true` | GPU |
|
||||
| `adr024_aether_reid` | 024 | 4 | `aether_embeddings`, `pose_tracker_17kp` | `aether_reid = true` | None |
|
||||
| `adr027_meridian` | 027 | 10 | `aether_embeddings`, `gpu_available` | `meridian_generalization = true` | GPU |
|
||||
| `adr025_wasm` | 025 | 5 | `sota_signal_processing` | `wasm_deployment = true` | None |
|
||||
| `adr011_mat` | 011 | 8 | `vital_signs_extraction`, `person_count_estimation` | `mat_survivor_detection = true` | None |
|
||||
| `adr031_ruview` | 031 | 4 | `persistent_field_model`, `coherence_gating` | `ruview_sensing_ui = true` | None |
|
||||
| `adr032_mesh_security` | 032 | 5 | `multistatic_mesh` | `mesh_security_hardened = true` | None |
|
||||
|
||||
### 2.3 Goal Specification
|
||||
|
||||
Goals are expressed as partial world states -- a set of conditions that must be satisfied.
|
||||
|
||||
```rust
|
||||
pub struct Goal {
|
||||
/// Human-readable description
|
||||
pub description: String,
|
||||
/// Conditions that define success
|
||||
pub conditions: Vec<Condition>,
|
||||
/// Priority weight (higher = more important when competing)
|
||||
pub priority: f32,
|
||||
}
|
||||
```
|
||||
|
||||
**Predefined goal templates:**
|
||||
|
||||
| Goal | Conditions | Typical Plan Length |
|
||||
|------|-----------|---------------------|
|
||||
| Multi-person tracking | `multi_skeleton_generation = true`, `max_persons_tracked >= 3` | 4-6 actions |
|
||||
| Vital sign monitoring | `vital_signs_esp32_validated = true`, `breathing_snr_db >= 10` | 2-3 actions |
|
||||
| Production accuracy | `pose_accuracy_pck02 >= 0.6`, `torso_jitter_mm <= 30` | 5-8 actions |
|
||||
| Browser deployment | `wasm_deployment = true` | 1-2 actions |
|
||||
| Disaster response (MAT) | `mat_survivor_detection = true`, `multi_skeleton_generation = true` | 5-7 actions |
|
||||
| Full multistatic mesh | `multistatic_mesh = true`, `coherence_gating = true`, `pose_tracker_17kp = true` | 5-7 actions |
|
||||
| Cross-environment robustness | `meridian_generalization = true` | 3-5 actions |
|
||||
|
||||
### 2.4 Sublinear Planning Algorithm
|
||||
|
||||
The planner avoids exhaustive A* search over the full state space using three techniques.
|
||||
|
||||
#### 2.4.1 Backward Relevance Pruning
|
||||
|
||||
Before search begins, identify which actions are **relevant** to the goal using backward chaining:
|
||||
|
||||
```
|
||||
function relevantActions(goal, allActions):
|
||||
relevant = {}
|
||||
frontier = {conditions in goal that are not satisfied}
|
||||
|
||||
while frontier is not empty:
|
||||
pick condition C from frontier
|
||||
for each action A in allActions:
|
||||
if A.effects satisfies C:
|
||||
relevant.add(A)
|
||||
for each precondition P of A:
|
||||
if P is not satisfied in current state:
|
||||
frontier.add(P)
|
||||
|
||||
return relevant
|
||||
```
|
||||
|
||||
This typically reduces the action set from ~80 to 5-15 for a specific goal. The search then operates only on relevant actions.
|
||||
|
||||
**Complexity**: O(G * A) where G is the number of unsatisfied goal/precondition properties and A is the total action count. Since G << 2^N and A is fixed at ~80, this is constant-time relative to the state space.
|
||||
|
||||
#### 2.4.2 Hierarchical Decomposition
|
||||
|
||||
Actions are organized into three tiers based on the ADR dependency structure:
|
||||
|
||||
```
|
||||
Tier 0 (Foundation): ADR-014, ADR-016, ADR-018
|
||||
No internal prerequisites. Always satisfiable.
|
||||
|
||||
Tier 1 (Infrastructure): ADR-017, ADR-021-core, ADR-022-p1, ADR-029-p1, ADR-030
|
||||
Depend only on Tier 0.
|
||||
|
||||
Tier 2 (Capability): ADR-024, ADR-029-p2/p3, ADR-037-p1/p2, ADR-021-esp32
|
||||
Depend on Tier 0-1.
|
||||
|
||||
Tier 3 (Integration): ADR-027, ADR-037-p3/p4, ADR-029-p4, ADR-011, ADR-031
|
||||
Depend on Tier 0-2.
|
||||
```
|
||||
|
||||
The planner first resolves Tier 0 preconditions (usually already satisfied), then plans Tier 1 actions, then Tier 2, then Tier 3. Within each tier, actions are independent and can be planned in parallel. This reduces the effective search depth from ~15 (worst case linear chain) to ~4 (tier depth).
|
||||
|
||||
#### 2.4.3 Incremental Replanning
|
||||
|
||||
When the world state changes (a test passes, hardware is plugged in, the user shifts goals), the planner does not replan from scratch. Instead:
|
||||
|
||||
1. **Invalidation**: Mark actions in the current plan whose preconditions are no longer satisfied or whose effects are already achieved.
|
||||
2. **Patch**: Remove invalidated actions and re-run backward relevance pruning only for the remaining unsatisfied goal conditions.
|
||||
3. **Merge**: Insert new actions into the existing plan at the correct dependency-ordered position.
|
||||
|
||||
This is sublinear in the total action count because only the delta is re-examined.
|
||||
|
||||
#### 2.4.4 Heuristic Cost Function
|
||||
|
||||
The A* heuristic estimates remaining cost as the sum of minimum-cost actions needed to satisfy each unsatisfied goal condition, divided by the maximum parallelism available (number of idle agents). This is admissible (never overestimates) because actions can satisfy multiple conditions.
|
||||
|
||||
```
|
||||
h(state, goal) = sum(min_cost_to_satisfy(c) for c in unsatisfied(state, goal)) / max_parallelism
|
||||
```
|
||||
|
||||
#### 2.4.5 Complexity Analysis
|
||||
|
||||
| Component | Naive GOAP | Sublinear GOAP |
|
||||
|-----------|-----------|----------------|
|
||||
| State space | 2^N (N=25 booleans) = 33M | Pruned to relevant subset |
|
||||
| Actions evaluated | All ~80 per expansion | 5-15 (backward pruning) |
|
||||
| Search depth | Up to 15 | Up to 4 (tier decomposition) |
|
||||
| Replan cost | Full re-search | Delta patch only |
|
||||
| Typical plan time | ~100ms | <5ms |
|
||||
|
||||
### 2.5 State Observation
|
||||
|
||||
The planner queries the real project state before planning. Each property has a defined observation method.
|
||||
|
||||
| Property | Observation Command | Cache TTL |
|
||||
|----------|-------------------|-----------|
|
||||
| `sota_signal_processing` | `cargo test -p wifi-densepose-signal --no-default-features 2>&1 \| grep "test result"` | 10 min |
|
||||
| `esp32_connected` | Platform-specific USB serial probe | 30 sec |
|
||||
| `esp32_count` | Count ESP32 VID/PID USB devices | 30 sec |
|
||||
| `gpu_available` | `nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null` | 5 min |
|
||||
| `rust_test_count` | Parse `cargo test --workspace --no-default-features` output | 10 min |
|
||||
| `wifi_adapter_present` | OS-specific WiFi interface enumeration | 5 min |
|
||||
| Module existence flags | `test -f <path>` for key source files | 1 min |
|
||||
|
||||
Observations are cached with TTL to avoid re-running expensive commands (cargo test) on every plan request. Cache invalidation occurs on file change events or explicit user request.
|
||||
|
||||
### 2.6 Plan Execution via Swarm
|
||||
|
||||
Once the planner produces an ordered action list, execution is dispatched through the claude-flow swarm system.
|
||||
|
||||
#### 2.6.1 GOAP Coordinator Agent
|
||||
|
||||
The planner runs as a `goap-coordinator` agent within a hierarchical swarm topology:
|
||||
|
||||
```
|
||||
goap-coordinator (planner + dispatcher)
|
||||
|
|
||||
+-- researcher (dependency analysis, API review)
|
||||
+-- coder (implementation)
|
||||
+-- tester (validation, state observation)
|
||||
+-- reviewer (code review, security check)
|
||||
```
|
||||
|
||||
The coordinator:
|
||||
1. Observes current world state
|
||||
2. Accepts a goal from the user
|
||||
3. Runs the sublinear planner to produce an action sequence
|
||||
4. Dispatches each action to appropriate agent types (from the action's `agent_types` field)
|
||||
5. Monitors action completion via the memory system
|
||||
6. Updates the world state after each action completes
|
||||
7. Re-plans if the world state diverges from expectations
|
||||
|
||||
#### 2.6.2 State Persistence via Memory
|
||||
|
||||
World state is stored in the claude-flow memory system under the `goap` namespace:
|
||||
|
||||
```bash
|
||||
# Store observed state
|
||||
npx @claude-flow/cli@latest memory store \
|
||||
--namespace goap \
|
||||
--key "world-state" \
|
||||
--value '{"sota_signal_processing": true, "esp32_connected": false, ...}'
|
||||
|
||||
# Store current plan
|
||||
npx @claude-flow/cli@latest memory store \
|
||||
--namespace goap \
|
||||
--key "current-plan" \
|
||||
--value '{"goal": "multi-person tracking", "actions": ["adr037_p1", "adr037_p2", ...], "progress": 1}'
|
||||
|
||||
# Search for past successful plans
|
||||
npx @claude-flow/cli@latest memory search \
|
||||
--namespace goap \
|
||||
--query "multi-person tracking plan"
|
||||
```
|
||||
|
||||
#### 2.6.3 Action-to-Agent Routing
|
||||
|
||||
Each action declares which agent types are needed. The coordinator maps these to swarm agents:
|
||||
|
||||
| Agent Type | Role in GOAP Action | Example Actions |
|
||||
|-----------|---------------------|-----------------|
|
||||
| `researcher` | Analyze dependencies, review papers, check API compatibility | Pre-action analysis for any ADR |
|
||||
| `coder` | Write implementation code | All implementation actions |
|
||||
| `tester` | Run tests, observe state, validate effects | Post-action verification |
|
||||
| `reviewer` | Code review, security audit | ADR-032 mesh security, any PR |
|
||||
| `performance-engineer` | Benchmark, optimize latency | ADR-029 pipeline timing |
|
||||
| `security-architect` | Threat model, audit | ADR-032 security hardening |
|
||||
|
||||
#### 2.6.4 Execution Protocol
|
||||
|
||||
For each action in the plan:
|
||||
|
||||
```
|
||||
1. PRE-CHECK: Observe preconditions. If any unsatisfied, re-plan.
|
||||
2. DISPATCH: Spawn required agents with action context.
|
||||
3. EXECUTE: Agents implement the action (write code, run tests).
|
||||
4. VERIFY: Tester agent observes the world state.
|
||||
5. UPDATE: If effects achieved, mark action complete, update state.
|
||||
6. REPLAN: If effects not achieved, flag failure, re-plan with updated state.
|
||||
```
|
||||
|
||||
### 2.7 Dependency Graph Visualization
|
||||
|
||||
The planner can emit its action graph in DOT format for visualization:
|
||||
|
||||
```
|
||||
digraph goap {
|
||||
rankdir=LR;
|
||||
node [shape=box, style=rounded];
|
||||
|
||||
// Tier 0 (green = complete)
|
||||
adr014 [label="ADR-014\nSOTA Signal", color=green];
|
||||
adr016 [label="ADR-016\nRuVector Train", color=green];
|
||||
adr018 [label="ADR-018\nESP32 Base", color=green];
|
||||
|
||||
// Tier 1 (blue = in progress)
|
||||
adr017 [label="ADR-017\nRuVector Signal", color=blue];
|
||||
adr030 [label="ADR-030\nField Model", color=orange];
|
||||
|
||||
// Tier 2 (orange = planned)
|
||||
adr037_p1 [label="ADR-037 P1\nPerson Count", color=orange];
|
||||
adr037_p2 [label="ADR-037 P2\nNMF Decomp", color=orange];
|
||||
adr024 [label="ADR-024\nAETHER", color=orange];
|
||||
|
||||
// Tier 3 (gray = future)
|
||||
adr037_p3 [label="ADR-037 P3\nMulti-Skeleton", color=gray];
|
||||
adr027 [label="ADR-027\nMERIDIAN", color=gray];
|
||||
|
||||
// Edges
|
||||
adr014 -> adr037_p1;
|
||||
adr037_p1 -> adr037_p2;
|
||||
adr037_p2 -> adr037_p3;
|
||||
adr014 -> adr024;
|
||||
adr024 -> adr037_p3;
|
||||
adr024 -> adr027;
|
||||
adr014 -> adr017;
|
||||
adr017 -> adr030;
|
||||
}
|
||||
```
|
||||
|
||||
### 2.8 PageRank-Based Prioritization
|
||||
|
||||
When the user has not specified a single goal but asks "what should I work on next?", the planner uses PageRank on the action dependency graph to identify the highest-leverage actions:
|
||||
|
||||
1. Construct the adjacency matrix where `A[i][j] = 1` if action j depends on action i (i.e., completing i unblocks j).
|
||||
2. Run PageRank with damping factor 0.85.
|
||||
3. Actions with the highest PageRank scores are the most "load-bearing" -- they unblock the most downstream work.
|
||||
4. Filter to actions whose preconditions are currently satisfiable.
|
||||
5. Return the top-K actions ranked by `PageRank_score * (1 / cost_days)` (value per effort).
|
||||
|
||||
This naturally surfaces foundation actions (ADR-014, ADR-016) over leaf actions (ADR-032 security), matching the intuition that infrastructure work has the highest leverage.
|
||||
|
||||
---
|
||||
|
||||
## 3. Implementation
|
||||
|
||||
### 3.1 Module Structure
|
||||
|
||||
The GOAP planner is implemented as a TypeScript module within the claude-flow coordination layer (not in the Rust workspace, since it orchestrates Rust development rather than being part of the Rust product).
|
||||
|
||||
```
|
||||
.claude-flow/goap/
|
||||
state.ts -- World state model and observation
|
||||
actions.ts -- Action catalog (all ~80 actions)
|
||||
planner.ts -- Sublinear A* planner with backward pruning
|
||||
goals.ts -- Goal templates and user goal parser
|
||||
executor.ts -- Swarm dispatch and action lifecycle
|
||||
pagerank.ts -- Dependency graph prioritization
|
||||
visualize.ts -- DOT graph export
|
||||
```
|
||||
|
||||
### 3.2 CLI Integration
|
||||
|
||||
```bash
|
||||
# Plan: produce an action sequence for a goal
|
||||
npx @claude-flow/cli@latest goap plan --goal "multi-person tracking"
|
||||
|
||||
# Observe: snapshot current world state
|
||||
npx @claude-flow/cli@latest goap observe
|
||||
|
||||
# Prioritize: PageRank-based "what next?" recommendation
|
||||
npx @claude-flow/cli@latest goap prioritize --top-k 5
|
||||
|
||||
# Execute: run the plan via swarm
|
||||
npx @claude-flow/cli@latest goap execute --goal "vital sign monitoring"
|
||||
|
||||
# Visualize: emit DOT dependency graph
|
||||
npx @claude-flow/cli@latest goap graph --format dot > goap.dot
|
||||
```
|
||||
|
||||
### 3.3 Integration Points
|
||||
|
||||
| System | Integration | Purpose |
|
||||
|--------|------------|---------|
|
||||
| claude-flow memory | `goap` namespace | Persist world state, plans, execution history |
|
||||
| claude-flow swarm | Hierarchical coordinator | Dispatch actions to agent teams |
|
||||
| claude-flow hooks | `pre-task` / `post-task` | Trigger state observation before/after work |
|
||||
| cargo test | State observation | Detect which crates/modules pass tests |
|
||||
| USB device enumeration | Hardware observation | Detect ESP32 availability |
|
||||
| Git status | Implementation detection | Check if files/modules exist |
|
||||
|
||||
---
|
||||
|
||||
## 4. Consequences
|
||||
|
||||
### 4.1 Positive
|
||||
|
||||
- **Eliminates manual priority analysis**: The developer states a goal; the planner produces a concrete, dependency-ordered action list.
|
||||
- **Hardware-aware planning**: Actions requiring ESP32 or GPU are automatically excluded when hardware is unavailable, preventing dead-end plans.
|
||||
- **Sublinear plan time**: Backward pruning + tier decomposition keeps planning under 5ms for typical goals, enabling interactive replanning.
|
||||
- **Incremental replanning**: When state changes (a test starts passing, hardware is plugged in), only the delta is re-evaluated.
|
||||
- **Swarm integration**: Actions are dispatched to specialized agents, enabling parallel execution of independent actions within the same tier.
|
||||
- **Cross-session continuity**: World state and plan progress persist in the memory system, so the planner resumes where it left off.
|
||||
- **PageRank prioritization**: When no specific goal is given, the planner identifies the highest-leverage next action based on the dependency graph structure.
|
||||
- **Transparent reasoning**: The dependency graph can be visualized in DOT format, making the planner's reasoning inspectable.
|
||||
|
||||
### 4.2 Negative
|
||||
|
||||
- **Action catalog maintenance**: Every new ADR or ADR phase must be added to the action catalog with correct preconditions and effects. Stale actions produce incorrect plans.
|
||||
- **State observation overhead**: Some state checks (running `cargo test`) are expensive. Caching with TTL mitigates this but introduces staleness risk.
|
||||
- **Approximate cost model**: Action costs in developer-days are estimates. Actual effort varies with developer experience and codebase familiarity.
|
||||
- **Boolean state simplification**: Some capabilities are continuous (accuracy improves gradually) but are modeled as boolean thresholds, losing nuance.
|
||||
|
||||
### 4.3 Risks
|
||||
|
||||
| Risk | Probability | Impact | Mitigation |
|
||||
|------|-------------|--------|------------|
|
||||
| Action catalog diverges from reality | Medium | Plans reference nonexistent or completed actions | Validate catalog against ADR directory at plan time |
|
||||
| State observation produces false positives | Low | Planner skips needed actions | Cross-validate with multiple observation methods |
|
||||
| User goals conflict (accuracy vs latency) | Medium | Planner produces suboptimal compromise | Support multi-objective goals with explicit weights |
|
||||
| Swarm agents fail during action execution | Medium | Plan stalls | Timeout + automatic replan with failure noted in state |
|
||||
|
||||
---
|
||||
|
||||
## 5. Affected Components
|
||||
|
||||
| Component | Change | Description |
|
||||
|-----------|--------|-------------|
|
||||
| `.claude-flow/goap/` | New | GOAP planner module (TypeScript) |
|
||||
| claude-flow memory (`goap` namespace) | New | World state and plan persistence |
|
||||
| claude-flow swarm coordinator | Extended | GOAP coordinator agent type |
|
||||
| claude-flow CLI | Extended | `goap` subcommand (plan, observe, prioritize, execute, graph) |
|
||||
|
||||
---
|
||||
|
||||
## 6. Performance Budget
|
||||
|
||||
| Operation | Budget | Method |
|
||||
|-----------|--------|--------|
|
||||
| World state observation (cached) | < 100ms | Read from memory cache |
|
||||
| World state observation (fresh) | < 30s | Run cargo test + hardware probes |
|
||||
| Plan generation (sublinear) | < 5ms | Backward pruning + tier A* |
|
||||
| PageRank prioritization | < 2ms | Sparse matrix iteration |
|
||||
| Incremental replan | < 1ms | Delta patch on existing plan |
|
||||
| DOT graph generation | < 1ms | Traverse action catalog |
|
||||
|
||||
---
|
||||
|
||||
## 7. Alternatives Considered
|
||||
|
||||
1. **Manual priority spreadsheet**: Maintain a spreadsheet of ADR priorities and dependencies. Rejected because it requires manual updates, does not adapt to hardware availability, and cannot be queried programmatically by agents.
|
||||
|
||||
2. **Full A* over raw state space**: Standard GOAP without sublinear optimizations. Rejected because 2^25 boolean states is unnecessarily large when most actions are irrelevant to any given goal.
|
||||
|
||||
3. **Hierarchical Task Network (HTN)**: HTN decomposes tasks into subtasks using predefined methods. More powerful than GOAP but requires hand-authored decomposition methods for every task. GOAP's flat action model with automatic planning is simpler to maintain as ADRs evolve.
|
||||
|
||||
4. **Reinforcement learning planner**: Train an RL agent to select actions. Rejected because the action space changes as ADRs are added, the reward signal is sparse (project completion), and the sample complexity is too high for a planning problem with known structure.
|
||||
|
||||
5. **Simple topological sort**: Sort actions by dependency order and execute top-down. Rejected because it does not consider goals (executes everything), does not handle hardware constraints, and does not support replanning.
|
||||
|
||||
---
|
||||
|
||||
## 8. References
|
||||
|
||||
1. Orkin, J. (2003). "Applying Goal-Oriented Action Planning to Games." AI Game Programming Wisdom 2.
|
||||
2. Orkin, J. (2006). "Three States and a Plan: The A.I. of F.E.A.R." Game Developers Conference.
|
||||
3. Page, L., Brin, S., Motwani, R., Winograd, T. (1999). "The PageRank Citation Ranking: Bringing Order to the Web." Stanford InfoLab.
|
||||
4. Ghallab, M., Nau, D., Traverso, P. (2004). "Automated Planning: Theory and Practice." Morgan Kaufmann.
|
||||
5. Russell, S., Norvig, P. (2020). "Artificial Intelligence: A Modern Approach." 4th ed., Chapter 11: Automated Planning.
|
||||
@@ -0,0 +1,299 @@
|
||||
# ADR-039: ESP32-S3 Edge Intelligence — On-Device Signal Processing and RuVector Integration
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-03-03 |
|
||||
| **Depends on** | ADR-018 (binary frame format), ADR-014 (SOTA signal processing), ADR-021 (vital sign extraction), ADR-029 (multistatic sensing), ADR-030 (persistent field model), ADR-031 (RuView sensing-first RF) |
|
||||
| **Supersedes** | None |
|
||||
|
||||
## Context
|
||||
|
||||
The current ESP32-S3 firmware (1,018 lines, 7 files) is a "dumb sensor" — it captures raw CSI frames and streams them unprocessed over UDP at ~20 Hz. All signal processing, feature extraction, presence detection, vital sign estimation, and pose inference happen server-side in the Rust crates.
|
||||
|
||||
This creates several limitations:
|
||||
1. **Bandwidth waste** — raw CSI frames are 128-384 bytes each at 20 Hz = ~60 KB/s per node. Most of this is noise.
|
||||
2. **Latency** — round-trip to server adds 5-50ms depending on network.
|
||||
3. **Server dependency** — nodes are useless without an active aggregator.
|
||||
4. **Scalability ceiling** — 6-node mesh at 20 Hz = 120 frames/s = server bottleneck.
|
||||
5. **No local alerting** — fall detection, breathing anomaly, or intrusion must wait for server roundtrip.
|
||||
|
||||
The ESP32-S3 has significant untapped compute:
|
||||
- **Dual-core Xtensa LX7** at 240 MHz
|
||||
- **512 KB SRAM** + optional 8 MB PSRAM (our board has 8 MB flash)
|
||||
- **Vector/DSP instructions** (PIE — Processor Instruction Extensions)
|
||||
- **FPU** — hardware single-precision floating point
|
||||
- **~80% idle CPU** — current firmware uses <20% (WiFi + CSI callback + UDP send)
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a **3-tier edge intelligence pipeline** on the ESP32-S3 firmware, progressively offloading signal processing from the server to the device. Each tier is independently toggleable via NVS configuration.
|
||||
|
||||
### Tier 1: Smart Filtering & Compression (Firmware C)
|
||||
|
||||
Lightweight processing in the CSI callback path. Zero additional latency.
|
||||
|
||||
| Feature | Source ADR | Algorithm | Memory | CPU |
|
||||
|---------|-----------|-----------|--------|-----|
|
||||
| **Phase sanitization** | ADR-014 | Linear phase unwrap + conjugate multiply | 256 B | <1% |
|
||||
| **Amplitude normalization** | ADR-014 | Per-subcarrier running mean/std (Welford) | 512 B | <1% |
|
||||
| **Subcarrier selection** | ADR-016 (ruvector-mincut) | Top-K variance subcarriers | 128 B | <1% |
|
||||
| **Static environment suppression** | ADR-030 | Exponential moving average subtraction | 512 B | <1% |
|
||||
| **Adaptive frame decimation** | New | Skip frames when CSI variance < threshold | 8 B | <1% |
|
||||
| **Delta compression** | New | XOR + RLE vs. previous frame | 512 B | <2% |
|
||||
|
||||
**Bandwidth reduction**: 60-80% (send only changed, high-variance subcarriers).
|
||||
|
||||
**ADR-018 v2 frame extension** (backward-compatible):
|
||||
|
||||
```
|
||||
Existing 20-byte header unchanged.
|
||||
New optional trailer (if magic bit set):
|
||||
[N*2] Compressed I/Q (delta-coded, only selected subcarriers)
|
||||
[2] Subcarrier bitmap (which of 64 subcarriers included)
|
||||
[1] Frame flags: bit0=compressed, bit1=phase-sanitized, bit2=amplitude-normed
|
||||
[1] Motion score (0-255)
|
||||
[1] Presence confidence (0-255)
|
||||
[1] Reserved
|
||||
```
|
||||
|
||||
### Tier 2: On-Device Vital Signs & Presence (Firmware C + fixed-point DSP)
|
||||
|
||||
Runs as a FreeRTOS task on Core 1 (CSI collection on Core 0), processing a sliding window of CSI frames.
|
||||
|
||||
| Feature | Source ADR | Algorithm | Memory | CPU (Core 1) |
|
||||
|---------|-----------|-----------|--------|--------------|
|
||||
| **Presence detection** | ADR-029 | Variance threshold on amplitude envelope | 2 KB | 5% |
|
||||
| **Motion scoring** | ADR-014 | Subcarrier correlation coefficient | 1 KB | 3% |
|
||||
| **Breathing rate** | ADR-021 | Bandpass 0.1-0.5 Hz + peak detection on CSI phase | 8 KB | 10% |
|
||||
| **Heart rate** | ADR-021 | Bandpass 0.8-2.0 Hz + autocorrelation on CSI phase | 8 KB | 15% |
|
||||
| **Fall detection** | ADR-029 | Sudden variance spike + sustained stillness | 1 KB | 2% |
|
||||
| **Room occupancy count** | ADR-037 | CSI rank estimation (eigenvalue spread) | 4 KB | 8% |
|
||||
| **Coherence gate** | ADR-029 (ruvsense) | Z-score coherence, accept/reject/recalibrate | 1 KB | 2% |
|
||||
|
||||
**Total memory**: ~25 KB (fits in SRAM, no PSRAM needed).
|
||||
**Total CPU**: ~45% of Core 1.
|
||||
|
||||
**Output**: Compact vital-signs UDP packet (32 bytes) at 1 Hz:
|
||||
|
||||
```
|
||||
Offset Size Field
|
||||
0 4 Magic: 0xC5110002 (vitals packet)
|
||||
4 1 Node ID
|
||||
5 1 Packet type (0x02 = vitals)
|
||||
6 2 Sequence (LE u16)
|
||||
8 1 Presence (0=empty, 1=present, 2=moving)
|
||||
9 1 Motion score (0-255)
|
||||
10 1 Occupancy estimate (0-8 persons)
|
||||
11 1 Coherence gate (0=reject, 1=predict, 2=accept, 3=recalibrate)
|
||||
12 2 Breathing rate (BPM * 100, LE u16) — 0 if not detected
|
||||
14 2 Heart rate (BPM * 100, LE u16) — 0 if not detected
|
||||
16 2 Breathing confidence (0-10000, LE u16)
|
||||
18 2 Heart rate confidence (0-10000, LE u16)
|
||||
20 1 Fall detected (0/1)
|
||||
21 1 Anomaly flags (bitfield)
|
||||
22 2 Ambient RSSI mean (LE i16)
|
||||
24 4 CSI frame count since last report (LE u32)
|
||||
28 4 Uptime seconds (LE u32)
|
||||
```
|
||||
|
||||
### Tier 3: Lightweight Feature Extraction (Firmware C + optional PSRAM)
|
||||
|
||||
Pre-compute features that the server-side neural network needs, reducing server CPU by 60-80%.
|
||||
|
||||
| Feature | Source ADR | Algorithm | Memory | CPU |
|
||||
|---------|-----------|-----------|--------|-----|
|
||||
| **Phase difference matrix** | ADR-014 | Adjacent subcarrier phase diff | 4 KB | 5% |
|
||||
| **Amplitude spectrogram** | ADR-014 | 64-bin FFT on 1s window per subcarrier | 32 KB | 15% |
|
||||
| **Doppler-time map** | ADR-029 | 2D FFT across subcarriers × time | 16 KB | 10% |
|
||||
| **Fresnel zone crossing** | ADR-014 | First Fresnel radius + fade count | 1 KB | 2% |
|
||||
| **Cross-link correlation** | ADR-029 | Pearson correlation between TX-RX pairs | 2 KB | 5% |
|
||||
| **Environment fingerprint** | ADR-027 (MERIDIAN) | PCA-compressed 16-dim CSI signature | 4 KB | 5% |
|
||||
| **Gesture template match** | ADR-029 (ruvsense) | DTW on 8-dim feature vector | 8 KB | 10% |
|
||||
|
||||
**Total memory**: ~67 KB (SRAM) or up to 256 KB with PSRAM.
|
||||
**Total CPU**: ~52% of Core 1.
|
||||
|
||||
**Output**: Feature vector UDP packet (variable size, ~200-500 bytes) at 4 Hz:
|
||||
|
||||
```
|
||||
Offset Size Field
|
||||
0 4 Magic: 0xC5110003 (feature packet)
|
||||
4 1 Node ID
|
||||
5 1 Packet type (0x03 = features)
|
||||
6 2 Feature bitmap (which features included)
|
||||
8 4 Timestamp ms (LE u32)
|
||||
12 N Feature payloads (concatenated, lengths determined by bitmap)
|
||||
```
|
||||
|
||||
## NVS Configuration
|
||||
|
||||
All tiers controllable via NVS without reflashing:
|
||||
|
||||
| NVS Key | Type | Default | Description |
|
||||
|---------|------|---------|-------------|
|
||||
| `edge_tier` | u8 | 0 | 0=raw only, 1=smart filter, 2=+vitals, 3=+features |
|
||||
| `decim_thresh` | u16 | 100 | Adaptive decimation variance threshold |
|
||||
| `subk_count` | u8 | 32 | Top-K subcarriers to keep (Tier 1) |
|
||||
| `vital_window` | u16 | 300 | Vital sign window frames (15s at 20 Hz) |
|
||||
| `vital_interval` | u16 | 1000 | Vital report interval ms |
|
||||
| `feature_hz` | u8 | 4 | Feature extraction rate |
|
||||
| `fall_thresh` | u16 | 500 | Fall detection variance spike threshold |
|
||||
| `presence_thresh` | u16 | 50 | Presence detection threshold |
|
||||
|
||||
Provisioning:
|
||||
```bash
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--edge-tier 2 --vital-window 300 --presence-thresh 50
|
||||
```
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Infrastructure (1 week)
|
||||
|
||||
1. **Dual-core task architecture**
|
||||
- Core 0: WiFi + CSI callback (existing)
|
||||
- Core 1: Edge processing task (new FreeRTOS task)
|
||||
- Lock-free ring buffer between cores (producer-consumer)
|
||||
|
||||
2. **Ring buffer design**
|
||||
```c
|
||||
#define RING_BUF_FRAMES 64 // ~3.2s at 20 Hz
|
||||
typedef struct {
|
||||
wifi_csi_info_t info;
|
||||
int8_t iq_data[384]; // Max I/Q payload
|
||||
uint32_t timestamp_ms;
|
||||
uint8_t tx_mac[6];
|
||||
} csi_ring_entry_t;
|
||||
```
|
||||
|
||||
3. **NVS config extension** — add `edge_tier` and tier-specific params
|
||||
4. **ADR-018 v2 header** — backward-compatible extension bit
|
||||
|
||||
### Phase 2: Tier 1 — Smart Filtering (1 week)
|
||||
|
||||
1. **Phase unwrap** — O(N) linear scan, in-place
|
||||
2. **Welford running stats** — per-subcarrier mean/variance, O(1) update
|
||||
3. **Top-K subcarrier selection** — partial sort, O(N) with selection algorithm
|
||||
4. **Delta compression** — XOR vs previous frame, RLE encode
|
||||
5. **Adaptive decimation** — skip frame if total variance < threshold
|
||||
|
||||
### Phase 3: Tier 2 — Vital Signs (2 weeks)
|
||||
|
||||
1. **Presence detector** — amplitude variance over 1s window
|
||||
2. **Motion scorer** — correlation coefficient between consecutive frames
|
||||
3. **Breathing extractor** — port from `wifi-densepose-vitals::BreathingExtractor::esp32_default()`
|
||||
- Bandpass via biquad IIR filter (0.1-0.5 Hz)
|
||||
- Peak detection with parabolic interpolation
|
||||
- Fixed-point arithmetic (Q15.16) for efficiency
|
||||
4. **Heart rate extractor** — port from `wifi-densepose-vitals::HeartRateExtractor::esp32_default()`
|
||||
- Bandpass via biquad IIR (0.8-2.0 Hz)
|
||||
- Autocorrelation peak search
|
||||
5. **Fall detection** — variance spike (>5σ) followed by sustained stillness (>3s)
|
||||
6. **Coherence gate** — port from `ruvsense::coherence_gate` (Z-score threshold)
|
||||
|
||||
### Phase 4: Tier 3 — Feature Extraction (2 weeks)
|
||||
|
||||
1. **FFT engine** — fixed-point 64-point FFT (radix-2 DIT, no library needed)
|
||||
2. **Amplitude spectrogram** — 1s sliding window FFT per subcarrier
|
||||
3. **Doppler-time map** — 2D FFT across subcarrier × time dimensions
|
||||
4. **Phase difference matrix** — adjacent subcarrier Δφ
|
||||
5. **Environment fingerprint** — online PCA (incremental SVD, 16 components)
|
||||
6. **Gesture DTW** — 8 stored templates, dynamic time warping on 8-dim feature
|
||||
|
||||
### Phase 5: CI/CD + Testing (1 week)
|
||||
|
||||
1. **GitHub Actions firmware build** — Docker `espressif/idf:v5.2` on every PR
|
||||
2. **Host-side unit tests** — compile edge processing functions on x86 with mock CSI data
|
||||
3. **Credential leak check** — binary string scan in CI
|
||||
4. **Binary size tracking** — fail CI if firmware exceeds 90% of partition
|
||||
5. **QEMU smoke test** — boot verification, NVS load, task creation
|
||||
|
||||
## ESP32-S3 Resource Budget
|
||||
|
||||
| Resource | Available | Tier 1 | Tier 2 | Tier 3 | Remaining |
|
||||
|----------|-----------|--------|--------|--------|-----------|
|
||||
| **SRAM** | 512 KB | 2 KB | 25 KB | 67 KB | 418 KB |
|
||||
| **Core 0 CPU** | 100% | 5% | 0% | 0% | 75% (WiFi uses ~20%) |
|
||||
| **Core 1 CPU** | 100% | 0% | 45% | 52% | 3% (Tier 2+3 exclusive) |
|
||||
| **Flash** | 1 MB partition | 4 KB code | 12 KB code | 20 KB code | 964 KB |
|
||||
|
||||
Note: Tier 2 and Tier 3 run on Core 1 but are time-multiplexed — vitals at 1 Hz, features at 4 Hz. Combined peak load is ~60% of Core 1.
|
||||
|
||||
## Mapping to Existing ADRs
|
||||
|
||||
| Existing ADR | Capability | Edge Tier | Implementation |
|
||||
|-------------|------------|-----------|----------------|
|
||||
| **ADR-014** (SOTA signal) | Phase sanitization | 1 | Linear unwrap in CSI callback |
|
||||
| **ADR-014** | Amplitude normalization | 1 | Welford running stats |
|
||||
| **ADR-014** | Feature extraction | 3 | FFT spectrogram + phase diff matrix |
|
||||
| **ADR-014** | Fresnel zone detection | 3 | Fade counting + first Fresnel radius |
|
||||
| **ADR-016** (RuVector) | Subcarrier selection | 1 | Top-K variance (simplified mincut) |
|
||||
| **ADR-021** (Vitals) | Breathing rate | 2 | Biquad IIR + peak detect |
|
||||
| **ADR-021** | Heart rate | 2 | Biquad IIR + autocorrelation |
|
||||
| **ADR-021** | Anomaly detection | 2 | Z-score on vital readings |
|
||||
| **ADR-027** (MERIDIAN) | Environment fingerprint | 3 | Online PCA, 16-dim signature |
|
||||
| **ADR-029** (RuvSense) | Coherence gate | 2 | Z-score coherence scoring |
|
||||
| **ADR-029** | Multistatic correlation | 3 | Pearson cross-link correlation |
|
||||
| **ADR-029** | Gesture recognition | 3 | DTW template matching |
|
||||
| **ADR-030** (Field model) | Static suppression | 1 | EMA background subtraction |
|
||||
| **ADR-031** (RuView) | Sensing-first NDP | Existing | Already in firmware (stub) |
|
||||
| **ADR-037** (Multi-person) | Occupancy counting | 2 | CSI rank estimation |
|
||||
|
||||
## Server-Side Changes
|
||||
|
||||
The Rust aggregator (`wifi-densepose-hardware`) needs to handle the new packet types:
|
||||
|
||||
```rust
|
||||
match magic {
|
||||
0xC5110001 => parse_raw_csi_frame(buf), // Existing
|
||||
0xC5110002 => parse_vitals_packet(buf), // New: Tier 2
|
||||
0xC5110003 => parse_feature_packet(buf), // New: Tier 3
|
||||
_ => Err(ParseError::UnknownMagic(magic)),
|
||||
}
|
||||
```
|
||||
|
||||
When edge tier ≥ 1, the server can skip its own phase sanitization and amplitude normalization. When edge tier = 3, the server skips feature extraction entirely and feeds pre-computed features directly to the neural network.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
| Test Type | Tool | What |
|
||||
|-----------|------|------|
|
||||
| **Host unit tests** | gcc + Unity + mock CSI data | Phase unwrap, Welford stats, IIR filter, peak detect, DTW |
|
||||
| **QEMU smoke test** | Docker QEMU | Boot, NVS load, task creation, ring buffer |
|
||||
| **Hardware regression** | ESP32-S3 + serial log | Full pipeline: CSI → edge processing → UDP → server |
|
||||
| **Accuracy validation** | Python reference impl | Compare edge vitals vs. server vitals on same CSI data |
|
||||
| **Stress test** | 6-node mesh | Tier 3 at 20 Hz sustained, no frame drops |
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **Rust on ESP32 (esp-rs)** — More type-safe, could share code with server crates. Rejected: larger binary, longer compile times, less mature ESP-IDF support for CSI APIs.
|
||||
|
||||
2. **MicroPython on ESP32** — Easier prototyping. Rejected: too slow for 20 Hz real-time processing, no fixed-point DSP.
|
||||
|
||||
3. **External co-processor (FPGA/DSP)** — Maximum throughput. Rejected: cost ($50+ per node), defeats the $8 ESP32 value proposition.
|
||||
|
||||
4. **Server-only processing** — Keep firmware dumb. Rejected: doesn't solve bandwidth, latency, or standalone operation requirements.
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Core 1 processing exceeds real-time budget | Adaptive quality: reduce feature_hz or fall back to lower tier |
|
||||
| Fixed-point arithmetic introduces accuracy drift | Validate against Rust f64 reference on same CSI data; track error bounds |
|
||||
| NVS config complexity overwhelms users | Sensible defaults; provision.py presets: `--preset home`, `--preset medical`, `--preset security` |
|
||||
| ADR-018 v2 header breaks old aggregators | Backward-compatible: old magic = old format. New bit in flags field signals extension |
|
||||
| Memory fragmentation from ring buffer | Static allocation only; no malloc in edge processing path |
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] Tier 1 reduces bandwidth by ≥60% with <1 dB SNR loss
|
||||
- [ ] Tier 2 breathing rate within ±1 BPM of server-side estimate
|
||||
- [ ] Tier 2 heart rate within ±3 BPM of server-side estimate
|
||||
- [ ] Tier 2 fall detection latency <500ms (vs. ~2s server roundtrip)
|
||||
- [ ] Tier 2 presence detection accuracy ≥95%
|
||||
- [ ] Tier 3 feature extraction matches server output within 5% RMSE
|
||||
- [ ] All tiers: zero frame drops at 20 Hz sustained on single node
|
||||
- [ ] Firmware binary stays under 90% of 1 MB app partition
|
||||
- [ ] SRAM usage stays under 400 KB (leave headroom for WiFi stack)
|
||||
- [ ] CI pipeline: build + host unit tests + binary size check on every PR
|
||||
@@ -484,12 +484,12 @@ The training pipeline is implemented in pure Rust (7,832 lines, zero external ML
|
||||
|
||||
The system supports two public WiFi CSI datasets:
|
||||
|
||||
| Dataset | Source | Format | Subjects | Environments |
|
||||
|---------|--------|--------|----------|-------------|
|
||||
| [MM-Fi](https://mmfi.github.io/) | NeurIPS 2023 | `.npy` | 40 | 4 rooms |
|
||||
| [Wi-Pose](https://github.com/aiot-lab/Wi-Pose) | AAAI 2024 | `.mat` | 8 | 3 rooms |
|
||||
| Dataset | Source | Format | Subjects | Environments | Download |
|
||||
|---------|--------|--------|----------|-------------|----------|
|
||||
| [MM-Fi](https://ntu-aiot-lab.github.io/mm-fi) | NeurIPS 2023 | `.npy` | 40 | 4 rooms | [GitHub repo](https://github.com/ybhbingo/MMFi_dataset) (Google Drive / Baidu links inside) |
|
||||
| [Wi-Pose](https://github.com/NjtechCVLab/Wi-PoseDataset) | Entropy 2023 | `.mat` | 12 | 1 room | [GitHub repo](https://github.com/NjtechCVLab/Wi-PoseDataset) (Google Drive / Baidu links inside) |
|
||||
|
||||
Download and place in a `data/` directory.
|
||||
Download the dataset files and place them in a `data/` directory.
|
||||
|
||||
### Step 2: Train
|
||||
|
||||
@@ -612,7 +612,7 @@ A 3-6 node ESP32-S3 mesh provides full CSI at 20 Hz. Total cost: ~$54 for a 3-no
|
||||
|
||||
**Flashing firmware:**
|
||||
|
||||
Pre-built binaries are available at [Releases](https://github.com/ruvnet/wifi-densepose/releases/tag/v0.1.0-esp32).
|
||||
Pre-built binaries are available at [Releases](https://github.com/ruvnet/wifi-densepose/releases/tag/v0.2.0-esp32).
|
||||
|
||||
```bash
|
||||
# Flash an ESP32-S3 (requires esptool: pip install esptool)
|
||||
@@ -624,7 +624,7 @@ python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
**Provisioning:**
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourWiFi" --password "YourPassword" --target-ip 192.168.1.20
|
||||
```
|
||||
|
||||
@@ -635,7 +635,7 @@ Replace `192.168.1.20` with the IP of the machine running the sensing server.
|
||||
For multistatic mesh deployments with authenticated beacons (ADR-032), provision a shared mesh key:
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourWiFi" --password "YourPassword" --target-ip 192.168.1.20 \
|
||||
--mesh-key "$(openssl rand -hex 32)"
|
||||
```
|
||||
@@ -648,13 +648,13 @@ Each node in a multistatic mesh needs a unique TDM slot ID (0-based):
|
||||
|
||||
```bash
|
||||
# Node 0 (slot 0) — first transmitter
|
||||
python scripts/provision.py --port COM7 --tdm-slot 0 --tdm-total 3
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 --tdm-slot 0 --tdm-total 3
|
||||
|
||||
# Node 1 (slot 1)
|
||||
python scripts/provision.py --port COM8 --tdm-slot 1 --tdm-total 3
|
||||
python firmware/esp32-csi-node/provision.py --port COM8 --tdm-slot 1 --tdm-total 3
|
||||
|
||||
# Node 2 (slot 2)
|
||||
python scripts/provision.py --port COM9 --tdm-slot 2 --tdm-total 3
|
||||
python firmware/esp32-csi-node/provision.py --port COM9 --tdm-slot 2 --tdm-total 3
|
||||
```
|
||||
|
||||
**Start the aggregator:**
|
||||
@@ -720,7 +720,7 @@ docker run -p 3000:3000 -p 3001:3001 ruvnet/wifi-densepose:latest
|
||||
### ESP32: No data arriving
|
||||
|
||||
1. Verify the ESP32 is connected to the same WiFi network
|
||||
2. Check the target IP matches the sensing server machine: `python scripts/provision.py --port COM7 --target-ip <YOUR_IP>`
|
||||
2. Check the target IP matches the sensing server machine: `python firmware/esp32-csi-node/provision.py --port COM7 --target-ip <YOUR_IP>`
|
||||
3. Verify UDP port 5005 is not blocked by firewall
|
||||
4. Test with: `nc -lu 5005` (Linux) or similar UDP listener
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
idf_component_register(
|
||||
SRCS "main.c" "csi_collector.c" "stream_sender.c" "nvs_config.c"
|
||||
SRCS "main.c" "csi_collector.c" "stream_sender.c" "nvs_config.c" "edge_processing.c"
|
||||
INCLUDE_DIRS "."
|
||||
)
|
||||
|
||||
@@ -39,4 +39,93 @@ menu "CSI Node Configuration"
|
||||
help
|
||||
WiFi channel to listen on for CSI data.
|
||||
|
||||
config CSI_FILTER_MAC
|
||||
string "CSI source MAC filter (AA:BB:CC:DD:EE:FF or empty)"
|
||||
default ""
|
||||
help
|
||||
When set to a valid MAC address (e.g. "AA:BB:CC:DD:EE:FF"),
|
||||
only CSI frames from that transmitter are processed. All
|
||||
other frames are silently dropped. This prevents signal
|
||||
mixing in multi-AP environments.
|
||||
|
||||
Leave empty to accept CSI from all transmitters.
|
||||
|
||||
Can be overridden at runtime via NVS key "filter_mac"
|
||||
(6-byte blob) without reflashing.
|
||||
|
||||
endmenu
|
||||
|
||||
menu "Edge Intelligence (ADR-039)"
|
||||
|
||||
config EDGE_TIER
|
||||
int "Edge processing tier (0=off, 1=phase/stats, 2=vitals)"
|
||||
default 0
|
||||
range 0 3
|
||||
help
|
||||
Controls the level of on-device CSI processing:
|
||||
|
||||
0 = Disabled. Raw CSI frames are streamed unchanged (default).
|
||||
This preserves full backward compatibility.
|
||||
|
||||
1 = Phase sanitization + Welford statistics + top-K subcarrier
|
||||
selection + delta compression. Runs on Core 1.
|
||||
|
||||
2 = All of Tier 1, plus presence detection, vital signs
|
||||
extraction (breathing/heart rate), motion scoring,
|
||||
and fall detection. Sends vitals packets over UDP.
|
||||
|
||||
3 = Reserved for future ML inference tier.
|
||||
|
||||
config EDGE_PRESENCE_THRESH
|
||||
int "Presence detection threshold (0-65535)"
|
||||
default 50
|
||||
range 0 65535
|
||||
depends on EDGE_TIER > 0
|
||||
help
|
||||
Amplitude variance threshold for presence detection.
|
||||
Higher = less sensitive. Values below threshold/2 indicate
|
||||
empty room; values above threshold indicate motion.
|
||||
|
||||
config EDGE_FALL_THRESH
|
||||
int "Fall detection threshold (0-65535)"
|
||||
default 500
|
||||
range 0 65535
|
||||
depends on EDGE_TIER > 0
|
||||
help
|
||||
Minimum variance spike (scaled by 100) required for fall
|
||||
detection. The actual threshold is also gated by 5-sigma
|
||||
above the running mean, whichever is higher.
|
||||
|
||||
config EDGE_VITAL_WINDOW
|
||||
int "Vital signs window (frames, 60-600)"
|
||||
default 300
|
||||
range 60 600
|
||||
depends on EDGE_TIER > 0
|
||||
help
|
||||
Number of phase history samples used for vital signs
|
||||
estimation. At 20 Hz CSI rate, 300 frames = 15 seconds.
|
||||
Larger windows give more stable estimates but respond
|
||||
more slowly to changes.
|
||||
|
||||
config EDGE_VITAL_INTERVAL
|
||||
int "Vitals packet send interval (ms)"
|
||||
default 1000
|
||||
range 100 60000
|
||||
depends on EDGE_TIER > 0
|
||||
help
|
||||
How often to send a vitals summary packet over UDP.
|
||||
1000 ms (1 Hz) is recommended for real-time dashboards.
|
||||
Increase to reduce network bandwidth.
|
||||
|
||||
config EDGE_SUBK_COUNT
|
||||
int "Top-K subcarrier count (1-192)"
|
||||
default 32
|
||||
range 1 192
|
||||
depends on EDGE_TIER > 0
|
||||
help
|
||||
Number of highest-variance subcarriers to select for
|
||||
downstream processing (vital signs, delta compression).
|
||||
32 is a good default for HT20 (64 subcarriers).
|
||||
Increase for HT40 (128 subcarriers).
|
||||
|
||||
endmenu
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#include "csi_collector.h"
|
||||
#include "stream_sender.h"
|
||||
#include "edge_processing.h"
|
||||
|
||||
#include <string.h>
|
||||
#include "esp_log.h"
|
||||
@@ -26,6 +27,15 @@ static uint32_t s_sequence = 0;
|
||||
static uint32_t s_cb_count = 0;
|
||||
static uint32_t s_send_ok = 0;
|
||||
static uint32_t s_send_fail = 0;
|
||||
static uint32_t s_filtered = 0;
|
||||
|
||||
/* ---- MAC address filter (Issue #98) ---- */
|
||||
|
||||
/** When non-zero, only CSI from s_filter_mac is accepted. */
|
||||
static uint8_t s_filter_enabled = 0;
|
||||
|
||||
/** The accepted transmitter MAC address (6 bytes). */
|
||||
static uint8_t s_filter_mac[6] = {0};
|
||||
|
||||
/* ---- ADR-029: Channel-hop state ---- */
|
||||
|
||||
@@ -124,20 +134,58 @@ size_t csi_serialize_frame(const wifi_csi_info_t *info, uint8_t *buf, size_t buf
|
||||
return frame_size;
|
||||
}
|
||||
|
||||
void csi_collector_set_filter_mac(const uint8_t *mac)
|
||||
{
|
||||
if (mac == NULL) {
|
||||
s_filter_enabled = 0;
|
||||
memset(s_filter_mac, 0, 6);
|
||||
ESP_LOGI(TAG, "MAC filter disabled — accepting CSI from all transmitters");
|
||||
} else {
|
||||
memcpy(s_filter_mac, mac, 6);
|
||||
s_filter_enabled = 1;
|
||||
ESP_LOGI(TAG, "MAC filter enabled: only accepting %02X:%02X:%02X:%02X:%02X:%02X",
|
||||
mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
|
||||
}
|
||||
s_filtered = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* WiFi CSI callback — invoked by ESP-IDF when CSI data is available.
|
||||
*
|
||||
* When a MAC filter is active, frames from non-matching transmitters are
|
||||
* silently dropped to prevent signal mixing in multi-AP environments.
|
||||
*/
|
||||
static void wifi_csi_callback(void *ctx, wifi_csi_info_t *info)
|
||||
{
|
||||
(void)ctx;
|
||||
s_cb_count++;
|
||||
|
||||
if (s_cb_count <= 3 || (s_cb_count % 100) == 0) {
|
||||
ESP_LOGI(TAG, "CSI cb #%lu: len=%d rssi=%d ch=%d",
|
||||
(unsigned long)s_cb_count, info->len,
|
||||
info->rx_ctrl.rssi, info->rx_ctrl.channel);
|
||||
/* ---- MAC address filter (Issue #98) ---- */
|
||||
if (s_filter_enabled) {
|
||||
if (memcmp(info->mac, s_filter_mac, 6) != 0) {
|
||||
s_filtered++;
|
||||
if (s_filtered <= 3 || (s_filtered % 500) == 0) {
|
||||
ESP_LOGD(TAG, "Filtered CSI from %02X:%02X:%02X:%02X:%02X:%02X (dropped %lu)",
|
||||
info->mac[0], info->mac[1], info->mac[2],
|
||||
info->mac[3], info->mac[4], info->mac[5],
|
||||
(unsigned long)s_filtered);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (s_cb_count <= 3 || (s_cb_count % 100) == 0) {
|
||||
ESP_LOGI(TAG, "CSI cb #%lu: len=%d rssi=%d ch=%d mac=%02X:%02X:%02X:%02X:%02X:%02X",
|
||||
(unsigned long)s_cb_count, info->len,
|
||||
info->rx_ctrl.rssi, info->rx_ctrl.channel,
|
||||
info->mac[0], info->mac[1], info->mac[2],
|
||||
info->mac[3], info->mac[4], info->mac[5]);
|
||||
}
|
||||
|
||||
/* ADR-039: Feed edge processing ring buffer (lock-free, O(1)).
|
||||
* This is a no-op when edge_tier == 0. */
|
||||
edge_push_csi(info);
|
||||
|
||||
uint8_t frame_buf[CSI_MAX_FRAME_SIZE];
|
||||
size_t frame_len = csi_serialize_frame(info, frame_buf, sizeof(frame_buf));
|
||||
|
||||
|
||||
@@ -22,12 +22,28 @@
|
||||
/** Maximum number of channels in the hop table (ADR-029). */
|
||||
#define CSI_HOP_CHANNELS_MAX 6
|
||||
|
||||
/** Length of a MAC address in bytes. */
|
||||
#define CSI_MAC_LEN 6
|
||||
|
||||
/**
|
||||
* Initialize CSI collection.
|
||||
* Registers the WiFi CSI callback.
|
||||
*/
|
||||
void csi_collector_init(void);
|
||||
|
||||
/**
|
||||
* Set a MAC address filter for CSI collection.
|
||||
*
|
||||
* When set, only CSI frames from the specified transmitter MAC are processed;
|
||||
* all others are silently dropped. This prevents signal mixing in multi-AP
|
||||
* environments.
|
||||
*
|
||||
* Pass NULL to disable filtering (accept CSI from all transmitters).
|
||||
*
|
||||
* @param mac 6-byte MAC address to accept, or NULL to disable filtering.
|
||||
*/
|
||||
void csi_collector_set_filter_mac(const uint8_t *mac);
|
||||
|
||||
/**
|
||||
* Serialize CSI data into ADR-018 binary frame format.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,932 @@
|
||||
/**
|
||||
* @file edge_processing.c
|
||||
* @brief ADR-039 Edge Intelligence — on-device CSI processing.
|
||||
*
|
||||
* Implements a dual-core pipeline:
|
||||
* Core 0 (ISR context): wifi_csi_callback -> edge_push_csi() -> SPSC ring
|
||||
* Core 1 (edge_task): ring -> phase unwrap -> Welford -> top-K -> compress
|
||||
* -> (Tier 2) presence / vitals / fall
|
||||
*
|
||||
* Memory budget (static):
|
||||
* Ring buffer: 64 * ~400 B = ~25 KB
|
||||
* Tier 1 state: ~4 KB
|
||||
* Tier 2 state: ~2 KB
|
||||
* Scratch: ~2 KB
|
||||
* Total: ~33 KB on Core 1 stack + BSS
|
||||
*
|
||||
* All DSP uses the ESP32-S3 hardware single-precision FPU.
|
||||
*/
|
||||
|
||||
#include "edge_processing.h"
|
||||
#include "stream_sender.h"
|
||||
#include "nvs_config.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "freertos/semphr.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_timer.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
static const char *TAG = "edge_proc";
|
||||
|
||||
/* ================================================================== */
|
||||
/* Configuration (loaded from nvs_config at init) */
|
||||
/* ================================================================== */
|
||||
|
||||
static uint8_t s_tier = 0;
|
||||
static uint8_t s_node_id = 0;
|
||||
static uint16_t s_presence_thresh = 50;
|
||||
static uint16_t s_fall_thresh = 500;
|
||||
static uint16_t s_vital_window = 300;
|
||||
static uint16_t s_vital_interval_ms = 1000;
|
||||
static uint8_t s_subk_count = 32;
|
||||
|
||||
/* ================================================================== */
|
||||
/* Lock-free SPSC ring buffer */
|
||||
/* ================================================================== */
|
||||
|
||||
/**
|
||||
* Lock-free single-producer single-consumer ring buffer.
|
||||
*
|
||||
* Producer (Core 0, ISR-safe): increments s_ring_write after writing.
|
||||
* Consumer (Core 1, edge_task): increments s_ring_read after reading.
|
||||
* Both indices are volatile to prevent compiler reordering.
|
||||
* Ring capacity is EDGE_RING_SIZE - 1 to distinguish full from empty.
|
||||
*/
|
||||
static edge_csi_entry_t s_ring[EDGE_RING_SIZE];
|
||||
static volatile uint32_t s_ring_write = 0; /**< Next write position (producer). */
|
||||
static volatile uint32_t s_ring_read = 0; /**< Next read position (consumer). */
|
||||
|
||||
/** Notification semaphore: producer gives, consumer takes. */
|
||||
static SemaphoreHandle_t s_ring_sem = NULL;
|
||||
|
||||
/** Number of entries in the ring (lock-free). */
|
||||
static inline uint32_t ring_count(void)
|
||||
{
|
||||
uint32_t w = s_ring_write;
|
||||
uint32_t r = s_ring_read;
|
||||
return (w - r) & (EDGE_RING_SIZE - 1);
|
||||
}
|
||||
|
||||
/** Check if ring is full. */
|
||||
static inline bool ring_full(void)
|
||||
{
|
||||
return ring_count() >= (EDGE_RING_SIZE - 1);
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Processing state (Core 1 only — no synchronization needed) */
|
||||
/* ================================================================== */
|
||||
|
||||
static edge_tier1_state_t s_t1;
|
||||
static edge_tier2_state_t s_t2;
|
||||
|
||||
/** Scratch buffers for DSP on Core 1. */
|
||||
static float s_phase_buf[EDGE_MAX_SUBCARRIERS];
|
||||
static float s_amp_buf[EDGE_MAX_SUBCARRIERS];
|
||||
static float s_var_buf[EDGE_MAX_SUBCARRIERS];
|
||||
static uint8_t s_topk_idx[EDGE_MAX_SUBCARRIERS]; /* worst case k == n */
|
||||
|
||||
/** Compressed output buffer. */
|
||||
static uint8_t s_compress_buf[EDGE_MAX_IQ_LEN * 2];
|
||||
|
||||
/** Running RSSI accumulator (for vitals packet). */
|
||||
static float s_rssi_sum = 0.0f;
|
||||
static uint32_t s_rssi_count = 0;
|
||||
|
||||
/** Total frames processed and vitals sequence counter. */
|
||||
static uint32_t s_frame_count = 0;
|
||||
static uint16_t s_vitals_seq = 0;
|
||||
|
||||
/** Vitals packet send timer. */
|
||||
static esp_timer_handle_t s_vitals_timer = NULL;
|
||||
static volatile bool s_vitals_due = false;
|
||||
|
||||
/* ================================================================== */
|
||||
/* Biquad IIR filter for vital signs (Tier 2) */
|
||||
/* ================================================================== */
|
||||
|
||||
/**
|
||||
* Second-order IIR (biquad) filter coefficients.
|
||||
* Direct Form II Transposed.
|
||||
*/
|
||||
typedef struct {
|
||||
float b0, b1, b2;
|
||||
float a1, a2;
|
||||
float z1, z2; /**< State variables. */
|
||||
} biquad_t;
|
||||
|
||||
/**
|
||||
* Pre-computed biquad coefficients for 20 Hz sample rate.
|
||||
* These are bandpass filters designed with the bilinear transform.
|
||||
*
|
||||
* Breathing band: 0.1 - 0.5 Hz (6 - 30 BPM)
|
||||
* Heart rate band: 0.8 - 2.0 Hz (48 - 120 BPM)
|
||||
*
|
||||
* Coefficients were computed offline using scipy.signal.iirfilter
|
||||
* with Butterworth type, order=2, fs=20.
|
||||
*/
|
||||
|
||||
/** Breathing bandpass: 0.1-0.5 Hz at 20 Hz sample rate, 2nd order Butterworth. */
|
||||
static biquad_t s_bq_breath = {
|
||||
.b0 = 0.02008337f,
|
||||
.b1 = 0.0f,
|
||||
.b2 = -0.02008337f,
|
||||
.a1 = -1.93803473f,
|
||||
.a2 = 0.95983326f,
|
||||
.z1 = 0.0f, .z2 = 0.0f,
|
||||
};
|
||||
|
||||
/** Heart rate bandpass: 0.8-2.0 Hz at 20 Hz sample rate, 2nd order Butterworth. */
|
||||
static biquad_t s_bq_heart = {
|
||||
.b0 = 0.09853117f,
|
||||
.b1 = 0.0f,
|
||||
.b2 = -0.09853117f,
|
||||
.a1 = -1.53073372f,
|
||||
.a2 = 0.80293766f,
|
||||
.z1 = 0.0f, .z2 = 0.0f,
|
||||
};
|
||||
|
||||
/** Apply biquad filter to a single sample (Direct Form II Transposed). */
|
||||
static inline float biquad_process(biquad_t *bq, float x)
|
||||
{
|
||||
float y = bq->b0 * x + bq->z1;
|
||||
bq->z1 = bq->b1 * x - bq->a1 * y + bq->z2;
|
||||
bq->z2 = bq->b2 * x - bq->a2 * y;
|
||||
return y;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 1: Phase unwrap */
|
||||
/* ================================================================== */
|
||||
|
||||
void edge_phase_unwrap(const int8_t *iq, uint16_t n_sc,
|
||||
float *phase_out, float *phase_prev)
|
||||
{
|
||||
if (iq == NULL || phase_out == NULL || phase_prev == NULL || n_sc == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint16_t i = 0; i < n_sc; i++) {
|
||||
float ii = (float)iq[2 * i];
|
||||
float qq = (float)iq[2 * i + 1];
|
||||
|
||||
/* atan2 gives phase in [-pi, pi]. ESP32-S3 FPU handles this. */
|
||||
float phase = atan2f(qq, ii);
|
||||
|
||||
/* Unwrap: correct jumps > pi relative to previous phase. */
|
||||
float diff = phase - phase_prev[i];
|
||||
if (diff > (float)M_PI) {
|
||||
phase -= 2.0f * (float)M_PI;
|
||||
} else if (diff < -(float)M_PI) {
|
||||
phase += 2.0f * (float)M_PI;
|
||||
}
|
||||
|
||||
phase_out[i] = phase;
|
||||
phase_prev[i] = phase;
|
||||
}
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 1: Welford online statistics */
|
||||
/* ================================================================== */
|
||||
|
||||
void edge_welford_update(float value, float *mean, float *m2, uint32_t *count)
|
||||
{
|
||||
(*count)++;
|
||||
float delta = value - *mean;
|
||||
*mean += delta / (float)(*count);
|
||||
float delta2 = value - *mean;
|
||||
*m2 += delta * delta2;
|
||||
}
|
||||
|
||||
float edge_welford_variance(float m2, uint32_t count)
|
||||
{
|
||||
if (count < 2) {
|
||||
return 0.0f;
|
||||
}
|
||||
return m2 / (float)count;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 1: Top-K subcarrier selection (partial sort) */
|
||||
/* ================================================================== */
|
||||
|
||||
uint16_t edge_select_top_k(const float *variances, uint16_t n,
|
||||
uint8_t k, uint8_t *selected)
|
||||
{
|
||||
if (variances == NULL || selected == NULL || n == 0 || k == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Clamp k to available subcarriers and uint8_t max (255). */
|
||||
uint16_t actual_k = (k < n) ? k : n;
|
||||
if (actual_k > 255) {
|
||||
actual_k = 255;
|
||||
}
|
||||
|
||||
/*
|
||||
* Simple O(n*k) selection — good enough for n <= 192, k <= 64.
|
||||
* A full partial-sort (quickselect) is overkill at these sizes.
|
||||
*
|
||||
* We maintain a sorted (descending) list of the top-k seen so far.
|
||||
*/
|
||||
float top_val[255];
|
||||
uint8_t top_idx_local[255];
|
||||
|
||||
/* Initialize with -infinity. */
|
||||
for (uint16_t i = 0; i < actual_k; i++) {
|
||||
top_val[i] = -1.0e30f;
|
||||
top_idx_local[i] = 0;
|
||||
}
|
||||
|
||||
for (uint16_t i = 0; i < n; i++) {
|
||||
float v = variances[i];
|
||||
|
||||
/* Check if v belongs in the top-k list. */
|
||||
if (v > top_val[actual_k - 1]) {
|
||||
/* Find insertion point (linear scan of small array). */
|
||||
uint16_t pos = actual_k - 1;
|
||||
while (pos > 0 && v > top_val[pos - 1]) {
|
||||
top_val[pos] = top_val[pos - 1];
|
||||
top_idx_local[pos] = top_idx_local[pos - 1];
|
||||
pos--;
|
||||
}
|
||||
top_val[pos] = v;
|
||||
top_idx_local[pos] = (uint8_t)i;
|
||||
}
|
||||
}
|
||||
|
||||
for (uint16_t i = 0; i < actual_k; i++) {
|
||||
selected[i] = top_idx_local[i];
|
||||
}
|
||||
|
||||
return (uint16_t)actual_k;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 1: Delta compression (XOR + RLE) */
|
||||
/* ================================================================== */
|
||||
|
||||
uint16_t edge_delta_compress(const int8_t *cur, const int8_t *prev,
|
||||
uint16_t len, uint8_t *out, uint16_t out_len)
|
||||
{
|
||||
if (cur == NULL || prev == NULL || out == NULL || len == 0 || out_len < 2) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Algorithm:
|
||||
* 1. XOR current with previous frame (delta).
|
||||
* 2. RLE encode the delta: (count, value) pairs.
|
||||
* - count is stored as uint8_t (max 255 consecutive same-value bytes).
|
||||
* - This works well because CSI delta is often near-zero.
|
||||
*/
|
||||
uint16_t out_pos = 0;
|
||||
|
||||
uint16_t i = 0;
|
||||
while (i < len) {
|
||||
uint8_t delta_val = (uint8_t)(cur[i] ^ prev[i]);
|
||||
uint8_t run_len = 1;
|
||||
|
||||
/* Count consecutive identical delta values. */
|
||||
while (i + run_len < len && run_len < 255) {
|
||||
uint8_t next_delta = (uint8_t)(cur[i + run_len] ^ prev[i + run_len]);
|
||||
if (next_delta != delta_val) {
|
||||
break;
|
||||
}
|
||||
run_len++;
|
||||
}
|
||||
|
||||
/* Write (count, value) pair. */
|
||||
if (out_pos + 2 > out_len) {
|
||||
/* Output buffer full — compression failed to save space. */
|
||||
return 0;
|
||||
}
|
||||
out[out_pos++] = run_len;
|
||||
out[out_pos++] = delta_val;
|
||||
|
||||
i += run_len;
|
||||
}
|
||||
|
||||
return out_pos;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 2: Presence detection */
|
||||
/* ================================================================== */
|
||||
|
||||
void edge_update_presence(edge_tier2_state_t *state,
|
||||
const float *amplitudes, uint16_t n)
|
||||
{
|
||||
if (state == NULL || amplitudes == NULL || n == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compute total amplitude variance across all subcarriers.
|
||||
* High variance = motion. Low but nonzero = static presence.
|
||||
* Near-zero = empty room.
|
||||
*/
|
||||
float sum = 0.0f;
|
||||
float sum_sq = 0.0f;
|
||||
|
||||
for (uint16_t i = 0; i < n; i++) {
|
||||
sum += amplitudes[i];
|
||||
sum_sq += amplitudes[i] * amplitudes[i];
|
||||
}
|
||||
|
||||
float mean = sum / (float)n;
|
||||
float var = (sum_sq / (float)n) - (mean * mean);
|
||||
if (var < 0.0f) {
|
||||
var = 0.0f;
|
||||
}
|
||||
|
||||
/* Convert variance to an integer score. */
|
||||
float var_scaled = var * 10.0f;
|
||||
uint16_t var_int = (var_scaled > 65535.0f) ? 65535 : (uint16_t)var_scaled;
|
||||
|
||||
if (var_int < s_presence_thresh / 2) {
|
||||
state->presence = 0; /* Empty */
|
||||
state->motion_score = 0;
|
||||
} else if (var_int < s_presence_thresh) {
|
||||
state->presence = 1; /* Present (static) */
|
||||
state->motion_score = (uint8_t)(var_int * 128 / s_presence_thresh);
|
||||
} else {
|
||||
state->presence = 2; /* Moving */
|
||||
uint32_t score = (uint32_t)var_int * 255 / (s_presence_thresh * 10);
|
||||
state->motion_score = (score > 255) ? 255 : (uint8_t)score;
|
||||
}
|
||||
|
||||
/* Simple occupancy estimate: if motion on many subcarriers, likely > 1 person.
|
||||
* Count subcarriers with amplitude > 2 * mean as "active". */
|
||||
uint16_t active_count = 0;
|
||||
float thresh = mean * 2.0f;
|
||||
for (uint16_t i = 0; i < n; i++) {
|
||||
if (amplitudes[i] > thresh) {
|
||||
active_count++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Heuristic: every ~24 active subcarriers roughly corresponds to 1 person
|
||||
* in a typical 64-subcarrier environment. */
|
||||
uint8_t occ = (uint8_t)(active_count / 24);
|
||||
if (occ > 8) occ = 8;
|
||||
if (state->presence == 0) occ = 0;
|
||||
state->occupancy = occ;
|
||||
|
||||
/* Fall detection via variance spike. */
|
||||
state->fall_detected = edge_detect_fall(state, var) ? 1 : 0;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 2: Vital signs extraction */
|
||||
/* ================================================================== */
|
||||
|
||||
void edge_update_vitals(edge_tier2_state_t *state,
|
||||
const float *phases, uint16_t n)
|
||||
{
|
||||
if (state == NULL || phases == NULL || n == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Use the first subcarrier's phase (caller should pass the best
|
||||
* subcarrier selected by top-K). Push into circular buffer.
|
||||
*/
|
||||
float phase_val = phases[0];
|
||||
|
||||
state->phase_history[state->history_idx] = phase_val;
|
||||
state->history_idx = (state->history_idx + 1) % EDGE_PHASE_HISTORY_LEN;
|
||||
if (state->history_len < EDGE_PHASE_HISTORY_LEN) {
|
||||
state->history_len++;
|
||||
}
|
||||
|
||||
/*
|
||||
* Only estimate vitals when we have at least 3 seconds of data (60 samples at 20 Hz).
|
||||
* Full confidence requires the full window.
|
||||
*/
|
||||
if (state->history_len < 60) {
|
||||
state->breathing_bpm = 0.0f;
|
||||
state->heartrate_bpm = 0.0f;
|
||||
state->breathing_confidence = 0.0f;
|
||||
state->heartrate_confidence = 0.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Process the most recent samples through biquad bandpass filters.
|
||||
* We filter the latest sample and count zero-crossings over the buffer.
|
||||
*
|
||||
* For real-time use we filter each incoming sample and count peaks
|
||||
* over a sliding window.
|
||||
*/
|
||||
float breath_val = biquad_process(&s_bq_breath, phase_val);
|
||||
float heart_val = biquad_process(&s_bq_heart, phase_val);
|
||||
|
||||
/*
|
||||
* Peak counting: count positive zero-crossings over the history.
|
||||
* We re-scan the last 'window' samples each time for simplicity.
|
||||
* On ESP32-S3 at 20 Hz, scanning 300 floats is trivial (<0.1 ms).
|
||||
*/
|
||||
uint16_t window = state->history_len;
|
||||
if (window > s_vital_window) {
|
||||
window = s_vital_window;
|
||||
}
|
||||
|
||||
/* Apply bandpass to the entire window and count peaks.
|
||||
* We use temporary biquads for the full-window scan so as not to
|
||||
* disturb the streaming filter state. */
|
||||
biquad_t bq_br_tmp = s_bq_breath;
|
||||
biquad_t bq_hr_tmp = s_bq_heart;
|
||||
|
||||
/* Reset temporary filter state. */
|
||||
bq_br_tmp.z1 = 0.0f; bq_br_tmp.z2 = 0.0f;
|
||||
bq_hr_tmp.z1 = 0.0f; bq_hr_tmp.z2 = 0.0f;
|
||||
|
||||
uint16_t breath_crossings = 0;
|
||||
uint16_t heart_crossings = 0;
|
||||
float prev_br = 0.0f;
|
||||
float prev_hr = 0.0f;
|
||||
|
||||
/* Walk the circular buffer from oldest to newest. */
|
||||
uint16_t start_idx;
|
||||
if (state->history_len < EDGE_PHASE_HISTORY_LEN) {
|
||||
start_idx = 0;
|
||||
} else {
|
||||
start_idx = state->history_idx; /* Oldest entry. */
|
||||
}
|
||||
|
||||
for (uint16_t j = 0; j < window; j++) {
|
||||
uint16_t idx = (start_idx + j) % EDGE_PHASE_HISTORY_LEN;
|
||||
float sample = state->phase_history[idx];
|
||||
|
||||
float br = biquad_process(&bq_br_tmp, sample);
|
||||
float hr = biquad_process(&bq_hr_tmp, sample);
|
||||
|
||||
/* Positive zero crossing. */
|
||||
if (j > 0) {
|
||||
if (prev_br <= 0.0f && br > 0.0f) {
|
||||
breath_crossings++;
|
||||
}
|
||||
if (prev_hr <= 0.0f && hr > 0.0f) {
|
||||
heart_crossings++;
|
||||
}
|
||||
}
|
||||
|
||||
prev_br = br;
|
||||
prev_hr = hr;
|
||||
}
|
||||
|
||||
/* Convert crossings to BPM.
|
||||
* Each positive zero crossing corresponds to one cycle.
|
||||
* window samples at 20 Hz = window/20 seconds. */
|
||||
float duration_s = (float)window / 20.0f;
|
||||
if (duration_s > 0.0f) {
|
||||
state->breathing_bpm = (float)breath_crossings * 60.0f / duration_s;
|
||||
state->heartrate_bpm = (float)heart_crossings * 60.0f / duration_s;
|
||||
}
|
||||
|
||||
/* Clamp to physiological ranges. */
|
||||
if (state->breathing_bpm < 4.0f) state->breathing_bpm = 0.0f;
|
||||
if (state->breathing_bpm > 40.0f) state->breathing_bpm = 0.0f;
|
||||
if (state->heartrate_bpm < 40.0f) state->heartrate_bpm = 0.0f;
|
||||
if (state->heartrate_bpm > 150.0f) state->heartrate_bpm = 0.0f;
|
||||
|
||||
/* Confidence: based on signal amplitude relative to noise floor.
|
||||
* Higher filtered amplitude = more confident. */
|
||||
float br_amp = fabsf(breath_val);
|
||||
float hr_amp = fabsf(heart_val);
|
||||
|
||||
state->breathing_confidence = (br_amp > 0.5f) ? 1.0f : br_amp * 2.0f;
|
||||
state->heartrate_confidence = (hr_amp > 0.3f) ? 1.0f : hr_amp * 3.33f;
|
||||
|
||||
if (state->breathing_confidence > 1.0f) state->breathing_confidence = 1.0f;
|
||||
if (state->heartrate_confidence > 1.0f) state->heartrate_confidence = 1.0f;
|
||||
|
||||
/* If no presence detected, zero out vitals. */
|
||||
if (state->presence == 0) {
|
||||
state->breathing_bpm = 0.0f;
|
||||
state->heartrate_bpm = 0.0f;
|
||||
state->breathing_confidence = 0.0f;
|
||||
state->heartrate_confidence = 0.0f;
|
||||
}
|
||||
|
||||
(void)breath_val;
|
||||
(void)heart_val;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Tier 2: Fall detection */
|
||||
/* ================================================================== */
|
||||
|
||||
bool edge_detect_fall(edge_tier2_state_t *state, float current_variance)
|
||||
{
|
||||
if (state == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Store current variance in history ring. */
|
||||
state->variance_history[state->var_idx] = current_variance;
|
||||
state->var_idx = (state->var_idx + 1) % EDGE_VAR_HISTORY_LEN;
|
||||
|
||||
/*
|
||||
* Fall detection heuristic:
|
||||
* 1. Compute mean and stdev of variance history.
|
||||
* 2. If current variance > mean + 5*stdev, that is a "spike".
|
||||
* 3. If the last 3 entries after the spike show low variance
|
||||
* (< mean), declare a fall (spike + stillness).
|
||||
*
|
||||
* At 20 Hz and 20-entry history, this covers the last 1 second.
|
||||
* We check the last ~3 seconds by requiring the spike to have
|
||||
* happened recently and stillness to follow.
|
||||
*/
|
||||
float sum = 0.0f;
|
||||
float sum_sq = 0.0f;
|
||||
uint8_t valid = 0;
|
||||
|
||||
for (uint8_t i = 0; i < EDGE_VAR_HISTORY_LEN; i++) {
|
||||
float v = state->variance_history[i];
|
||||
if (v >= 0.0f) {
|
||||
sum += v;
|
||||
sum_sq += v * v;
|
||||
valid++;
|
||||
}
|
||||
}
|
||||
|
||||
if (valid < 10) {
|
||||
return false; /* Not enough history yet. */
|
||||
}
|
||||
|
||||
float mean = sum / (float)valid;
|
||||
float var_of_var = (sum_sq / (float)valid) - (mean * mean);
|
||||
if (var_of_var < 0.0f) var_of_var = 0.0f;
|
||||
float stdev = sqrtf(var_of_var);
|
||||
|
||||
float spike_thresh = mean + 5.0f * stdev;
|
||||
if (spike_thresh < (float)s_fall_thresh / 100.0f) {
|
||||
spike_thresh = (float)s_fall_thresh / 100.0f;
|
||||
}
|
||||
|
||||
/* Check if there was a recent spike (within last 10 entries)
|
||||
* followed by low values (last 3 entries). */
|
||||
bool saw_spike = false;
|
||||
for (uint8_t i = 0; i < 10; i++) {
|
||||
uint8_t idx = (state->var_idx + EDGE_VAR_HISTORY_LEN - 1 - i) % EDGE_VAR_HISTORY_LEN;
|
||||
if (state->variance_history[idx] > spike_thresh) {
|
||||
saw_spike = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!saw_spike) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Check if the last 3 entries show stillness. */
|
||||
uint8_t still_count = 0;
|
||||
for (uint8_t i = 0; i < 3; i++) {
|
||||
uint8_t idx = (state->var_idx + EDGE_VAR_HISTORY_LEN - 1 - i) % EDGE_VAR_HISTORY_LEN;
|
||||
if (state->variance_history[idx] < mean * 0.5f) {
|
||||
still_count++;
|
||||
}
|
||||
}
|
||||
|
||||
return (still_count >= 2);
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Vitals packet construction and send */
|
||||
/* ================================================================== */
|
||||
|
||||
static void send_vitals_packet(void)
|
||||
{
|
||||
edge_vitals_packet_t pkt;
|
||||
memset(&pkt, 0, sizeof(pkt));
|
||||
|
||||
pkt.magic = EDGE_VITALS_MAGIC;
|
||||
pkt.node_id = s_node_id;
|
||||
pkt.pkt_type = EDGE_PKT_TYPE_VITALS;
|
||||
pkt.sequence = s_vitals_seq++;
|
||||
|
||||
pkt.presence = s_t2.presence;
|
||||
pkt.motion_score = s_t2.motion_score;
|
||||
pkt.occupancy = s_t2.occupancy;
|
||||
pkt.coherence_gate = 0; /* Reserved. */
|
||||
|
||||
pkt.breathing_bpm_x100 = (uint16_t)(s_t2.breathing_bpm * 100.0f);
|
||||
pkt.heartrate_bpm_x100 = (uint16_t)(s_t2.heartrate_bpm * 100.0f);
|
||||
pkt.breathing_conf = (uint16_t)(s_t2.breathing_confidence * 10000.0f);
|
||||
pkt.heartrate_conf = (uint16_t)(s_t2.heartrate_confidence * 10000.0f);
|
||||
|
||||
pkt.fall_detected = s_t2.fall_detected;
|
||||
pkt.anomaly_flags = 0;
|
||||
|
||||
if (s_rssi_count > 0) {
|
||||
pkt.rssi_mean = (int16_t)(s_rssi_sum / (float)s_rssi_count);
|
||||
} else {
|
||||
pkt.rssi_mean = 0;
|
||||
}
|
||||
|
||||
pkt.csi_count = s_frame_count;
|
||||
pkt.uptime_s = (uint32_t)(esp_timer_get_time() / 1000000ULL);
|
||||
|
||||
/* Send via existing UDP sender. */
|
||||
stream_sender_send((const uint8_t *)&pkt, sizeof(pkt));
|
||||
|
||||
ESP_LOGD(TAG, "Vitals pkt #%u: presence=%u motion=%u br=%.1f hr=%.1f",
|
||||
pkt.sequence, pkt.presence, pkt.motion_score,
|
||||
s_t2.breathing_bpm, s_t2.heartrate_bpm);
|
||||
}
|
||||
|
||||
/**
|
||||
* Timer callback for periodic vitals packet transmission.
|
||||
* Sets a flag that the edge task checks — avoids doing work in timer context.
|
||||
*/
|
||||
static void vitals_timer_cb(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
s_vitals_due = true;
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Edge processing task (pinned to Core 1) */
|
||||
/* ================================================================== */
|
||||
|
||||
/**
|
||||
* Process a single CSI frame through the Tier 1 pipeline.
|
||||
*/
|
||||
static void process_tier1(const edge_csi_entry_t *entry)
|
||||
{
|
||||
uint16_t n_sc = entry->iq_len / 2;
|
||||
if (n_sc == 0 || n_sc > EDGE_MAX_SUBCARRIERS) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Phase unwrap. */
|
||||
edge_phase_unwrap(entry->iq_data, n_sc, s_phase_buf, s_t1.phase_prev);
|
||||
|
||||
/* Compute amplitudes and update Welford stats. */
|
||||
for (uint16_t i = 0; i < n_sc; i++) {
|
||||
float ii = (float)entry->iq_data[2 * i];
|
||||
float qq = (float)entry->iq_data[2 * i + 1];
|
||||
s_amp_buf[i] = sqrtf(ii * ii + qq * qq);
|
||||
|
||||
edge_welford_update(s_amp_buf[i],
|
||||
&s_t1.amp_mean[i],
|
||||
&s_t1.amp_m2[i],
|
||||
&s_t1.amp_count);
|
||||
}
|
||||
|
||||
/* Note: amp_count is shared across subcarriers (they all advance together).
|
||||
* This is correct because we call Welford once per subcarrier per frame,
|
||||
* and all subcarriers receive the same frame count. The count represents
|
||||
* the number of frames seen, not per-subcarrier counts. */
|
||||
|
||||
/* Compute per-subcarrier variance for top-K selection. */
|
||||
for (uint16_t i = 0; i < n_sc; i++) {
|
||||
s_var_buf[i] = edge_welford_variance(s_t1.amp_m2[i], s_t1.amp_count);
|
||||
}
|
||||
|
||||
/* Select top-K highest-variance subcarriers. */
|
||||
uint8_t k = s_subk_count;
|
||||
if (k > n_sc) k = (uint8_t)n_sc;
|
||||
uint16_t selected = edge_select_top_k(s_var_buf, n_sc, k, s_topk_idx);
|
||||
(void)selected; /* Available for downstream use. */
|
||||
|
||||
/* Delta compress if we have a previous frame. */
|
||||
if (s_t1.has_prev) {
|
||||
uint16_t compressed_len = edge_delta_compress(
|
||||
entry->iq_data, s_t1.prev_iq,
|
||||
entry->iq_len, s_compress_buf, sizeof(s_compress_buf));
|
||||
(void)compressed_len; /* Will be used for Tier 3 compressed streaming. */
|
||||
}
|
||||
|
||||
/* Store current frame as previous for next delta. */
|
||||
memcpy(s_t1.prev_iq, entry->iq_data, entry->iq_len);
|
||||
s_t1.has_prev = true;
|
||||
|
||||
/* Accumulate RSSI for vitals packet. */
|
||||
s_rssi_sum += (float)entry->rssi;
|
||||
s_rssi_count++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single CSI frame through the Tier 2 pipeline.
|
||||
* Requires Tier 1 to have run first (uses s_phase_buf, s_amp_buf).
|
||||
*/
|
||||
static void process_tier2(const edge_csi_entry_t *entry)
|
||||
{
|
||||
uint16_t n_sc = entry->iq_len / 2;
|
||||
if (n_sc == 0 || n_sc > EDGE_MAX_SUBCARRIERS) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Presence and motion detection from amplitudes. */
|
||||
edge_update_presence(&s_t2, s_amp_buf, n_sc);
|
||||
|
||||
/* Vital signs from the best subcarrier's phase.
|
||||
* Use the first entry in the top-K list (highest variance). */
|
||||
if (s_subk_count > 0 && n_sc > 0) {
|
||||
uint8_t best_sc = s_topk_idx[0];
|
||||
if (best_sc < n_sc) {
|
||||
float best_phase = s_phase_buf[best_sc];
|
||||
edge_update_vitals(&s_t2, &best_phase, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main edge processing task — runs on Core 1.
|
||||
*
|
||||
* Blocks on the ring buffer semaphore, then drains all available entries.
|
||||
*/
|
||||
static void edge_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
|
||||
ESP_LOGI(TAG, "Edge task started on core %d (tier=%u)",
|
||||
xPortGetCoreID(), (unsigned)s_tier);
|
||||
|
||||
while (1) {
|
||||
/* Block until producer signals new data (or timeout for vitals). */
|
||||
xSemaphoreTake(s_ring_sem, pdMS_TO_TICKS(100));
|
||||
|
||||
/* Drain all available ring entries. */
|
||||
while (s_ring_read != s_ring_write) {
|
||||
uint32_t idx = s_ring_read & (EDGE_RING_SIZE - 1);
|
||||
const edge_csi_entry_t *entry = &s_ring[idx];
|
||||
|
||||
/* Tier 1: always run if tier >= 1. */
|
||||
process_tier1(entry);
|
||||
s_frame_count++;
|
||||
|
||||
/* Tier 2: run if tier >= 2. */
|
||||
if (s_tier >= 2) {
|
||||
process_tier2(entry);
|
||||
}
|
||||
|
||||
/* Advance read pointer (memory barrier via volatile). */
|
||||
s_ring_read++;
|
||||
}
|
||||
|
||||
/* Send vitals packet at configured interval (Tier 2). */
|
||||
if (s_tier >= 2 && s_vitals_due) {
|
||||
s_vitals_due = false;
|
||||
send_vitals_packet();
|
||||
|
||||
/* Reset RSSI accumulator. */
|
||||
s_rssi_sum = 0.0f;
|
||||
s_rssi_count = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Public API */
|
||||
/* ================================================================== */
|
||||
|
||||
void edge_push_csi(const wifi_csi_info_t *info)
|
||||
{
|
||||
if (s_tier == 0 || info == NULL || info->buf == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Check ring space. */
|
||||
if (ring_full()) {
|
||||
/* Drop frame — producer must never block in ISR context. */
|
||||
static uint32_t s_drop_count = 0;
|
||||
s_drop_count++;
|
||||
if (s_drop_count <= 3 || (s_drop_count % 1000) == 0) {
|
||||
ESP_LOGW(TAG, "Ring full, frame dropped (total=%lu)",
|
||||
(unsigned long)s_drop_count);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/* Write entry at current write position. */
|
||||
uint32_t idx = s_ring_write & (EDGE_RING_SIZE - 1);
|
||||
edge_csi_entry_t *entry = &s_ring[idx];
|
||||
|
||||
uint16_t iq_len = (uint16_t)info->len;
|
||||
if (iq_len > EDGE_MAX_IQ_LEN) {
|
||||
iq_len = EDGE_MAX_IQ_LEN;
|
||||
}
|
||||
|
||||
memcpy(entry->iq_data, info->buf, iq_len);
|
||||
entry->iq_len = iq_len;
|
||||
entry->rssi = (int8_t)info->rx_ctrl.rssi;
|
||||
entry->noise_floor = (int8_t)info->rx_ctrl.noise_floor;
|
||||
entry->channel = (uint8_t)info->rx_ctrl.channel;
|
||||
memcpy(entry->tx_mac, info->mac, 6);
|
||||
entry->timestamp_ms = (uint32_t)(esp_timer_get_time() / 1000ULL);
|
||||
|
||||
/* Advance write pointer (volatile write acts as release fence). */
|
||||
s_ring_write++;
|
||||
|
||||
/* Wake the consumer task. */
|
||||
if (s_ring_sem != NULL) {
|
||||
xSemaphoreGiveFromISR(s_ring_sem, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t edge_get_tier(void)
|
||||
{
|
||||
return s_tier;
|
||||
}
|
||||
|
||||
void edge_processing_init(uint8_t tier)
|
||||
{
|
||||
s_tier = tier;
|
||||
|
||||
if (tier == 0) {
|
||||
ESP_LOGI(TAG, "Edge processing disabled (tier=0)");
|
||||
return;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Initializing edge processing tier=%u", (unsigned)tier);
|
||||
|
||||
/* Read configuration from the extern nvs_config (already loaded in main). */
|
||||
/* These are set via the Kconfig / NVS defaults applied in nvs_config_load. */
|
||||
extern nvs_config_t s_cfg; /* Defined in main.c */
|
||||
s_node_id = s_cfg.node_id;
|
||||
s_presence_thresh = s_cfg.presence_thresh;
|
||||
s_fall_thresh = s_cfg.fall_thresh;
|
||||
s_vital_window = s_cfg.vital_window;
|
||||
s_vital_interval_ms = s_cfg.vital_interval_ms;
|
||||
s_subk_count = s_cfg.subk_count;
|
||||
|
||||
ESP_LOGI(TAG, " presence_thresh=%u fall_thresh=%u vital_window=%u interval=%ums subk=%u",
|
||||
s_presence_thresh, s_fall_thresh, s_vital_window,
|
||||
s_vital_interval_ms, s_subk_count);
|
||||
|
||||
/* Initialize state. */
|
||||
memset(&s_t1, 0, sizeof(s_t1));
|
||||
memset(&s_t2, 0, sizeof(s_t2));
|
||||
s_ring_write = 0;
|
||||
s_ring_read = 0;
|
||||
s_frame_count = 0;
|
||||
s_vitals_seq = 0;
|
||||
s_rssi_sum = 0.0f;
|
||||
s_rssi_count = 0;
|
||||
s_vitals_due = false;
|
||||
|
||||
/* Reset biquad filter state. */
|
||||
s_bq_breath.z1 = 0.0f; s_bq_breath.z2 = 0.0f;
|
||||
s_bq_heart.z1 = 0.0f; s_bq_heart.z2 = 0.0f;
|
||||
|
||||
/* Create notification semaphore (binary). */
|
||||
s_ring_sem = xSemaphoreCreateBinary();
|
||||
if (s_ring_sem == NULL) {
|
||||
ESP_LOGE(TAG, "Failed to create ring semaphore");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Create edge processing task pinned to Core 1.
|
||||
* Stack size: 8 KB is sufficient for our static-alloc pipeline. */
|
||||
BaseType_t ret = xTaskCreatePinnedToCore(
|
||||
edge_task,
|
||||
"edge_task",
|
||||
8192, /* Stack size in bytes. */
|
||||
NULL,
|
||||
5, /* Priority (above idle, below WiFi). */
|
||||
NULL,
|
||||
1 /* Core 1. */
|
||||
);
|
||||
|
||||
if (ret != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to create edge task");
|
||||
return;
|
||||
}
|
||||
|
||||
/* For Tier 2: start the periodic vitals packet timer. */
|
||||
if (tier >= 2 && s_vital_interval_ms > 0) {
|
||||
esp_timer_create_args_t timer_args = {
|
||||
.callback = vitals_timer_cb,
|
||||
.arg = NULL,
|
||||
.name = "vitals_tx",
|
||||
};
|
||||
|
||||
esp_err_t err = esp_timer_create(&timer_args, &s_vitals_timer);
|
||||
if (err == ESP_OK) {
|
||||
err = esp_timer_start_periodic(s_vitals_timer,
|
||||
(uint64_t)s_vital_interval_ms * 1000);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Failed to start vitals timer: %s",
|
||||
esp_err_to_name(err));
|
||||
} else {
|
||||
ESP_LOGI(TAG, "Vitals timer started: interval=%u ms",
|
||||
s_vital_interval_ms);
|
||||
}
|
||||
} else {
|
||||
ESP_LOGE(TAG, "Failed to create vitals timer: %s",
|
||||
esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Edge processing initialized (tier=%u, ring=%u slots)",
|
||||
(unsigned)tier, (unsigned)EDGE_RING_SIZE);
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* @file edge_processing.h
|
||||
* @brief ADR-039 Edge Intelligence — on-device CSI processing.
|
||||
*
|
||||
* Phase 1 + Tier 1: Phase sanitization, Welford running statistics,
|
||||
* subcarrier selection, and delta compression on the ESP32-S3.
|
||||
*
|
||||
* Tier 2 (optional): Presence detection, vital signs extraction,
|
||||
* motion scoring, and fall detection.
|
||||
*
|
||||
* Design:
|
||||
* - Lock-free SPSC ring buffer (Core 0 produces, Core 1 consumes).
|
||||
* - FreeRTOS task pinned to Core 1 for DSP.
|
||||
* - All static allocation, no malloc in hot path.
|
||||
* - edge_tier=0 disables edge processing (existing behavior preserved).
|
||||
*/
|
||||
|
||||
#ifndef EDGE_PROCESSING_H
|
||||
#define EDGE_PROCESSING_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include "esp_wifi_types.h"
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Ring buffer configuration */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** Ring buffer capacity (must be power of 2). */
|
||||
#define EDGE_RING_SIZE 64
|
||||
|
||||
/** Maximum I/Q data length per CSI frame (4 antennas * 256 subcarriers * 2). */
|
||||
#define EDGE_MAX_IQ_LEN 384
|
||||
|
||||
/** Ring buffer entry — copied from the CSI callback on Core 0. */
|
||||
typedef struct {
|
||||
int8_t iq_data[EDGE_MAX_IQ_LEN];
|
||||
uint16_t iq_len;
|
||||
int8_t rssi;
|
||||
int8_t noise_floor;
|
||||
uint8_t channel;
|
||||
uint8_t tx_mac[6];
|
||||
uint32_t timestamp_ms;
|
||||
} edge_csi_entry_t;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Tier 1: Phase sanitization and subcarrier selection */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** Maximum subcarriers we track (HT40 = 128 subcarriers, with margin). */
|
||||
#define EDGE_MAX_SUBCARRIERS 192
|
||||
|
||||
/** Per-subcarrier running statistics for phase unwrap and Welford. */
|
||||
typedef struct {
|
||||
float phase_prev[EDGE_MAX_SUBCARRIERS]; /**< Previous phase for unwrap. */
|
||||
float amp_mean[EDGE_MAX_SUBCARRIERS]; /**< Welford running mean of amplitude. */
|
||||
float amp_m2[EDGE_MAX_SUBCARRIERS]; /**< Welford M2 accumulator. */
|
||||
uint32_t amp_count; /**< Total sample count. */
|
||||
int8_t prev_iq[EDGE_MAX_IQ_LEN]; /**< Previous I/Q frame for delta compression. */
|
||||
bool has_prev; /**< True after first frame received. */
|
||||
} edge_tier1_state_t;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Tier 2: Vital signs and presence detection */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** Phase history depth: 15 seconds at 20 Hz. */
|
||||
#define EDGE_PHASE_HISTORY_LEN 300
|
||||
|
||||
/** Variance history depth for fall detection. */
|
||||
#define EDGE_VAR_HISTORY_LEN 20
|
||||
|
||||
typedef struct {
|
||||
float phase_history[EDGE_PHASE_HISTORY_LEN]; /**< Ring buffer of phases for vital signs. */
|
||||
uint16_t history_len; /**< Number of valid entries. */
|
||||
uint16_t history_idx; /**< Current write index. */
|
||||
float breathing_bpm; /**< Estimated breathing rate (BPM). */
|
||||
float heartrate_bpm; /**< Estimated heart rate (BPM). */
|
||||
float breathing_confidence; /**< Confidence [0..1]. */
|
||||
float heartrate_confidence; /**< Confidence [0..1]. */
|
||||
uint8_t presence; /**< 0=empty, 1=present, 2=moving. */
|
||||
uint8_t motion_score; /**< 0-255 motion intensity. */
|
||||
uint8_t occupancy; /**< Estimated occupant count (0-8). */
|
||||
uint8_t fall_detected; /**< 1 if fall detected in current window. */
|
||||
float variance_history[EDGE_VAR_HISTORY_LEN]; /**< Recent variance for fall detection. */
|
||||
uint8_t var_idx; /**< Write index into variance_history. */
|
||||
} edge_tier2_state_t;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Vitals UDP packet (Tier 2, Magic 0xC5110002) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** ADR-039 vitals packet magic number. */
|
||||
#define EDGE_VITALS_MAGIC 0xC5110002
|
||||
|
||||
/** Vitals packet type identifier. */
|
||||
#define EDGE_PKT_TYPE_VITALS 0x02
|
||||
|
||||
/**
|
||||
* Vitals packet — 32 bytes, sent at 1 Hz over UDP.
|
||||
* Compatible with the ADR-018 aggregator (different magic discriminates).
|
||||
*/
|
||||
typedef struct __attribute__((packed)) {
|
||||
uint32_t magic; /**< 0xC5110002 */
|
||||
uint8_t node_id;
|
||||
uint8_t pkt_type; /**< EDGE_PKT_TYPE_VITALS */
|
||||
uint16_t sequence;
|
||||
uint8_t presence; /**< 0=empty, 1=present, 2=moving */
|
||||
uint8_t motion_score; /**< 0-255 */
|
||||
uint8_t occupancy; /**< 0-8 */
|
||||
uint8_t coherence_gate; /**< Reserved for future use */
|
||||
uint16_t breathing_bpm_x100; /**< BPM * 100 */
|
||||
uint16_t heartrate_bpm_x100; /**< BPM * 100 */
|
||||
uint16_t breathing_conf; /**< Confidence * 10000 */
|
||||
uint16_t heartrate_conf; /**< Confidence * 10000 */
|
||||
uint8_t fall_detected;
|
||||
uint8_t anomaly_flags; /**< Reserved */
|
||||
int16_t rssi_mean; /**< Averaged RSSI */
|
||||
uint32_t csi_count; /**< Total frames processed */
|
||||
uint32_t uptime_s; /**< Seconds since boot */
|
||||
} edge_vitals_packet_t;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Public API */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Initialize edge processing.
|
||||
*
|
||||
* @param tier Processing tier (0=disabled, 1=phase/stats/compress, 2=vitals).
|
||||
* Tier 0 is a no-op for backward compatibility.
|
||||
*/
|
||||
void edge_processing_init(uint8_t tier);
|
||||
|
||||
/**
|
||||
* Push a CSI frame into the edge processing ring buffer.
|
||||
* Called from the CSI callback on Core 0. Lock-free, O(1).
|
||||
*
|
||||
* @param info WiFi CSI info from the ESP-IDF callback.
|
||||
*/
|
||||
void edge_push_csi(const wifi_csi_info_t *info);
|
||||
|
||||
/**
|
||||
* Get the currently configured edge processing tier.
|
||||
*
|
||||
* @return Tier (0-3).
|
||||
*/
|
||||
uint8_t edge_get_tier(void);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Tier 1 pure functions (suitable for unit testing) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Phase unwrap: extract phase from I/Q data with 2pi correction.
|
||||
*
|
||||
* @param iq Raw I/Q pairs (I0, Q0, I1, Q1, ...).
|
||||
* @param n_sc Number of subcarriers.
|
||||
* @param phase_out Output phases in radians (size >= n_sc).
|
||||
* @param phase_prev Previous phases for unwrap (updated in place).
|
||||
*/
|
||||
void edge_phase_unwrap(const int8_t *iq, uint16_t n_sc,
|
||||
float *phase_out, float *phase_prev);
|
||||
|
||||
/**
|
||||
* Welford online algorithm — update running mean and M2.
|
||||
*
|
||||
* @param value New sample value.
|
||||
* @param mean Running mean (updated in place).
|
||||
* @param m2 Running M2 (updated in place).
|
||||
* @param count Sample count (updated in place).
|
||||
*/
|
||||
void edge_welford_update(float value, float *mean, float *m2, uint32_t *count);
|
||||
|
||||
/**
|
||||
* Compute variance from Welford M2 accumulator.
|
||||
*
|
||||
* @param m2 M2 value.
|
||||
* @param count Sample count (must be >= 2).
|
||||
* @return Population variance, or 0 if count < 2.
|
||||
*/
|
||||
float edge_welford_variance(float m2, uint32_t count);
|
||||
|
||||
/**
|
||||
* Select top-K subcarriers by variance (partial sort).
|
||||
*
|
||||
* @param variances Variance array (size n).
|
||||
* @param n Total subcarrier count.
|
||||
* @param k Number to select.
|
||||
* @param selected Output array of selected indices (size >= k).
|
||||
* @return Actual number selected (min(k, n)).
|
||||
*/
|
||||
uint16_t edge_select_top_k(const float *variances, uint16_t n,
|
||||
uint8_t k, uint8_t *selected);
|
||||
|
||||
/**
|
||||
* Delta compress I/Q data: XOR with previous frame, then simple RLE.
|
||||
*
|
||||
* @param cur Current I/Q data.
|
||||
* @param prev Previous I/Q data.
|
||||
* @param len Length of I/Q data in bytes.
|
||||
* @param out Output buffer for compressed data.
|
||||
* @param out_len Size of output buffer.
|
||||
* @return Number of bytes written to out, or 0 if compression failed.
|
||||
*/
|
||||
uint16_t edge_delta_compress(const int8_t *cur, const int8_t *prev,
|
||||
uint16_t len, uint8_t *out, uint16_t out_len);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Tier 2 functions */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Update presence / motion detection from amplitude data.
|
||||
*
|
||||
* @param state Tier 2 state (updated in place).
|
||||
* @param amplitudes Amplitude array for current frame.
|
||||
* @param n Number of subcarriers.
|
||||
*/
|
||||
void edge_update_presence(edge_tier2_state_t *state,
|
||||
const float *amplitudes, uint16_t n);
|
||||
|
||||
/**
|
||||
* Update vital signs estimation from phase data.
|
||||
*
|
||||
* @param state Tier 2 state (updated in place).
|
||||
* @param phases Phase array for current frame.
|
||||
* @param n Number of subcarriers.
|
||||
*/
|
||||
void edge_update_vitals(edge_tier2_state_t *state,
|
||||
const float *phases, uint16_t n);
|
||||
|
||||
/**
|
||||
* Check for fall event: variance spike >5 sigma followed by stillness.
|
||||
*
|
||||
* @param state Tier 2 state (updated in place).
|
||||
* @param current_variance Current frame variance.
|
||||
* @return true if a fall is detected.
|
||||
*/
|
||||
bool edge_detect_fall(edge_tier2_state_t *state, float current_variance);
|
||||
|
||||
#endif /* EDGE_PROCESSING_H */
|
||||
@@ -21,11 +21,13 @@
|
||||
#include "csi_collector.h"
|
||||
#include "stream_sender.h"
|
||||
#include "nvs_config.h"
|
||||
#include "edge_processing.h"
|
||||
|
||||
static const char *TAG = "main";
|
||||
|
||||
/* Runtime configuration (loaded from NVS or Kconfig defaults). */
|
||||
static nvs_config_t s_cfg;
|
||||
/* Runtime configuration (loaded from NVS or Kconfig defaults).
|
||||
* Non-static so edge_processing.c can access it via extern. */
|
||||
nvs_config_t s_cfg;
|
||||
|
||||
/* Event group bits */
|
||||
#define WIFI_CONNECTED_BIT BIT0
|
||||
@@ -134,8 +136,18 @@ void app_main(void)
|
||||
/* Initialize CSI collection */
|
||||
csi_collector_init();
|
||||
|
||||
ESP_LOGI(TAG, "CSI streaming active → %s:%d",
|
||||
s_cfg.target_ip, s_cfg.target_port);
|
||||
/* Apply MAC address filter if configured (Issue #98) */
|
||||
if (s_cfg.filter_mac_enabled) {
|
||||
csi_collector_set_filter_mac(s_cfg.filter_mac);
|
||||
} else {
|
||||
ESP_LOGI(TAG, "No MAC filter — accepting CSI from all transmitters");
|
||||
}
|
||||
|
||||
/* ADR-039: Initialize edge processing (tier 0 = no-op for backward compat) */
|
||||
edge_processing_init(s_cfg.edge_tier);
|
||||
|
||||
ESP_LOGI(TAG, "CSI streaming active → %s:%d (edge_tier=%u)",
|
||||
s_cfg.target_ip, s_cfg.target_port, (unsigned)s_cfg.edge_tier);
|
||||
|
||||
/* Main loop — keep alive */
|
||||
while (1) {
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "nvs_config.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include "esp_log.h"
|
||||
#include "nvs_flash.h"
|
||||
#include "nvs.h"
|
||||
@@ -51,6 +52,66 @@ void nvs_config_load(nvs_config_t *cfg)
|
||||
cfg->tdm_slot_index = 0;
|
||||
cfg->tdm_node_count = 1;
|
||||
|
||||
/* MAC filter: default disabled (all zeros) */
|
||||
memset(cfg->filter_mac, 0, 6);
|
||||
cfg->filter_mac_enabled = 0;
|
||||
|
||||
/* ADR-039: Edge processing defaults */
|
||||
#ifdef CONFIG_EDGE_TIER
|
||||
cfg->edge_tier = (uint8_t)CONFIG_EDGE_TIER;
|
||||
#else
|
||||
cfg->edge_tier = 0;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_EDGE_PRESENCE_THRESH
|
||||
cfg->presence_thresh = (uint16_t)CONFIG_EDGE_PRESENCE_THRESH;
|
||||
#else
|
||||
cfg->presence_thresh = 50;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_EDGE_FALL_THRESH
|
||||
cfg->fall_thresh = (uint16_t)CONFIG_EDGE_FALL_THRESH;
|
||||
#else
|
||||
cfg->fall_thresh = 500;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_EDGE_VITAL_WINDOW
|
||||
cfg->vital_window = (uint16_t)CONFIG_EDGE_VITAL_WINDOW;
|
||||
#else
|
||||
cfg->vital_window = 300;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_EDGE_VITAL_INTERVAL
|
||||
cfg->vital_interval_ms = (uint16_t)CONFIG_EDGE_VITAL_INTERVAL;
|
||||
#else
|
||||
cfg->vital_interval_ms = 1000;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_EDGE_SUBK_COUNT
|
||||
cfg->subk_count = (uint8_t)CONFIG_EDGE_SUBK_COUNT;
|
||||
#else
|
||||
cfg->subk_count = 32;
|
||||
#endif
|
||||
|
||||
/* Parse compile-time Kconfig MAC filter if set (format: "AA:BB:CC:DD:EE:FF") */
|
||||
#ifdef CONFIG_CSI_FILTER_MAC
|
||||
{
|
||||
const char *mac_str = CONFIG_CSI_FILTER_MAC;
|
||||
unsigned int m[6];
|
||||
if (mac_str[0] != '\0' &&
|
||||
sscanf(mac_str, "%x:%x:%x:%x:%x:%x",
|
||||
&m[0], &m[1], &m[2], &m[3], &m[4], &m[5]) == 6) {
|
||||
for (int i = 0; i < 6; i++) {
|
||||
cfg->filter_mac[i] = (uint8_t)m[i];
|
||||
}
|
||||
cfg->filter_mac_enabled = 1;
|
||||
ESP_LOGI(TAG, "Kconfig MAC filter: %02X:%02X:%02X:%02X:%02X:%02X",
|
||||
cfg->filter_mac[0], cfg->filter_mac[1], cfg->filter_mac[2],
|
||||
cfg->filter_mac[3], cfg->filter_mac[4], cfg->filter_mac[5]);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Try to override from NVS */
|
||||
nvs_handle_t handle;
|
||||
esp_err_t err = nvs_open("csi_cfg", NVS_READONLY, &handle);
|
||||
@@ -152,6 +213,27 @@ void nvs_config_load(nvs_config_t *cfg)
|
||||
}
|
||||
}
|
||||
|
||||
/* MAC filter (stored as a 6-byte blob in NVS key "filter_mac") */
|
||||
uint8_t mac_blob[6];
|
||||
size_t mac_len = 6;
|
||||
if (nvs_get_blob(handle, "filter_mac", mac_blob, &mac_len) == ESP_OK && mac_len == 6) {
|
||||
/* Check it's not all zeros (which would mean "no filter") */
|
||||
uint8_t is_zero = 1;
|
||||
for (int i = 0; i < 6; i++) {
|
||||
if (mac_blob[i] != 0) { is_zero = 0; break; }
|
||||
}
|
||||
if (!is_zero) {
|
||||
memcpy(cfg->filter_mac, mac_blob, 6);
|
||||
cfg->filter_mac_enabled = 1;
|
||||
ESP_LOGI(TAG, "NVS override: filter_mac=%02X:%02X:%02X:%02X:%02X:%02X",
|
||||
mac_blob[0], mac_blob[1], mac_blob[2],
|
||||
mac_blob[3], mac_blob[4], mac_blob[5]);
|
||||
} else {
|
||||
cfg->filter_mac_enabled = 0;
|
||||
ESP_LOGI(TAG, "NVS override: filter_mac disabled (all zeros)");
|
||||
}
|
||||
}
|
||||
|
||||
/* Validate tdm_slot_index < tdm_node_count */
|
||||
if (cfg->tdm_slot_index >= cfg->tdm_node_count) {
|
||||
ESP_LOGW(TAG, "tdm_slot_index=%u >= tdm_node_count=%u, clamping to 0",
|
||||
@@ -159,5 +241,62 @@ void nvs_config_load(nvs_config_t *cfg)
|
||||
cfg->tdm_slot_index = 0;
|
||||
}
|
||||
|
||||
/* ADR-039: Edge processing overrides */
|
||||
uint8_t edge_tier_val;
|
||||
if (nvs_get_u8(handle, "edge_tier", &edge_tier_val) == ESP_OK) {
|
||||
if (edge_tier_val <= 3) {
|
||||
cfg->edge_tier = edge_tier_val;
|
||||
ESP_LOGI(TAG, "NVS override: edge_tier=%u", (unsigned)cfg->edge_tier);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS edge_tier=%u out of range [0..3], ignored",
|
||||
(unsigned)edge_tier_val);
|
||||
}
|
||||
}
|
||||
|
||||
uint16_t presence_val;
|
||||
if (nvs_get_u16(handle, "pres_thresh", &presence_val) == ESP_OK) {
|
||||
cfg->presence_thresh = presence_val;
|
||||
ESP_LOGI(TAG, "NVS override: presence_thresh=%u", cfg->presence_thresh);
|
||||
}
|
||||
|
||||
uint16_t fall_val;
|
||||
if (nvs_get_u16(handle, "fall_thresh", &fall_val) == ESP_OK) {
|
||||
cfg->fall_thresh = fall_val;
|
||||
ESP_LOGI(TAG, "NVS override: fall_thresh=%u", cfg->fall_thresh);
|
||||
}
|
||||
|
||||
uint16_t vital_win_val;
|
||||
if (nvs_get_u16(handle, "vital_win", &vital_win_val) == ESP_OK) {
|
||||
if (vital_win_val >= 60 && vital_win_val <= 600) {
|
||||
cfg->vital_window = vital_win_val;
|
||||
ESP_LOGI(TAG, "NVS override: vital_window=%u", cfg->vital_window);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS vital_win=%u out of range [60..600], ignored",
|
||||
(unsigned)vital_win_val);
|
||||
}
|
||||
}
|
||||
|
||||
uint16_t vital_int_val;
|
||||
if (nvs_get_u16(handle, "vital_int", &vital_int_val) == ESP_OK) {
|
||||
if (vital_int_val >= 100) {
|
||||
cfg->vital_interval_ms = vital_int_val;
|
||||
ESP_LOGI(TAG, "NVS override: vital_interval_ms=%u", cfg->vital_interval_ms);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS vital_int=%u too small, ignored",
|
||||
(unsigned)vital_int_val);
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t subk_val;
|
||||
if (nvs_get_u8(handle, "subk_count", &subk_val) == ESP_OK) {
|
||||
if (subk_val >= 1 && subk_val <= 192) {
|
||||
cfg->subk_count = subk_val;
|
||||
ESP_LOGI(TAG, "NVS override: subk_count=%u", (unsigned)cfg->subk_count);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS subk_count=%u out of range [1..192], ignored",
|
||||
(unsigned)subk_val);
|
||||
}
|
||||
}
|
||||
|
||||
nvs_close(handle);
|
||||
}
|
||||
|
||||
@@ -35,6 +35,18 @@ typedef struct {
|
||||
uint32_t dwell_ms; /**< Dwell time per channel in ms. */
|
||||
uint8_t tdm_slot_index; /**< This node's TDM slot index (0-based). */
|
||||
uint8_t tdm_node_count; /**< Total nodes in the TDM schedule. */
|
||||
|
||||
/* MAC address filter for CSI source selection (Issue #98) */
|
||||
uint8_t filter_mac[6]; /**< Transmitter MAC to accept (all zeros = no filter). */
|
||||
uint8_t filter_mac_enabled; /**< 1 = filter active, 0 = accept all. */
|
||||
|
||||
/* ADR-039: Edge intelligence configuration */
|
||||
uint8_t edge_tier; /**< 0=disabled, 1=phase/stats, 2=vitals, 3=reserved. */
|
||||
uint16_t presence_thresh; /**< Presence detection threshold (default 50). */
|
||||
uint16_t fall_thresh; /**< Fall detection threshold (default 500). */
|
||||
uint16_t vital_window; /**< Vital signs window in frames (default 300). */
|
||||
uint16_t vital_interval_ms; /**< Vitals packet send interval in ms (default 1000). */
|
||||
uint8_t subk_count; /**< Top-K subcarrier count (default 32). */
|
||||
} nvs_config_t;
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ESP32-S3 CSI Node Provisioning Script
|
||||
|
||||
Writes WiFi credentials and aggregator target to the ESP32's NVS partition
|
||||
so users can configure a pre-built firmware binary without recompiling.
|
||||
|
||||
Usage:
|
||||
python provision.py --port COM7 --ssid "MyWiFi" --password "secret" --target-ip 192.168.1.20
|
||||
|
||||
Requirements:
|
||||
pip install esptool nvs-partition-gen
|
||||
(or use the nvs_partition_gen.py bundled with ESP-IDF)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
# NVS partition table offset — default for ESP-IDF 4MB flash with standard
|
||||
# partition scheme. The "nvs" partition starts at 0x9000 (36864) and is
|
||||
# 0x6000 (24576) bytes.
|
||||
NVS_PARTITION_OFFSET = 0x9000
|
||||
NVS_PARTITION_SIZE = 0x6000 # 24 KiB
|
||||
|
||||
|
||||
def build_nvs_csv(ssid, password, target_ip, target_port, node_id):
|
||||
"""Build an NVS CSV string for the csi_cfg namespace."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf)
|
||||
writer.writerow(["key", "type", "encoding", "value"])
|
||||
writer.writerow(["csi_cfg", "namespace", "", ""])
|
||||
if ssid:
|
||||
writer.writerow(["ssid", "data", "string", ssid])
|
||||
if password is not None:
|
||||
writer.writerow(["password", "data", "string", password])
|
||||
if target_ip:
|
||||
writer.writerow(["target_ip", "data", "string", target_ip])
|
||||
if target_port is not None:
|
||||
writer.writerow(["target_port", "data", "u16", str(target_port)])
|
||||
if node_id is not None:
|
||||
writer.writerow(["node_id", "data", "u8", str(node_id)])
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def generate_nvs_binary(csv_content, size):
|
||||
"""Generate an NVS partition binary from CSV using nvs_partition_gen.py."""
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f_csv:
|
||||
f_csv.write(csv_content)
|
||||
csv_path = f_csv.name
|
||||
|
||||
bin_path = csv_path.replace(".csv", ".bin")
|
||||
|
||||
try:
|
||||
# Try the pip-installed version first
|
||||
try:
|
||||
import nvs_partition_gen
|
||||
nvs_partition_gen.generate(csv_path, bin_path, size)
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fall back to calling the ESP-IDF script directly
|
||||
idf_path = os.environ.get("IDF_PATH", "")
|
||||
gen_script = os.path.join(idf_path, "components", "nvs_flash",
|
||||
"nvs_partition_generator", "nvs_partition_gen.py")
|
||||
if os.path.isfile(gen_script):
|
||||
subprocess.check_call([
|
||||
sys.executable, gen_script, "generate",
|
||||
csv_path, bin_path, hex(size)
|
||||
])
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
# Last resort: try as a module
|
||||
subprocess.check_call([
|
||||
sys.executable, "-m", "nvs_partition_gen", "generate",
|
||||
csv_path, bin_path, hex(size)
|
||||
])
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
finally:
|
||||
for p in (csv_path, bin_path):
|
||||
if os.path.isfile(p):
|
||||
os.unlink(p)
|
||||
|
||||
|
||||
def flash_nvs(port, baud, nvs_bin):
|
||||
"""Flash the NVS partition binary to the ESP32."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
|
||||
f.write(nvs_bin)
|
||||
bin_path = f.name
|
||||
|
||||
try:
|
||||
cmd = [
|
||||
sys.executable, "-m", "esptool",
|
||||
"--chip", "esp32s3",
|
||||
"--port", port,
|
||||
"--baud", str(baud),
|
||||
"write_flash",
|
||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||
]
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
||||
subprocess.check_call(cmd)
|
||||
print("NVS provisioning complete!")
|
||||
finally:
|
||||
os.unlink(bin_path)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Provision ESP32-S3 CSI Node with WiFi and aggregator settings",
|
||||
epilog="Example: python provision.py --port COM7 --ssid MyWiFi --password secret --target-ip 192.168.1.20",
|
||||
)
|
||||
parser.add_argument("--port", required=True, help="Serial port (e.g. COM7, /dev/ttyUSB0)")
|
||||
parser.add_argument("--baud", type=int, default=460800, help="Flash baud rate (default: 460800)")
|
||||
parser.add_argument("--ssid", help="WiFi SSID")
|
||||
parser.add_argument("--password", help="WiFi password")
|
||||
parser.add_argument("--target-ip", help="Aggregator host IP (e.g. 192.168.1.20)")
|
||||
parser.add_argument("--target-port", type=int, help="Aggregator UDP port (default: 5005)")
|
||||
parser.add_argument("--node-id", type=int, help="Node ID 0-255 (default: 1)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Generate NVS binary but don't flash")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not any([args.ssid, args.password is not None, args.target_ip,
|
||||
args.target_port, args.node_id is not None]):
|
||||
parser.error("At least one config value must be specified "
|
||||
"(--ssid, --password, --target-ip, --target-port, --node-id)")
|
||||
|
||||
print("Building NVS configuration:")
|
||||
if args.ssid:
|
||||
print(f" WiFi SSID: {args.ssid}")
|
||||
if args.password is not None:
|
||||
print(f" WiFi Password: {'*' * len(args.password)}")
|
||||
if args.target_ip:
|
||||
print(f" Target IP: {args.target_ip}")
|
||||
if args.target_port:
|
||||
print(f" Target Port: {args.target_port}")
|
||||
if args.node_id is not None:
|
||||
print(f" Node ID: {args.node_id}")
|
||||
|
||||
csv_content = build_nvs_csv(args.ssid, args.password, args.target_ip,
|
||||
args.target_port, args.node_id)
|
||||
|
||||
try:
|
||||
nvs_bin = generate_nvs_binary(csv_content, NVS_PARTITION_SIZE)
|
||||
except Exception as e:
|
||||
print(f"\nError generating NVS binary: {e}", file=sys.stderr)
|
||||
print("\nFallback: save CSV and flash manually with ESP-IDF tools.", file=sys.stderr)
|
||||
fallback_path = "nvs_config.csv"
|
||||
with open(fallback_path, "w") as f:
|
||||
f.write(csv_content)
|
||||
print(f"Saved NVS CSV to {fallback_path}", file=sys.stderr)
|
||||
print(f"Flash with: python $IDF_PATH/components/nvs_flash/"
|
||||
f"nvs_partition_generator/nvs_partition_gen.py generate "
|
||||
f"{fallback_path} nvs.bin 0x6000", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.dry_run:
|
||||
out = "nvs_provision.bin"
|
||||
with open(out, "wb") as f:
|
||||
f.write(nvs_bin)
|
||||
print(f"NVS binary saved to {out} ({len(nvs_bin)} bytes)")
|
||||
print(f"Flash manually: python -m esptool --chip esp32s3 --port {args.port} "
|
||||
f"write_flash 0x9000 {out}")
|
||||
return
|
||||
|
||||
flash_nvs(args.port, args.baud, nvs_bin)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,482 @@
|
||||
//! Model loading and lifecycle management API.
|
||||
//!
|
||||
//! Provides REST endpoints for listing, loading, and unloading `.rvf` models.
|
||||
//! Models are stored in `data/models/` and inspected using `RvfReader`.
|
||||
//!
|
||||
//! Endpoints:
|
||||
//! - `GET /api/v1/models` — list all available models
|
||||
//! - `GET /api/v1/models/:id` — detailed info for a specific model
|
||||
//! - `POST /api/v1/models/load` — load a model for inference
|
||||
//! - `POST /api/v1/models/unload` — unload the active model
|
||||
//! - `GET /api/v1/models/active` — get active model info
|
||||
//! - `POST /api/v1/models/lora/activate` — activate a LoRA profile
|
||||
//! - `GET /api/v1/models/lora/profiles` — list LoRA profiles for active model
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::{
|
||||
extract::{Path as AxumPath, State},
|
||||
response::Json,
|
||||
routing::{get, post},
|
||||
Router,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::rvf_container::RvfReader;
|
||||
|
||||
// ── Models data directory ────────────────────────────────────────────────────
|
||||
|
||||
/// Base directory for RVF model files.
|
||||
pub const MODELS_DIR: &str = "data/models";
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Summary information for a model discovered on disk.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelInfo {
|
||||
pub id: String,
|
||||
pub filename: String,
|
||||
pub version: String,
|
||||
pub description: String,
|
||||
pub size_bytes: u64,
|
||||
pub created_at: String,
|
||||
pub pck_score: Option<f64>,
|
||||
pub has_quantization: bool,
|
||||
pub lora_profiles: Vec<String>,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
/// Information about the currently loaded model, including runtime stats.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ActiveModelInfo {
|
||||
pub model_id: String,
|
||||
pub filename: String,
|
||||
pub version: String,
|
||||
pub description: String,
|
||||
pub avg_inference_ms: f64,
|
||||
pub frames_processed: u64,
|
||||
pub pose_source: String,
|
||||
pub lora_profiles: Vec<String>,
|
||||
pub active_lora_profile: Option<String>,
|
||||
}
|
||||
|
||||
/// Runtime state for the loaded model.
|
||||
///
|
||||
/// Stored inside `AppStateInner` and read by the inference path.
|
||||
pub struct LoadedModelState {
|
||||
/// Model identifier (derived from filename).
|
||||
pub model_id: String,
|
||||
/// Original filename.
|
||||
pub filename: String,
|
||||
/// Version string from the RVF manifest.
|
||||
pub version: String,
|
||||
/// Description from the RVF manifest.
|
||||
pub description: String,
|
||||
/// LoRA profiles available in this model.
|
||||
pub lora_profiles: Vec<String>,
|
||||
/// Currently active LoRA profile (if any).
|
||||
pub active_lora_profile: Option<String>,
|
||||
/// Model weights (f32 parameters).
|
||||
pub weights: Vec<f32>,
|
||||
/// Number of frames processed since load.
|
||||
pub frames_processed: u64,
|
||||
/// Cumulative inference time for avg calculation.
|
||||
pub total_inference_ms: f64,
|
||||
/// When the model was loaded.
|
||||
pub loaded_at: Instant,
|
||||
}
|
||||
|
||||
/// Request body for `POST /api/v1/models/load`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct LoadModelRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
/// Request body for `POST /api/v1/models/lora/activate`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ActivateLoraRequest {
|
||||
pub model_id: String,
|
||||
pub profile_name: String,
|
||||
}
|
||||
|
||||
/// Shared application state type.
|
||||
pub type AppState = Arc<RwLock<super::AppStateInner>>;
|
||||
|
||||
// ── Internal helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Scan the models directory and build `ModelInfo` for each `.rvf` file.
|
||||
async fn scan_models() -> Vec<ModelInfo> {
|
||||
let dir = PathBuf::from(MODELS_DIR);
|
||||
let mut models = Vec::new();
|
||||
|
||||
let mut entries = match tokio::fs::read_dir(&dir).await {
|
||||
Ok(e) => e,
|
||||
Err(_) => return models,
|
||||
};
|
||||
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("rvf") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let filename = path
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
let id = filename.trim_end_matches(".rvf").to_string();
|
||||
|
||||
let size_bytes = tokio::fs::metadata(&path)
|
||||
.await
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
|
||||
// Read the RVF to extract manifest info.
|
||||
// This is a blocking I/O operation so we use spawn_blocking.
|
||||
let path_clone = path.clone();
|
||||
let info = tokio::task::spawn_blocking(move || {
|
||||
RvfReader::from_file(&path_clone).ok()
|
||||
})
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
|
||||
let (version, description, pck_score, has_quant, lora_profiles, segment_count, created_at) =
|
||||
if let Some(reader) = &info {
|
||||
let manifest = reader.manifest().unwrap_or_default();
|
||||
let metadata = reader.metadata().unwrap_or_default();
|
||||
let version = manifest
|
||||
.get("version")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
let description = manifest
|
||||
.get("description")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let created_at = manifest
|
||||
.get("created_at")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let pck = metadata
|
||||
.get("training")
|
||||
.and_then(|t| t.get("best_pck"))
|
||||
.and_then(|v| v.as_f64());
|
||||
let has_quant = reader.quant_info().is_some();
|
||||
let lora = reader.lora_profiles();
|
||||
let seg_count = reader.segment_count();
|
||||
(version, description, pck, has_quant, lora, seg_count, created_at)
|
||||
} else {
|
||||
(
|
||||
"unknown".to_string(),
|
||||
String::new(),
|
||||
None,
|
||||
false,
|
||||
Vec::new(),
|
||||
0,
|
||||
String::new(),
|
||||
)
|
||||
};
|
||||
|
||||
models.push(ModelInfo {
|
||||
id,
|
||||
filename,
|
||||
version,
|
||||
description,
|
||||
size_bytes,
|
||||
created_at,
|
||||
pck_score,
|
||||
has_quantization: has_quant,
|
||||
lora_profiles,
|
||||
segment_count,
|
||||
});
|
||||
}
|
||||
|
||||
models.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
models
|
||||
}
|
||||
|
||||
/// Load a model from disk by ID and return its `LoadedModelState`.
|
||||
fn load_model_from_disk(model_id: &str) -> Result<LoadedModelState, String> {
|
||||
let file_path = PathBuf::from(MODELS_DIR).join(format!("{model_id}.rvf"));
|
||||
let reader = RvfReader::from_file(&file_path)?;
|
||||
|
||||
let manifest = reader.manifest().unwrap_or_default();
|
||||
let version = manifest
|
||||
.get("version")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
let description = manifest
|
||||
.get("description")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let filename = format!("{model_id}.rvf");
|
||||
let lora_profiles = reader.lora_profiles();
|
||||
let weights = reader.weights().unwrap_or_default();
|
||||
|
||||
Ok(LoadedModelState {
|
||||
model_id: model_id.to_string(),
|
||||
filename,
|
||||
version,
|
||||
description,
|
||||
lora_profiles,
|
||||
active_lora_profile: None,
|
||||
weights,
|
||||
frames_processed: 0,
|
||||
total_inference_ms: 0.0,
|
||||
loaded_at: Instant::now(),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Axum handlers ────────────────────────────────────────────────────────────
|
||||
|
||||
async fn list_models(State(_state): State<AppState>) -> Json<serde_json::Value> {
|
||||
let models = scan_models().await;
|
||||
Json(serde_json::json!({
|
||||
"models": models,
|
||||
"count": models.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_model(
|
||||
State(_state): State<AppState>,
|
||||
AxumPath(id): AxumPath<String>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let models = scan_models().await;
|
||||
match models.into_iter().find(|m| m.id == id) {
|
||||
Some(model) => Json(serde_json::to_value(&model).unwrap_or_default()),
|
||||
None => Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Model '{id}' not found"),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_model(
|
||||
State(state): State<AppState>,
|
||||
Json(body): Json<LoadModelRequest>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let model_id = body.model_id.clone();
|
||||
|
||||
// Perform blocking file I/O on spawn_blocking.
|
||||
let load_result = tokio::task::spawn_blocking(move || load_model_from_disk(&model_id))
|
||||
.await
|
||||
.map_err(|e| format!("spawn_blocking panicked: {e}"));
|
||||
|
||||
let loaded = match load_result {
|
||||
Ok(Ok(loaded)) => loaded,
|
||||
Ok(Err(e)) => {
|
||||
error!("Failed to load model '{}': {e}", body.model_id);
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Failed to load model: {e}"),
|
||||
}));
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Internal error loading model: {e}");
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Internal error: {e}"),
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
let model_id = loaded.model_id.clone();
|
||||
let weight_count = loaded.weights.len();
|
||||
|
||||
{
|
||||
let mut s = state.write().await;
|
||||
s.loaded_model = Some(loaded);
|
||||
s.model_loaded = true;
|
||||
}
|
||||
|
||||
info!("Model loaded: {model_id} ({weight_count} params)");
|
||||
|
||||
Json(serde_json::json!({
|
||||
"status": "loaded",
|
||||
"model_id": model_id,
|
||||
"weight_count": weight_count,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn unload_model(State(state): State<AppState>) -> Json<serde_json::Value> {
|
||||
let mut s = state.write().await;
|
||||
if s.loaded_model.is_none() {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": "No model is currently loaded.",
|
||||
}));
|
||||
}
|
||||
|
||||
let model_id = s
|
||||
.loaded_model
|
||||
.as_ref()
|
||||
.map(|m| m.model_id.clone())
|
||||
.unwrap_or_default();
|
||||
s.loaded_model = None;
|
||||
s.model_loaded = false;
|
||||
|
||||
info!("Model unloaded: {model_id}");
|
||||
|
||||
Json(serde_json::json!({
|
||||
"status": "unloaded",
|
||||
"model_id": model_id,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn active_model(State(state): State<AppState>) -> Json<serde_json::Value> {
|
||||
let s = state.read().await;
|
||||
match &s.loaded_model {
|
||||
Some(model) => {
|
||||
let avg_ms = if model.frames_processed > 0 {
|
||||
model.total_inference_ms / model.frames_processed as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let info = ActiveModelInfo {
|
||||
model_id: model.model_id.clone(),
|
||||
filename: model.filename.clone(),
|
||||
version: model.version.clone(),
|
||||
description: model.description.clone(),
|
||||
avg_inference_ms: avg_ms,
|
||||
frames_processed: model.frames_processed,
|
||||
pose_source: "model_inference".to_string(),
|
||||
lora_profiles: model.lora_profiles.clone(),
|
||||
active_lora_profile: model.active_lora_profile.clone(),
|
||||
};
|
||||
Json(serde_json::to_value(&info).unwrap_or_default())
|
||||
}
|
||||
None => Json(serde_json::json!({
|
||||
"status": "no_model",
|
||||
"message": "No model is currently loaded.",
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async fn activate_lora(
|
||||
State(state): State<AppState>,
|
||||
Json(body): Json<ActivateLoraRequest>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let mut s = state.write().await;
|
||||
let model = match s.loaded_model.as_mut() {
|
||||
Some(m) => m,
|
||||
None => {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": "No model is loaded. Load a model first.",
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
if model.model_id != body.model_id {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!(
|
||||
"Model '{}' is not loaded. Active model: '{}'",
|
||||
body.model_id, model.model_id
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
if !model.lora_profiles.contains(&body.profile_name) {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!(
|
||||
"LoRA profile '{}' not found. Available: {:?}",
|
||||
body.profile_name, model.lora_profiles
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
model.active_lora_profile = Some(body.profile_name.clone());
|
||||
info!(
|
||||
"LoRA profile activated: {} on model {}",
|
||||
body.profile_name, body.model_id
|
||||
);
|
||||
|
||||
Json(serde_json::json!({
|
||||
"status": "activated",
|
||||
"model_id": body.model_id,
|
||||
"profile_name": body.profile_name,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn list_lora_profiles(State(state): State<AppState>) -> Json<serde_json::Value> {
|
||||
let s = state.read().await;
|
||||
match &s.loaded_model {
|
||||
Some(model) => Json(serde_json::json!({
|
||||
"model_id": model.model_id,
|
||||
"profiles": model.lora_profiles,
|
||||
"active": model.active_lora_profile,
|
||||
})),
|
||||
None => Json(serde_json::json!({
|
||||
"profiles": serde_json::Value::Array(vec![]),
|
||||
"message": "No model is loaded.",
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Router factory ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Build the model management sub-router.
|
||||
///
|
||||
/// All routes are prefixed with `/api/v1/models`.
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/v1/models", get(list_models))
|
||||
.route("/api/v1/models/active", get(active_model))
|
||||
.route("/api/v1/models/load", post(load_model))
|
||||
.route("/api/v1/models/unload", post(unload_model))
|
||||
.route("/api/v1/models/lora/activate", post(activate_lora))
|
||||
.route("/api/v1/models/lora/profiles", get(list_lora_profiles))
|
||||
.route("/api/v1/models/{id}", get(get_model))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn model_info_serializes() {
|
||||
let info = ModelInfo {
|
||||
id: "test-model".to_string(),
|
||||
filename: "test-model.rvf".to_string(),
|
||||
version: "1.0.0".to_string(),
|
||||
description: "A test model".to_string(),
|
||||
size_bytes: 1024,
|
||||
created_at: "2024-01-01T00:00:00Z".to_string(),
|
||||
pck_score: Some(0.85),
|
||||
has_quantization: false,
|
||||
lora_profiles: vec!["default".to_string()],
|
||||
segment_count: 5,
|
||||
};
|
||||
let json = serde_json::to_string(&info).unwrap();
|
||||
assert!(json.contains("test-model"));
|
||||
assert!(json.contains("0.85"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_model_info_serializes() {
|
||||
let info = ActiveModelInfo {
|
||||
model_id: "demo".to_string(),
|
||||
filename: "demo.rvf".to_string(),
|
||||
version: "0.1.0".to_string(),
|
||||
description: String::new(),
|
||||
avg_inference_ms: 2.5,
|
||||
frames_processed: 100,
|
||||
pose_source: "model_inference".to_string(),
|
||||
lora_profiles: vec![],
|
||||
active_lora_profile: None,
|
||||
};
|
||||
let json = serde_json::to_string(&info).unwrap();
|
||||
assert!(json.contains("model_inference"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
//! CSI frame recording API.
|
||||
//!
|
||||
//! Provides REST endpoints for recording CSI frames to `.csi.jsonl` files.
|
||||
//! When recording is active, each processed CSI frame is appended as a JSON
|
||||
//! line to the current session file stored under `data/recordings/`.
|
||||
//!
|
||||
//! Endpoints:
|
||||
//! - `POST /api/v1/recording/start` — start a new recording session
|
||||
//! - `POST /api/v1/recording/stop` — stop the active recording
|
||||
//! - `GET /api/v1/recording/list` — list all recording sessions
|
||||
//! - `GET /api/v1/recording/download/:id` — download a recording file
|
||||
//! - `DELETE /api/v1/recording/:id` — delete a recording
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::{
|
||||
extract::{Path as AxumPath, State},
|
||||
response::{IntoResponse, Json},
|
||||
routing::{delete, get, post},
|
||||
Router,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
// ── Recording data directory ─────────────────────────────────────────────────
|
||||
|
||||
/// Base directory for recording files.
|
||||
pub const RECORDINGS_DIR: &str = "data/recordings";
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Request body for `POST /api/v1/recording/start`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct StartRecordingRequest {
|
||||
pub session_name: String,
|
||||
pub label: Option<String>,
|
||||
pub duration_secs: Option<u64>,
|
||||
}
|
||||
|
||||
/// Metadata for a completed or active recording session.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RecordingSession {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub started_at: String,
|
||||
pub ended_at: Option<String>,
|
||||
pub frame_count: u64,
|
||||
pub file_size_bytes: u64,
|
||||
pub file_path: String,
|
||||
}
|
||||
|
||||
/// A single recorded CSI frame line (JSONL format).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RecordedFrame {
|
||||
pub timestamp: f64,
|
||||
pub subcarriers: Vec<f64>,
|
||||
pub rssi: f64,
|
||||
pub noise_floor: f64,
|
||||
pub features: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Runtime state for the active recording session.
|
||||
///
|
||||
/// Stored inside `AppStateInner` and checked on each CSI frame tick.
|
||||
pub struct RecordingState {
|
||||
/// Whether a recording is currently active.
|
||||
pub active: bool,
|
||||
/// Session ID of the active recording.
|
||||
pub session_id: String,
|
||||
/// Session display name.
|
||||
pub session_name: String,
|
||||
/// Optional label / activity tag.
|
||||
pub label: Option<String>,
|
||||
/// Path to the JSONL file being written.
|
||||
pub file_path: PathBuf,
|
||||
/// Number of frames written so far.
|
||||
pub frame_count: u64,
|
||||
/// When the recording started.
|
||||
pub start_time: Instant,
|
||||
/// ISO-8601 start timestamp for metadata.
|
||||
pub started_at: String,
|
||||
/// Optional auto-stop duration.
|
||||
pub duration_secs: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for RecordingState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
active: false,
|
||||
session_id: String::new(),
|
||||
session_name: String::new(),
|
||||
label: None,
|
||||
file_path: PathBuf::new(),
|
||||
frame_count: 0,
|
||||
start_time: Instant::now(),
|
||||
started_at: String::new(),
|
||||
duration_secs: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared application state type used across all handlers.
|
||||
pub type AppState = Arc<RwLock<super::AppStateInner>>;
|
||||
|
||||
// ── Public helpers (called from the CSI processing loop in main.rs) ──────────
|
||||
|
||||
/// Append a single frame to the active recording file.
|
||||
///
|
||||
/// This is designed to be called from the main CSI processing tick.
|
||||
/// If recording is not active, it returns immediately.
|
||||
pub async fn maybe_record_frame(
|
||||
state: &AppState,
|
||||
subcarriers: &[f64],
|
||||
rssi: f64,
|
||||
noise_floor: f64,
|
||||
features: &serde_json::Value,
|
||||
) {
|
||||
let should_write;
|
||||
let file_path;
|
||||
let auto_stop;
|
||||
{
|
||||
let s = state.read().await;
|
||||
let rec = &s.recording_state;
|
||||
if !rec.active {
|
||||
return;
|
||||
}
|
||||
should_write = true;
|
||||
file_path = rec.file_path.clone();
|
||||
auto_stop = rec.duration_secs.map(|d| rec.start_time.elapsed().as_secs() >= d).unwrap_or(false);
|
||||
}
|
||||
|
||||
if auto_stop {
|
||||
// Duration exceeded — stop recording.
|
||||
stop_recording_inner(state).await;
|
||||
return;
|
||||
}
|
||||
|
||||
if !should_write {
|
||||
return;
|
||||
}
|
||||
|
||||
let frame = RecordedFrame {
|
||||
timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0,
|
||||
subcarriers: subcarriers.to_vec(),
|
||||
rssi,
|
||||
noise_floor,
|
||||
features: features.clone(),
|
||||
};
|
||||
|
||||
let line = match serde_json::to_string(&frame) {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
warn!("Failed to serialize recording frame: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Append line to file (async).
|
||||
if let Err(e) = append_line(&file_path, &line).await {
|
||||
warn!("Failed to write recording frame: {e}");
|
||||
return;
|
||||
}
|
||||
|
||||
// Increment frame counter.
|
||||
{
|
||||
let mut s = state.write().await;
|
||||
s.recording_state.frame_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async fn append_line(path: &Path, line: &str) -> std::io::Result<()> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let mut file = tokio::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)
|
||||
.await?;
|
||||
file.write_all(line.as_bytes()).await?;
|
||||
file.write_all(b"\n").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Internal helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Stop the active recording and write session metadata.
|
||||
async fn stop_recording_inner(state: &AppState) {
|
||||
let mut s = state.write().await;
|
||||
if !s.recording_state.active {
|
||||
return;
|
||||
}
|
||||
s.recording_state.active = false;
|
||||
|
||||
let ended_at = chrono::Utc::now().to_rfc3339();
|
||||
let session = RecordingSession {
|
||||
id: s.recording_state.session_id.clone(),
|
||||
name: s.recording_state.session_name.clone(),
|
||||
label: s.recording_state.label.clone(),
|
||||
started_at: s.recording_state.started_at.clone(),
|
||||
ended_at: Some(ended_at),
|
||||
frame_count: s.recording_state.frame_count,
|
||||
file_size_bytes: std::fs::metadata(&s.recording_state.file_path)
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0),
|
||||
file_path: s.recording_state.file_path.to_string_lossy().to_string(),
|
||||
};
|
||||
|
||||
// Write a companion .meta.json alongside the JSONL file.
|
||||
let meta_path = s.recording_state.file_path.with_extension("meta.json");
|
||||
if let Ok(json) = serde_json::to_string_pretty(&session) {
|
||||
if let Err(e) = tokio::fs::write(&meta_path, json).await {
|
||||
warn!("Failed to write recording metadata: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Recording stopped: {} ({} frames)",
|
||||
session.id, session.frame_count
|
||||
);
|
||||
}
|
||||
|
||||
/// Scan the recordings directory and return all sessions with metadata.
|
||||
async fn list_sessions() -> Vec<RecordingSession> {
|
||||
let dir = PathBuf::from(RECORDINGS_DIR);
|
||||
let mut sessions = Vec::new();
|
||||
|
||||
let mut entries = match tokio::fs::read_dir(&dir).await {
|
||||
Ok(e) => e,
|
||||
Err(_) => return sessions,
|
||||
};
|
||||
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) == Some("json")
|
||||
&& path.to_string_lossy().contains(".meta.")
|
||||
{
|
||||
if let Ok(data) = tokio::fs::read_to_string(&path).await {
|
||||
if let Ok(session) = serde_json::from_str::<RecordingSession>(&data) {
|
||||
sessions.push(session);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by started_at descending (newest first).
|
||||
sessions.sort_by(|a, b| b.started_at.cmp(&a.started_at));
|
||||
sessions
|
||||
}
|
||||
|
||||
// ── Axum handlers ────────────────────────────────────────────────────────────
|
||||
|
||||
async fn start_recording(
|
||||
State(state): State<AppState>,
|
||||
Json(body): Json<StartRecordingRequest>,
|
||||
) -> Json<serde_json::Value> {
|
||||
// Ensure recordings directory exists.
|
||||
if let Err(e) = tokio::fs::create_dir_all(RECORDINGS_DIR).await {
|
||||
error!("Failed to create recordings directory: {e}");
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Cannot create recordings directory: {e}"),
|
||||
}));
|
||||
}
|
||||
|
||||
let mut s = state.write().await;
|
||||
if s.recording_state.active {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": "A recording is already active. Stop it first.",
|
||||
"active_session": s.recording_state.session_id,
|
||||
}));
|
||||
}
|
||||
|
||||
let session_id = format!(
|
||||
"{}-{}",
|
||||
body.session_name.replace(' ', "_"),
|
||||
chrono::Utc::now().format("%Y%m%d_%H%M%S")
|
||||
);
|
||||
let file_name = format!("{session_id}.csi.jsonl");
|
||||
let file_path = PathBuf::from(RECORDINGS_DIR).join(&file_name);
|
||||
let started_at = chrono::Utc::now().to_rfc3339();
|
||||
|
||||
s.recording_state = RecordingState {
|
||||
active: true,
|
||||
session_id: session_id.clone(),
|
||||
session_name: body.session_name.clone(),
|
||||
label: body.label.clone(),
|
||||
file_path: file_path.clone(),
|
||||
frame_count: 0,
|
||||
start_time: Instant::now(),
|
||||
started_at: started_at.clone(),
|
||||
duration_secs: body.duration_secs,
|
||||
};
|
||||
|
||||
info!(
|
||||
"Recording started: {session_id} (label={:?}, duration={:?}s)",
|
||||
body.label, body.duration_secs
|
||||
);
|
||||
|
||||
Json(serde_json::json!({
|
||||
"status": "recording",
|
||||
"session_id": session_id,
|
||||
"session_name": body.session_name,
|
||||
"label": body.label,
|
||||
"started_at": started_at,
|
||||
"file_path": file_path.to_string_lossy(),
|
||||
"duration_secs": body.duration_secs,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn stop_recording(State(state): State<AppState>) -> Json<serde_json::Value> {
|
||||
{
|
||||
let s = state.read().await;
|
||||
if !s.recording_state.active {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": "No active recording to stop.",
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
stop_recording_inner(&state).await;
|
||||
|
||||
let s = state.read().await;
|
||||
Json(serde_json::json!({
|
||||
"status": "stopped",
|
||||
"session_id": s.recording_state.session_id,
|
||||
"frame_count": s.recording_state.frame_count,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn list_recordings(
|
||||
State(_state): State<AppState>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let sessions = list_sessions().await;
|
||||
Json(serde_json::json!({
|
||||
"recordings": sessions,
|
||||
"count": sessions.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn download_recording(
|
||||
State(_state): State<AppState>,
|
||||
AxumPath(id): AxumPath<String>,
|
||||
) -> impl IntoResponse {
|
||||
let dir = PathBuf::from(RECORDINGS_DIR);
|
||||
// Find the JSONL file matching the ID.
|
||||
let file_path = dir.join(format!("{id}.csi.jsonl"));
|
||||
|
||||
if !file_path.exists() {
|
||||
return (
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Recording '{id}' not found"),
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
match tokio::fs::read(&file_path).await {
|
||||
Ok(data) => {
|
||||
let headers = [
|
||||
(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
"application/x-ndjson".to_string(),
|
||||
),
|
||||
(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
format!("attachment; filename=\"{id}.csi.jsonl\""),
|
||||
),
|
||||
];
|
||||
(headers, data).into_response()
|
||||
}
|
||||
Err(e) => (
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Failed to read recording: {e}"),
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_recording(
|
||||
State(_state): State<AppState>,
|
||||
AxumPath(id): AxumPath<String>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let dir = PathBuf::from(RECORDINGS_DIR);
|
||||
let jsonl_path = dir.join(format!("{id}.csi.jsonl"));
|
||||
let meta_path = dir.join(format!("{id}.csi.meta.json"));
|
||||
|
||||
if !jsonl_path.exists() && !meta_path.exists() {
|
||||
return Json(serde_json::json!({
|
||||
"status": "error",
|
||||
"message": format!("Recording '{id}' not found"),
|
||||
}));
|
||||
}
|
||||
|
||||
let mut deleted = Vec::new();
|
||||
if jsonl_path.exists() {
|
||||
if let Err(e) = tokio::fs::remove_file(&jsonl_path).await {
|
||||
warn!("Failed to delete {}: {e}", jsonl_path.display());
|
||||
} else {
|
||||
deleted.push(jsonl_path.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
if meta_path.exists() {
|
||||
if let Err(e) = tokio::fs::remove_file(&meta_path).await {
|
||||
warn!("Failed to delete {}: {e}", meta_path.display());
|
||||
} else {
|
||||
deleted.push(meta_path.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
Json(serde_json::json!({
|
||||
"status": "deleted",
|
||||
"id": id,
|
||||
"deleted_files": deleted,
|
||||
}))
|
||||
}
|
||||
|
||||
// ── Router factory ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Build the recording sub-router.
|
||||
///
|
||||
/// Mount this at the top level; all routes are prefixed with `/api/v1/recording`.
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/v1/recording/start", post(start_recording))
|
||||
.route("/api/v1/recording/stop", post(stop_recording))
|
||||
.route("/api/v1/recording/list", get(list_recordings))
|
||||
.route(
|
||||
"/api/v1/recording/download/{id}",
|
||||
get(download_recording),
|
||||
)
|
||||
.route("/api/v1/recording/{id}", delete(delete_recording))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_recording_state_is_inactive() {
|
||||
let rs = RecordingState::default();
|
||||
assert!(!rs.active);
|
||||
assert_eq!(rs.frame_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_frame_serializes_to_json() {
|
||||
let frame = RecordedFrame {
|
||||
timestamp: 1700000000.0,
|
||||
subcarriers: vec![1.0, 2.0, 3.0],
|
||||
rssi: -45.0,
|
||||
noise_floor: -90.0,
|
||||
features: serde_json::json!({"motion": 0.5}),
|
||||
};
|
||||
let json = serde_json::to_string(&frame).unwrap();
|
||||
assert!(json.contains("\"timestamp\""));
|
||||
assert!(json.contains("\"subcarriers\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_session_deserializes() {
|
||||
let json = r#"{
|
||||
"id": "test-20240101_120000",
|
||||
"name": "test",
|
||||
"label": "walking",
|
||||
"started_at": "2024-01-01T12:00:00Z",
|
||||
"ended_at": "2024-01-01T12:05:00Z",
|
||||
"frame_count": 3000,
|
||||
"file_size_bytes": 1500000,
|
||||
"file_path": "data/recordings/test-20240101_120000.csi.jsonl"
|
||||
}"#;
|
||||
let session: RecordingSession = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(session.id, "test-20240101_120000");
|
||||
assert_eq!(session.frame_count, 3000);
|
||||
assert_eq!(session.label, Some("walking".to_string()));
|
||||
}
|
||||
}
|
||||
@@ -1,46 +1,73 @@
|
||||
# WiFi DensePose UI
|
||||
|
||||
A modular, modern web interface for the WiFi DensePose human tracking system. This UI provides real-time monitoring, configuration, and visualization of WiFi-based pose estimation.
|
||||
A modular, modern web interface for the WiFi DensePose human tracking system. Provides real-time monitoring, WiFi sensing visualization, and pose estimation from CSI (Channel State Information).
|
||||
|
||||
## 🏗️ Architecture
|
||||
## Architecture
|
||||
|
||||
The UI follows a modular architecture with clear separation of concerns:
|
||||
|
||||
```
|
||||
ui/
|
||||
├── app.js # Main application entry point
|
||||
├── index.html # Updated HTML with modular structure
|
||||
├── style.css # Complete CSS with additional styles
|
||||
├── config/ # Configuration modules
|
||||
│ └── api.config.js # API endpoints and configuration
|
||||
├── services/ # Service layer for API communication
|
||||
│ ├── api.service.js # HTTP API client
|
||||
│ ├── websocket.service.js # WebSocket client
|
||||
│ ├── pose.service.js # Pose estimation API wrapper
|
||||
│ ├── health.service.js # Health monitoring API wrapper
|
||||
│ └── stream.service.js # Streaming API wrapper
|
||||
├── components/ # UI components
|
||||
│ ├── TabManager.js # Tab navigation component
|
||||
│ ├── DashboardTab.js # Dashboard component with live data
|
||||
│ ├── HardwareTab.js # Hardware configuration component
|
||||
│ └── LiveDemoTab.js # Live demo with streaming
|
||||
├── utils/ # Utility functions and helpers
|
||||
│ └── mock-server.js # Mock server for testing
|
||||
└── tests/ # Comprehensive test suite
|
||||
├── test-runner.html # Test runner UI
|
||||
├── test-runner.js # Test framework and cases
|
||||
└── integration-test.html # Integration testing page
|
||||
├── app.js # Main application entry point
|
||||
├── index.html # HTML shell with tab structure
|
||||
├── style.css # Complete CSS design system
|
||||
├── config/
|
||||
│ └── api.config.js # API endpoints and configuration
|
||||
├── services/
|
||||
│ ├── api.service.js # HTTP API client
|
||||
│ ├── websocket.service.js # WebSocket connection manager
|
||||
│ ├── websocket-client.js # Low-level WebSocket client
|
||||
│ ├── pose.service.js # Pose estimation API wrapper
|
||||
│ ├── sensing.service.js # WiFi sensing data service (live + simulation fallback)
|
||||
│ ├── health.service.js # Health monitoring API wrapper
|
||||
│ ├── stream.service.js # Streaming API wrapper
|
||||
│ └── data-processor.js # Signal data processing utilities
|
||||
├── components/
|
||||
│ ├── TabManager.js # Tab navigation component
|
||||
│ ├── DashboardTab.js # Dashboard with live system metrics
|
||||
│ ├── SensingTab.js # WiFi sensing visualization (3D signal field, metrics)
|
||||
│ ├── LiveDemoTab.js # Live pose detection with setup guide
|
||||
│ ├── HardwareTab.js # Hardware configuration
|
||||
│ ├── SettingsPanel.js # Settings panel
|
||||
│ ├── PoseDetectionCanvas.js # Canvas-based pose skeleton renderer
|
||||
│ ├── gaussian-splats.js # 3D Gaussian splat signal field renderer (Three.js)
|
||||
│ ├── body-model.js # 3D body model
|
||||
│ ├── scene.js # Three.js scene management
|
||||
│ ├── signal-viz.js # Signal visualization utilities
|
||||
│ ├── environment.js # Environment/room visualization
|
||||
│ └── dashboard-hud.js # Dashboard heads-up display
|
||||
├── utils/
|
||||
│ ├── backend-detector.js # Auto-detect backend availability
|
||||
│ ├── mock-server.js # Mock server for testing
|
||||
│ └── pose-renderer.js # Pose rendering utilities
|
||||
└── tests/
|
||||
├── test-runner.html # Test runner UI
|
||||
├── test-runner.js # Test framework and cases
|
||||
└── integration-test.html # Integration testing page
|
||||
```
|
||||
|
||||
## 🚀 Features
|
||||
## Features
|
||||
|
||||
### Smart Backend Detection
|
||||
- **Automatic Detection**: Automatically detects if your FastAPI backend is running
|
||||
- **Real Backend Priority**: Always uses the real backend when available
|
||||
- **Mock Fallback**: Falls back to mock server only when backend is unavailable
|
||||
- **Testing Mode**: Can force mock mode for testing and development
|
||||
### WiFi Sensing Tab
|
||||
- 3D Gaussian-splat signal field visualization (Three.js)
|
||||
- Real-time RSSI, variance, motion band, breathing band metrics
|
||||
- Presence/motion classification with confidence scores
|
||||
- **Data source banner**: green "LIVE - ESP32", yellow "RECONNECTING...", or red "SIMULATED DATA"
|
||||
- Sparkline RSSI history graph
|
||||
- "About This Data" card explaining CSI capabilities per sensor count
|
||||
|
||||
### Real-time Dashboard
|
||||
### Live Demo Tab
|
||||
- WebSocket-based real-time pose skeleton rendering
|
||||
- **Estimation Mode badge**: green "Signal-Derived" or blue "Model Inference"
|
||||
- **Setup Guide panel** showing what each ESP32 count provides:
|
||||
- 1 ESP32: presence, breathing, gross motion
|
||||
- 2-3 ESP32s: body localization, motion direction
|
||||
- 4+ ESP32s + trained model: individual limb tracking, full pose
|
||||
- Debug mode with log export
|
||||
- Zone selection and force-reconnect controls
|
||||
- Performance metrics sidebar (frames, uptime, errors)
|
||||
|
||||
### Dashboard
|
||||
- Live system health monitoring
|
||||
- Real-time pose detection statistics
|
||||
- Zone occupancy tracking
|
||||
@@ -53,284 +80,118 @@ ui/
|
||||
- Configuration panels
|
||||
- Hardware status monitoring
|
||||
|
||||
### Live Demo
|
||||
- WebSocket-based real-time streaming
|
||||
- Signal visualization
|
||||
- Pose detection visualization
|
||||
- Interactive controls
|
||||
## Data Sources
|
||||
|
||||
### API Integration
|
||||
- Complete REST API coverage
|
||||
- WebSocket streaming support
|
||||
- Authentication handling
|
||||
- Error management
|
||||
- Request/response interceptors
|
||||
The sensing service (`sensing.service.js`) supports three connection states:
|
||||
|
||||
## 📋 API Coverage
|
||||
| State | Banner Color | Description |
|
||||
|-------|-------------|-------------|
|
||||
| **LIVE - ESP32** | Green | Connected to the Rust sensing server receiving real CSI data |
|
||||
| **RECONNECTING** | Yellow (pulsing) | WebSocket disconnected, retrying (up to 20 attempts) |
|
||||
| **SIMULATED DATA** | Red | Fallback to client-side simulation after 5+ failed reconnects |
|
||||
|
||||
The UI integrates with all WiFi DensePose API endpoints:
|
||||
Simulated frames include a `_simulated: true` marker so code can detect synthetic data.
|
||||
|
||||
### Health Endpoints
|
||||
- `GET /health/health` - System health check
|
||||
- `GET /health/ready` - Readiness check
|
||||
- `GET /health/live` - Liveness check
|
||||
- `GET /health/metrics` - System metrics
|
||||
- `GET /health/version` - Version information
|
||||
## Backends
|
||||
|
||||
### Pose Estimation
|
||||
- `GET /api/v1/pose/current` - Current pose data
|
||||
- `POST /api/v1/pose/analyze` - Trigger analysis
|
||||
- `GET /api/v1/pose/zones/{zone_id}/occupancy` - Zone occupancy
|
||||
- `GET /api/v1/pose/zones/summary` - All zones summary
|
||||
- `POST /api/v1/pose/historical` - Historical data
|
||||
- `GET /api/v1/pose/activities` - Recent activities
|
||||
- `POST /api/v1/pose/calibrate` - System calibration
|
||||
- `GET /api/v1/pose/stats` - Statistics
|
||||
### Rust Sensing Server (primary)
|
||||
The Rust-based `wifi-densepose-sensing-server` serves the UI and provides:
|
||||
- `GET /health` — server health
|
||||
- `GET /api/v1/sensing/latest` — latest sensing features
|
||||
- `GET /api/v1/vital-signs` — vital sign estimates (HR/RR)
|
||||
- `GET /api/v1/model/info` — RVF model container info
|
||||
- `WS /ws/sensing` — real-time sensing data stream
|
||||
- `WS /api/v1/stream/pose` — real-time pose keypoint stream
|
||||
|
||||
### Streaming
|
||||
- `WS /api/v1/stream/pose` - Real-time pose stream
|
||||
- `WS /api/v1/stream/events` - Event stream
|
||||
- `GET /api/v1/stream/status` - Stream status
|
||||
- `POST /api/v1/stream/start` - Start streaming
|
||||
- `POST /api/v1/stream/stop` - Stop streaming
|
||||
- `GET /api/v1/stream/clients` - Connected clients
|
||||
- `DELETE /api/v1/stream/clients/{client_id}` - Disconnect client
|
||||
### Python FastAPI (legacy)
|
||||
The original Python backend on port 8000 is still supported. The UI auto-detects which backend is available via `backend-detector.js`.
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### Test Runner
|
||||
Open `tests/test-runner.html` to run the complete test suite:
|
||||
## Quick Start
|
||||
|
||||
### With Docker (recommended)
|
||||
```bash
|
||||
# Serve the UI directory on port 3000 (to avoid conflicts with FastAPI on 8000)
|
||||
cd /workspaces/wifi-densepose/ui
|
||||
cd docker/
|
||||
|
||||
# Default: auto-detects ESP32 on UDP 5005, falls back to simulation
|
||||
docker-compose up
|
||||
|
||||
# Force real ESP32 data
|
||||
CSI_SOURCE=esp32 docker-compose up
|
||||
|
||||
# Force simulation (no hardware needed)
|
||||
CSI_SOURCE=simulated docker-compose up
|
||||
```
|
||||
Open http://localhost:3000/ui/index.html
|
||||
|
||||
### With local Rust binary
|
||||
```bash
|
||||
cd rust-port/wifi-densepose-rs
|
||||
cargo build -p wifi-densepose-sensing-server --no-default-features
|
||||
|
||||
# Run with simulated data
|
||||
../../target/debug/sensing-server --source simulated --tick-ms 100 --ui-path ../../ui --http-port 3000
|
||||
|
||||
# Run with real ESP32
|
||||
../../target/debug/sensing-server --source esp32 --tick-ms 100 --ui-path ../../ui --http-port 3000
|
||||
```
|
||||
Open http://localhost:3000/ui/index.html
|
||||
|
||||
### With Python HTTP server (legacy)
|
||||
```bash
|
||||
# Start FastAPI backend on port 8000
|
||||
wifi-densepose start
|
||||
|
||||
# Serve the UI on port 3000
|
||||
cd ui/
|
||||
python -m http.server 3000
|
||||
# Open http://localhost:3000/tests/test-runner.html
|
||||
```
|
||||
Open http://localhost:3000
|
||||
|
||||
### Test Categories
|
||||
- **API Configuration Tests** - Configuration and URL building
|
||||
- **API Service Tests** - HTTP client functionality
|
||||
- **WebSocket Service Tests** - WebSocket connection management
|
||||
- **Pose Service Tests** - Pose estimation API wrapper
|
||||
- **Health Service Tests** - Health monitoring functionality
|
||||
- **UI Component Tests** - Component behavior and interaction
|
||||
- **Integration Tests** - End-to-end functionality
|
||||
## Pose Estimation Modes
|
||||
|
||||
### Integration Testing
|
||||
Use `tests/integration-test.html` for visual integration testing:
|
||||
| Mode | Badge | Requirements | Accuracy |
|
||||
|------|-------|-------------|----------|
|
||||
| **Signal-Derived** | Green | 1+ ESP32, no model needed | Presence, breathing, gross motion |
|
||||
| **Model Inference** | Blue | 4+ ESP32s + trained `.rvf` model | Full 17-keypoint COCO pose |
|
||||
|
||||
To use model inference, start the server with a trained model:
|
||||
```bash
|
||||
# Open http://localhost:3000/tests/integration-test.html
|
||||
sensing-server --source esp32 --model path/to/model.rvf --ui-path ./ui
|
||||
```
|
||||
|
||||
Features:
|
||||
- Mock server with realistic API responses
|
||||
- Visual testing of all components
|
||||
- Real-time data simulation
|
||||
- Error scenario testing
|
||||
- WebSocket stream testing
|
||||
|
||||
## 🛠️ Usage
|
||||
|
||||
### Basic Setup
|
||||
```html
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<!-- Your content -->
|
||||
</div>
|
||||
<script type="module" src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
### Using Services
|
||||
```javascript
|
||||
import { poseService } from './services/pose.service.js';
|
||||
import { healthService } from './services/health.service.js';
|
||||
|
||||
// Get current pose data
|
||||
const poseData = await poseService.getCurrentPose();
|
||||
|
||||
// Subscribe to health updates
|
||||
healthService.subscribeToHealth(health => {
|
||||
console.log('Health status:', health.status);
|
||||
});
|
||||
|
||||
// Start pose streaming
|
||||
poseService.startPoseStream({
|
||||
minConfidence: 0.7,
|
||||
maxFps: 30
|
||||
});
|
||||
|
||||
poseService.subscribeToPoseUpdates(update => {
|
||||
if (update.type === 'pose_update') {
|
||||
console.log('New pose data:', update.data);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Using Components
|
||||
```javascript
|
||||
import { TabManager } from './components/TabManager.js';
|
||||
import { DashboardTab } from './components/DashboardTab.js';
|
||||
|
||||
// Initialize tab manager
|
||||
const container = document.querySelector('.container');
|
||||
const tabManager = new TabManager(container);
|
||||
tabManager.init();
|
||||
|
||||
// Initialize dashboard
|
||||
const dashboardContainer = document.getElementById('dashboard');
|
||||
const dashboard = new DashboardTab(dashboardContainer);
|
||||
await dashboard.init();
|
||||
```
|
||||
|
||||
## 🔧 Configuration
|
||||
## Configuration
|
||||
|
||||
### API Configuration
|
||||
Edit `config/api.config.js` to modify API settings:
|
||||
Edit `config/api.config.js`:
|
||||
|
||||
```javascript
|
||||
export const API_CONFIG = {
|
||||
BASE_URL: window.location.origin,
|
||||
API_VERSION: '/api/v1',
|
||||
|
||||
// Rate limiting
|
||||
RATE_LIMITS: {
|
||||
REQUESTS_PER_MINUTE: 60,
|
||||
BURST_LIMIT: 10
|
||||
},
|
||||
|
||||
// WebSocket configuration
|
||||
WS_CONFIG: {
|
||||
RECONNECT_DELAY: 5000,
|
||||
MAX_RECONNECT_ATTEMPTS: 5,
|
||||
MAX_RECONNECT_ATTEMPTS: 20,
|
||||
PING_INTERVAL: 30000
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
### Authentication
|
||||
```javascript
|
||||
import { apiService } from './services/api.service.js';
|
||||
## Testing
|
||||
|
||||
// Set authentication token
|
||||
apiService.setAuthToken('your-jwt-token');
|
||||
Open `tests/test-runner.html` to run the test suite:
|
||||
|
||||
// Add request interceptor for auth
|
||||
apiService.addRequestInterceptor((url, options) => {
|
||||
// Modify request before sending
|
||||
return { url, options };
|
||||
});
|
||||
```
|
||||
|
||||
## 🎨 Styling
|
||||
|
||||
The UI uses a comprehensive CSS design system with:
|
||||
|
||||
- CSS Custom Properties for theming
|
||||
- Dark/light mode support
|
||||
- Responsive design
|
||||
- Component-based styling
|
||||
- Smooth animations and transitions
|
||||
|
||||
### Key CSS Variables
|
||||
```css
|
||||
:root {
|
||||
--color-primary: rgba(33, 128, 141, 1);
|
||||
--color-background: rgba(252, 252, 249, 1);
|
||||
--color-surface: rgba(255, 255, 253, 1);
|
||||
--color-text: rgba(19, 52, 59, 1);
|
||||
--space-16: 16px;
|
||||
--radius-lg: 12px;
|
||||
}
|
||||
```
|
||||
|
||||
## 🔍 Monitoring & Debugging
|
||||
|
||||
### Health Monitoring
|
||||
```javascript
|
||||
import { healthService } from './services/health.service.js';
|
||||
|
||||
// Start automatic health checks
|
||||
healthService.startHealthMonitoring(30000); // Every 30 seconds
|
||||
|
||||
// Check if system is healthy
|
||||
const isHealthy = healthService.isSystemHealthy();
|
||||
|
||||
// Get specific component status
|
||||
const apiStatus = healthService.getComponentStatus('api');
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
```javascript
|
||||
// Global error handling
|
||||
window.addEventListener('error', (event) => {
|
||||
console.error('Global error:', event.error);
|
||||
});
|
||||
|
||||
// API error handling
|
||||
apiService.addResponseInterceptor(async (response, url) => {
|
||||
if (!response.ok) {
|
||||
console.error(`API error: ${response.status} for ${url}`);
|
||||
}
|
||||
return response;
|
||||
});
|
||||
```
|
||||
|
||||
## 🚀 Deployment
|
||||
|
||||
### Development
|
||||
|
||||
**Option 1: Use the startup script**
|
||||
```bash
|
||||
cd /workspaces/wifi-densepose/ui
|
||||
./start-ui.sh
|
||||
```
|
||||
|
||||
**Option 2: Manual setup**
|
||||
```bash
|
||||
# First, start your FastAPI backend (runs on port 8000)
|
||||
wifi-densepose start
|
||||
# or from the main project directory:
|
||||
python -m wifi_densepose.main
|
||||
|
||||
# Then, start the UI server on a different port to avoid conflicts
|
||||
cd /workspaces/wifi-densepose/ui
|
||||
cd ui/
|
||||
python -m http.server 3000
|
||||
# or
|
||||
npx http-server . -p 3000
|
||||
|
||||
# Open the UI at http://localhost:3000
|
||||
# The UI will automatically detect and connect to your backend
|
||||
# Open http://localhost:3000/tests/test-runner.html
|
||||
```
|
||||
|
||||
### Backend Detection Behavior
|
||||
- **Real Backend Available**: UI connects to `http://localhost:8000` and shows ✅ "Connected to real backend"
|
||||
- **Backend Unavailable**: UI automatically uses mock server and shows ⚠️ "Mock server active - testing mode"
|
||||
- **Force Mock Mode**: Set `API_CONFIG.MOCK_SERVER.ENABLED = true` for testing
|
||||
Test categories: API configuration, API service, WebSocket, pose service, health service, UI components, integration.
|
||||
|
||||
### Production
|
||||
1. Configure `API_CONFIG.BASE_URL` for your backend
|
||||
2. Set up HTTPS for WebSocket connections
|
||||
3. Configure authentication if required
|
||||
4. Optimize assets (minify CSS/JS)
|
||||
5. Set up monitoring and logging
|
||||
## Styling
|
||||
|
||||
## 🤝 Contributing
|
||||
Uses a CSS design system with custom properties, dark/light mode, responsive layout, and component-based styling. Key variables in `:root` of `style.css`.
|
||||
|
||||
1. Follow the modular architecture
|
||||
2. Add tests for new functionality
|
||||
3. Update documentation
|
||||
4. Ensure TypeScript compatibility
|
||||
5. Test with mock server
|
||||
## License
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is part of the WiFi-DensePose system. See the main project LICENSE file for details.
|
||||
Part of the WiFi-DensePose system. See the main project LICENSE file.
|
||||
|
||||
@@ -130,6 +130,9 @@ class WiFiDensePoseApp {
|
||||
this.components.sensing = new SensingTab(sensingContainer);
|
||||
}
|
||||
|
||||
// Training tab - lazy load to avoid breaking other tabs if import fails
|
||||
this.initTrainingTab();
|
||||
|
||||
// Architecture tab - static content, no component needed
|
||||
|
||||
// Performance tab - static content, no component needed
|
||||
@@ -137,6 +140,28 @@ class WiFiDensePoseApp {
|
||||
// Applications tab - static content, no component needed
|
||||
}
|
||||
|
||||
// Lazy-load Training tab panels (dynamic import so failures don't break other tabs)
|
||||
async initTrainingTab() {
|
||||
try {
|
||||
const [{ default: TrainingPanel }, { default: ModelPanel }] = await Promise.all([
|
||||
import('./components/TrainingPanel.js'),
|
||||
import('./components/ModelPanel.js')
|
||||
]);
|
||||
|
||||
const trainingContainer = document.getElementById('training-panel-container');
|
||||
if (trainingContainer) {
|
||||
this.components.trainingPanel = new TrainingPanel(trainingContainer);
|
||||
}
|
||||
|
||||
const modelContainer = document.getElementById('model-panel-container');
|
||||
if (modelContainer) {
|
||||
this.components.modelPanel = new ModelPanel(modelContainer);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load Training tab components:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle tab changes
|
||||
handleTabChange(newTab, oldTab) {
|
||||
console.log(`Tab changed from ${oldTab} to ${newTab}`);
|
||||
@@ -168,6 +193,16 @@ class WiFiDensePoseApp {
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'training':
|
||||
// Refresh panels when training tab becomes visible
|
||||
if (this.components.trainingPanel && typeof this.components.trainingPanel.refresh === 'function') {
|
||||
this.components.trainingPanel.refresh();
|
||||
}
|
||||
if (this.components.modelPanel && typeof this.components.modelPanel.refresh === 'function') {
|
||||
this.components.modelPanel.refresh();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { healthService } from '../services/health.service.js';
|
||||
import { poseService } from '../services/pose.service.js';
|
||||
import { sensingService } from '../services/sensing.service.js';
|
||||
|
||||
export class DashboardTab {
|
||||
constructor(containerElement) {
|
||||
@@ -63,6 +64,17 @@ export class DashboardTab {
|
||||
this.updateHealthStatus(health);
|
||||
});
|
||||
|
||||
// Subscribe to sensing service state changes for data source indicator
|
||||
this._sensingUnsub = sensingService.onStateChange(() => {
|
||||
this.updateDataSourceIndicator();
|
||||
});
|
||||
// Also update on data — catches source changes mid-stream
|
||||
this._sensingDataUnsub = sensingService.onData(() => {
|
||||
this.updateDataSourceIndicator();
|
||||
});
|
||||
// Initial update
|
||||
this.updateDataSourceIndicator();
|
||||
|
||||
// Start periodic stats updates
|
||||
this.statsInterval = setInterval(() => {
|
||||
this.updateLiveStats();
|
||||
@@ -72,6 +84,25 @@ export class DashboardTab {
|
||||
healthService.startHealthMonitoring(30000);
|
||||
}
|
||||
|
||||
// Update the data source indicator on the dashboard
|
||||
updateDataSourceIndicator() {
|
||||
const el = this.container.querySelector('#dashboard-datasource');
|
||||
if (!el) return;
|
||||
const ds = sensingService.dataSource;
|
||||
const statusText = el.querySelector('.status-text');
|
||||
const statusMsg = el.querySelector('.status-message');
|
||||
const config = {
|
||||
'live': { text: 'ESP32', status: 'healthy', msg: 'Real hardware connected' },
|
||||
'server-simulated': { text: 'SIMULATED', status: 'warning', msg: 'Server running without hardware' },
|
||||
'reconnecting': { text: 'RECONNECTING', status: 'degraded', msg: 'Attempting to connect...' },
|
||||
'simulated': { text: 'OFFLINE', status: 'unhealthy', msg: 'Server unreachable, local fallback' },
|
||||
};
|
||||
const cfg = config[ds] || config['reconnecting'];
|
||||
el.className = `component-status status-${cfg.status}`;
|
||||
if (statusText) statusText.textContent = cfg.text;
|
||||
if (statusMsg) statusMsg.textContent = cfg.msg;
|
||||
}
|
||||
|
||||
// Update API info display
|
||||
updateApiInfo(info) {
|
||||
// Update version
|
||||
@@ -394,11 +425,13 @@ export class DashboardTab {
|
||||
if (this.healthSubscription) {
|
||||
this.healthSubscription();
|
||||
}
|
||||
|
||||
if (this._sensingUnsub) this._sensingUnsub();
|
||||
if (this._sensingDataUnsub) this._sensingDataUnsub();
|
||||
|
||||
if (this.statsInterval) {
|
||||
clearInterval(this.statsInterval);
|
||||
}
|
||||
|
||||
|
||||
healthService.stopHealthMonitoring();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// ModelPanel Component for WiFi-DensePose UI
|
||||
// Dark-mode panel for model management: listing, loading, LoRA profiles.
|
||||
|
||||
import { modelService } from '../services/model.service.js';
|
||||
|
||||
const MP_STYLES = `
|
||||
.mp-panel{background:rgba(17,24,39,.9);border:1px solid rgba(56,68,89,.6);border-radius:8px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;color:#e0e0e0;overflow:hidden}
|
||||
.mp-header{display:flex;align-items:center;justify-content:space-between;padding:14px 16px;background:rgba(13,17,23,.95);border-bottom:1px solid rgba(56,68,89,.6)}
|
||||
.mp-title{font-size:14px;font-weight:600;color:#e0e0e0}
|
||||
.mp-badge{background:rgba(102,126,234,.2);color:#8ea4f0;font-size:11px;font-weight:600;padding:2px 8px;border-radius:10px;border:1px solid rgba(102,126,234,.3)}
|
||||
.mp-error{background:rgba(220,53,69,.15);color:#f5a0a8;border:1px solid rgba(220,53,69,.3);border-radius:4px;padding:8px 12px;margin:10px 12px 0;font-size:12px}
|
||||
.mp-active-card{margin:12px;padding:12px;background:rgba(13,17,23,.8);border:1px solid rgba(56,68,89,.6);border-left:3px solid #28a745;border-radius:6px}
|
||||
.mp-active-name{font-size:14px;font-weight:600;color:#c8d0dc;margin-bottom:6px}
|
||||
.mp-active-meta{display:flex;gap:6px;flex-wrap:wrap;margin-bottom:8px}
|
||||
.mp-active-stats{font-size:12px;color:#8899aa;margin-bottom:10px}
|
||||
.mp-stat-label{color:#8899aa}.mp-stat-value{color:#c8d0dc;font-weight:500}.mp-stat-sep{color:rgba(56,68,89,.8);margin:0 6px}
|
||||
.mp-lora-row{display:flex;align-items:center;gap:8px;margin-bottom:10px}
|
||||
.mp-lora-label{font-size:12px;color:#8899aa}
|
||||
.mp-lora-select{flex:1;padding:4px 8px;background:rgba(30,40,60,.8);border:1px solid rgba(56,68,89,.6);border-radius:4px;color:#c8d0dc;font-size:12px}
|
||||
.mp-list-section{padding:0 12px 12px}
|
||||
.mp-section-title{font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.5px;color:#8899aa;padding:10px 0 8px}
|
||||
.mp-model-card{padding:10px;margin-bottom:8px;background:rgba(13,17,23,.6);border:1px solid rgba(56,68,89,.4);border-radius:6px;transition:border-color .2s}
|
||||
.mp-model-card:hover{border-color:rgba(102,126,234,.4)}
|
||||
.mp-card-name{font-size:13px;font-weight:500;color:#c8d0dc;margin-bottom:4px}
|
||||
.mp-card-meta{display:flex;gap:6px;flex-wrap:wrap;margin-bottom:8px}
|
||||
.mp-meta-tag{background:rgba(30,40,60,.8);color:#8899aa;font-size:10px;padding:2px 6px;border-radius:3px;border:1px solid rgba(56,68,89,.4)}
|
||||
.mp-card-actions{display:flex;gap:6px}
|
||||
.mp-empty{color:#6b7a8d;font-size:12px;padding:16px 0;text-align:center;line-height:1.5}
|
||||
.mp-footer{padding:10px 12px;border-top:1px solid rgba(56,68,89,.4);display:flex;justify-content:flex-end}
|
||||
.mp-btn{padding:5px 12px;border-radius:4px;font-size:12px;font-weight:500;cursor:pointer;border:1px solid transparent;transition:all .15s}
|
||||
.mp-btn:disabled{opacity:.5;cursor:not-allowed}
|
||||
.mp-btn-success{background:rgba(40,167,69,.2);color:#51cf66;border-color:rgba(40,167,69,.3)}
|
||||
.mp-btn-success:hover:not(:disabled){background:rgba(40,167,69,.35)}
|
||||
.mp-btn-danger{background:rgba(220,53,69,.2);color:#ff6b6b;border-color:rgba(220,53,69,.3)}
|
||||
.mp-btn-danger:hover:not(:disabled){background:rgba(220,53,69,.35)}
|
||||
.mp-btn-secondary{background:rgba(30,40,60,.8);color:#b0b8c8;border-color:rgba(56,68,89,.6)}
|
||||
.mp-btn-secondary:hover:not(:disabled){background:rgba(40,50,75,.9)}
|
||||
.mp-btn-muted{background:transparent;color:#6b7a8d;border-color:rgba(56,68,89,.4);font-size:11px;padding:4px 8px}
|
||||
.mp-btn-muted:hover:not(:disabled){color:#ff6b6b;border-color:rgba(220,53,69,.3)}
|
||||
`;
|
||||
|
||||
export default class ModelPanel {
|
||||
constructor(container) {
|
||||
this.container = typeof container === 'string'
|
||||
? document.getElementById(container) : container;
|
||||
if (!this.container) throw new Error('ModelPanel: container element not found');
|
||||
|
||||
this.state = { models: [], activeModel: null, loraProfiles: [], loading: false, error: null };
|
||||
this.unsubs = [];
|
||||
this._injectStyles();
|
||||
this.render();
|
||||
this.refresh();
|
||||
this.unsubs.push(
|
||||
modelService.on('model-loaded', () => this.refresh()),
|
||||
modelService.on('model-unloaded', () => this.refresh()),
|
||||
modelService.on('lora-activated', () => this.refresh())
|
||||
);
|
||||
}
|
||||
|
||||
// --- Data ---
|
||||
|
||||
async refresh() {
|
||||
this._set({ loading: true, error: null });
|
||||
try {
|
||||
const [listRes, active] = await Promise.all([
|
||||
modelService.listModels().catch(() => ({ models: [] })),
|
||||
modelService.getActiveModel().catch(() => null)
|
||||
]);
|
||||
let lora = [];
|
||||
if (active) lora = await modelService.getLoraProfiles().catch(() => []);
|
||||
this._set({ models: listRes?.models ?? [], activeModel: active, loraProfiles: lora, loading: false });
|
||||
} catch (e) { this._set({ loading: false, error: e.message }); }
|
||||
}
|
||||
|
||||
// --- Actions ---
|
||||
|
||||
async _load(id) {
|
||||
this._set({ loading: true, error: null });
|
||||
try { await modelService.loadModel(id); await this.refresh(); }
|
||||
catch (e) { this._set({ loading: false, error: `Load failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _unload() {
|
||||
this._set({ loading: true, error: null });
|
||||
try { await modelService.unloadModel(); await this.refresh(); }
|
||||
catch (e) { this._set({ loading: false, error: `Unload failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _delete(id) {
|
||||
this._set({ loading: true, error: null });
|
||||
try { await modelService.deleteModel(id); await this.refresh(); }
|
||||
catch (e) { this._set({ loading: false, error: `Delete failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _loraChange(modelId, profile) {
|
||||
if (!profile) return;
|
||||
this._set({ loading: true, error: null });
|
||||
try { await modelService.activateLoraProfile(modelId, profile); await this.refresh(); }
|
||||
catch (e) { this._set({ loading: false, error: `LoRA failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
_set(p) { Object.assign(this.state, p); this.render(); }
|
||||
|
||||
// --- Render ---
|
||||
|
||||
render() {
|
||||
const el = this.container;
|
||||
el.innerHTML = '';
|
||||
const panel = this._el('div', 'mp-panel');
|
||||
|
||||
// Header
|
||||
const hdr = this._el('div', 'mp-header');
|
||||
hdr.appendChild(this._el('span', 'mp-title', 'Model Library'));
|
||||
hdr.appendChild(this._el('span', 'mp-badge', String(this.state.models.length)));
|
||||
panel.appendChild(hdr);
|
||||
|
||||
if (this.state.error) panel.appendChild(this._el('div', 'mp-error', this.state.error));
|
||||
|
||||
// Active model
|
||||
if (this.state.activeModel) panel.appendChild(this._renderActive());
|
||||
|
||||
// List
|
||||
const ls = this._el('div', 'mp-list-section');
|
||||
ls.appendChild(this._el('div', 'mp-section-title', 'Available Models'));
|
||||
const models = this.state.models.filter(
|
||||
m => !(this.state.activeModel && this.state.activeModel.model_id === m.id)
|
||||
);
|
||||
if (models.length === 0 && !this.state.loading) {
|
||||
ls.appendChild(this._el('div', 'mp-empty', 'No .rvf models found. Train a model or place .rvf files in data/models/'));
|
||||
} else {
|
||||
models.forEach(m => ls.appendChild(this._renderCard(m)));
|
||||
}
|
||||
panel.appendChild(ls);
|
||||
|
||||
// Footer
|
||||
const ft = this._el('div', 'mp-footer');
|
||||
const rb = this._btn('Refresh', 'mp-btn mp-btn-secondary', () => this.refresh());
|
||||
rb.disabled = this.state.loading;
|
||||
ft.appendChild(rb);
|
||||
panel.appendChild(ft);
|
||||
|
||||
el.appendChild(panel);
|
||||
}
|
||||
|
||||
_renderActive() {
|
||||
const am = this.state.activeModel;
|
||||
const card = this._el('div', 'mp-active-card');
|
||||
card.appendChild(this._el('div', 'mp-active-name', am.model_id || 'Active Model'));
|
||||
|
||||
const full = this.state.models.find(m => m.id === am.model_id);
|
||||
if (full) {
|
||||
const meta = this._el('div', 'mp-active-meta');
|
||||
if (full.version) meta.appendChild(this._tag('v' + full.version));
|
||||
if (full.pck_score != null) meta.appendChild(this._tag('PCK ' + (full.pck_score * 100).toFixed(1) + '%'));
|
||||
card.appendChild(meta);
|
||||
}
|
||||
|
||||
if (am.avg_inference_ms != null) {
|
||||
const st = this._el('div', 'mp-active-stats');
|
||||
st.innerHTML = `<span class="mp-stat-label">Inference:</span> <span class="mp-stat-value">${am.avg_inference_ms.toFixed(1)} ms</span><span class="mp-stat-sep">|</span><span class="mp-stat-label">Frames:</span> <span class="mp-stat-value">${am.frames_processed ?? 0}</span>`;
|
||||
card.appendChild(st);
|
||||
}
|
||||
|
||||
if (this.state.loraProfiles.length > 0) {
|
||||
const row = this._el('div', 'mp-lora-row');
|
||||
row.appendChild(this._el('span', 'mp-lora-label', 'LoRA Profile:'));
|
||||
const sel = document.createElement('select');
|
||||
sel.className = 'mp-lora-select';
|
||||
const def = document.createElement('option');
|
||||
def.value = ''; def.textContent = '-- none --'; sel.appendChild(def);
|
||||
this.state.loraProfiles.forEach(p => {
|
||||
const o = document.createElement('option');
|
||||
o.value = p; o.textContent = p; sel.appendChild(o);
|
||||
});
|
||||
sel.addEventListener('change', () => this._loraChange(am.model_id, sel.value));
|
||||
row.appendChild(sel);
|
||||
card.appendChild(row);
|
||||
}
|
||||
|
||||
const ub = this._btn('Unload', 'mp-btn mp-btn-danger', () => this._unload());
|
||||
ub.disabled = this.state.loading;
|
||||
card.appendChild(ub);
|
||||
return card;
|
||||
}
|
||||
|
||||
_renderCard(model) {
|
||||
const card = this._el('div', 'mp-model-card');
|
||||
card.appendChild(this._el('div', 'mp-card-name', model.filename || model.id));
|
||||
const meta = this._el('div', 'mp-card-meta');
|
||||
if (model.version) meta.appendChild(this._tag('v' + model.version));
|
||||
if (model.size_bytes != null) meta.appendChild(this._tag(this._fmtB(model.size_bytes)));
|
||||
if (model.pck_score != null) meta.appendChild(this._tag('PCK ' + (model.pck_score * 100).toFixed(1) + '%'));
|
||||
if (model.lora_profiles && model.lora_profiles.length > 0) meta.appendChild(this._tag(model.lora_profiles.length + ' LoRA'));
|
||||
card.appendChild(meta);
|
||||
|
||||
const acts = this._el('div', 'mp-card-actions');
|
||||
const lb = this._btn('Load', 'mp-btn mp-btn-success', () => this._load(model.id));
|
||||
lb.disabled = this.state.loading;
|
||||
const db = this._btn('Delete', 'mp-btn mp-btn-muted', () => this._delete(model.id));
|
||||
db.disabled = this.state.loading;
|
||||
acts.appendChild(lb); acts.appendChild(db);
|
||||
card.appendChild(acts);
|
||||
return card;
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
_el(tag, cls, txt) { const e = document.createElement(tag); if (cls) e.className = cls; if (txt != null) e.textContent = txt; return e; }
|
||||
_btn(txt, cls, fn) { const b = document.createElement('button'); b.className = cls; b.textContent = txt; b.addEventListener('click', fn); return b; }
|
||||
_tag(txt) { return this._el('span', 'mp-meta-tag', txt); }
|
||||
_fmtB(b) { return b < 1024 ? b + ' B' : b < 1048576 ? (b / 1024).toFixed(1) + ' KB' : (b / 1048576).toFixed(1) + ' MB'; }
|
||||
|
||||
_injectStyles() {
|
||||
if (document.getElementById('model-panel-styles')) return;
|
||||
const s = document.createElement('style');
|
||||
s.id = 'model-panel-styles';
|
||||
s.textContent = MP_STYLES;
|
||||
document.head.appendChild(s);
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.unsubs.forEach(fn => fn());
|
||||
this.unsubs = [];
|
||||
if (this.container) this.container.innerHTML = '';
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.destroy();
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,12 @@ export class PoseDetectionCanvas {
|
||||
|
||||
// Initialize settings panel
|
||||
this.settingsPanel = null;
|
||||
|
||||
|
||||
// Pose trail state
|
||||
this.poseTrail = [];
|
||||
this.showTrail = false;
|
||||
this.maxTrailLength = 10;
|
||||
|
||||
// Initialize component
|
||||
this.initializeComponent();
|
||||
}
|
||||
@@ -88,22 +93,19 @@ export class PoseDetectionCanvas {
|
||||
<span class="status-text" id="status-text-${this.containerId}">Disconnected</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pose-canvas-controls" id="controls-${this.containerId}">
|
||||
<div class="control-group primary-controls">
|
||||
<button class="btn btn-start" id="start-btn-${this.containerId}">Start</button>
|
||||
<button class="btn btn-stop" id="stop-btn-${this.containerId}" disabled>Stop</button>
|
||||
<button class="btn btn-reconnect" id="reconnect-btn-${this.containerId}" disabled>Reconnect</button>
|
||||
<button class="btn btn-demo" id="demo-btn-${this.containerId}">Demo</button>
|
||||
</div>
|
||||
<div class="control-group secondary-controls">
|
||||
<select class="mode-select" id="mode-select-${this.containerId}">
|
||||
<option value="skeleton">Skeleton</option>
|
||||
<option value="keypoints">Keypoints</option>
|
||||
<option value="heatmap">Heatmap</option>
|
||||
<option value="dense">Dense</option>
|
||||
</select>
|
||||
<button class="btn btn-settings" id="settings-btn-${this.containerId}">⚙️ Settings</button>
|
||||
</div>
|
||||
<div class="pose-canvas-controls" id="controls-${this.containerId}" ${!this.config.enableControls ? 'style="display:none"' : ''}>
|
||||
<button class="btn btn-start" id="start-btn-${this.containerId}">▶ Start</button>
|
||||
<button class="btn btn-stop" id="stop-btn-${this.containerId}" disabled>■ Stop</button>
|
||||
<button class="btn btn-reconnect" id="reconnect-btn-${this.containerId}" disabled>↻ Reconnect</button>
|
||||
<button class="btn btn-demo" id="demo-btn-${this.containerId}">⚙ Demo</button>
|
||||
<select class="mode-select" id="mode-select-${this.containerId}">
|
||||
<option value="skeleton">Skeleton</option>
|
||||
<option value="keypoints">Keypoints</option>
|
||||
<option value="heatmap">Heatmap</option>
|
||||
<option value="dense">Dense</option>
|
||||
</select>
|
||||
<button class="btn btn-trail" id="trail-btn-${this.containerId}">◌ Trail</button>
|
||||
<button class="btn btn-settings" id="settings-btn-${this.containerId}">⚙ Settings</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pose-canvas-container">
|
||||
@@ -124,20 +126,20 @@ export class PoseDetectionCanvas {
|
||||
const style = document.createElement('style');
|
||||
style.textContent = `
|
||||
.pose-detection-canvas-wrapper {
|
||||
border: 1px solid #ddd;
|
||||
border: 1px solid rgba(255, 255, 255, 0.06);
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
background: #f9f9f9;
|
||||
font-family: Arial, sans-serif;
|
||||
background: #0d1117;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
}
|
||||
|
||||
.pose-canvas-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px 15px;
|
||||
background: #f0f0f0;
|
||||
border-bottom: 1px solid #ddd;
|
||||
padding: 12px 16px;
|
||||
background: rgba(15, 20, 35, 0.95);
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
|
||||
}
|
||||
|
||||
.pose-canvas-title {
|
||||
@@ -148,156 +150,185 @@ export class PoseDetectionCanvas {
|
||||
|
||||
.pose-canvas-title h3 {
|
||||
margin: 0;
|
||||
color: #333;
|
||||
color: #e0e0e0;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.connection-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
gap: 6px;
|
||||
padding: 4px 10px;
|
||||
background: rgba(30, 40, 60, 0.6);
|
||||
border-radius: 12px;
|
||||
border: 1px solid rgba(255, 255, 255, 0.06);
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #ccc;
|
||||
background: #4a5568;
|
||||
transition: background-color 0.3s;
|
||||
}
|
||||
|
||||
.status-indicator.connected { background: #28a745; }
|
||||
.status-indicator.connecting { background: #ffc107; }
|
||||
.status-indicator.error { background: #dc3545; }
|
||||
.status-indicator.disconnected { background: #6c757d; }
|
||||
.status-indicator.connected { background: #00cc88; box-shadow: 0 0 6px rgba(0, 204, 136, 0.5); }
|
||||
.status-indicator.connecting { background: #fbbf24; box-shadow: 0 0 6px rgba(251, 191, 36, 0.5); animation: pulse 1.5s ease-in-out infinite; }
|
||||
.status-indicator.error { background: #ef4444; box-shadow: 0 0 6px rgba(239, 68, 68, 0.5); }
|
||||
.status-indicator.disconnected { background: #4a5568; }
|
||||
|
||||
.status-text {
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
min-width: 80px;
|
||||
font-size: 11px;
|
||||
color: #8899aa;
|
||||
min-width: 70px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.pose-canvas-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 15px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.control-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.primary-controls {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.secondary-controls {
|
||||
flex-shrink: 0;
|
||||
flex-wrap: nowrap;
|
||||
}
|
||||
|
||||
.btn {
|
||||
padding: 8px 16px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 6px;
|
||||
background: #ffffff;
|
||||
color: #333333;
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
border-radius: 8px;
|
||||
background: rgba(30, 40, 60, 0.8);
|
||||
color: #c8d0dc;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
transition: all 0.2s ease;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
min-width: 80px;
|
||||
text-align: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.btn:hover:not(:disabled) {
|
||||
background: #f8f9fa;
|
||||
border-color: #adb5bd;
|
||||
box-shadow: 0 2px 6px rgba(0,0,0,0.15);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.btn:active:not(:disabled) {
|
||||
transform: translateY(0);
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
.btn:disabled {
|
||||
opacity: 0.6;
|
||||
opacity: 0.35;
|
||||
cursor: not-allowed;
|
||||
background: #e9ecef;
|
||||
color: #6c757d;
|
||||
background: rgba(20, 30, 50, 0.6);
|
||||
color: #4a5568;
|
||||
transform: none !important;
|
||||
box-shadow: none !important;
|
||||
}
|
||||
|
||||
.btn-start {
|
||||
background: #28a745;
|
||||
color: white;
|
||||
border-color: #28a745;
|
||||
.btn-start {
|
||||
background: rgba(0, 204, 136, 0.15);
|
||||
color: #00cc88;
|
||||
border-color: rgba(0, 204, 136, 0.3);
|
||||
}
|
||||
|
||||
.btn-start:hover:not(:disabled) {
|
||||
background: #218838;
|
||||
border-color: #1e7e34;
|
||||
.btn-start:hover:not(:disabled) {
|
||||
background: rgba(0, 204, 136, 0.25);
|
||||
border-color: rgba(0, 204, 136, 0.5);
|
||||
box-shadow: 0 4px 12px rgba(0, 204, 136, 0.2);
|
||||
}
|
||||
|
||||
.btn-stop {
|
||||
background: #dc3545;
|
||||
color: white;
|
||||
border-color: #dc3545;
|
||||
.btn-stop {
|
||||
background: rgba(239, 68, 68, 0.15);
|
||||
color: #ef4444;
|
||||
border-color: rgba(239, 68, 68, 0.3);
|
||||
}
|
||||
|
||||
.btn-stop:hover:not(:disabled) {
|
||||
background: #c82333;
|
||||
border-color: #bd2130;
|
||||
.btn-stop:hover:not(:disabled) {
|
||||
background: rgba(239, 68, 68, 0.25);
|
||||
border-color: rgba(239, 68, 68, 0.5);
|
||||
box-shadow: 0 4px 12px rgba(239, 68, 68, 0.2);
|
||||
}
|
||||
|
||||
.btn-reconnect {
|
||||
background: #17a2b8;
|
||||
color: white;
|
||||
border-color: #17a2b8;
|
||||
.btn-reconnect {
|
||||
background: rgba(59, 130, 246, 0.15);
|
||||
color: #60a5fa;
|
||||
border-color: rgba(59, 130, 246, 0.3);
|
||||
}
|
||||
|
||||
.btn-reconnect:hover:not(:disabled) {
|
||||
background: #138496;
|
||||
border-color: #117a8b;
|
||||
.btn-reconnect:hover:not(:disabled) {
|
||||
background: rgba(59, 130, 246, 0.25);
|
||||
border-color: rgba(59, 130, 246, 0.5);
|
||||
box-shadow: 0 4px 12px rgba(59, 130, 246, 0.2);
|
||||
}
|
||||
|
||||
.btn-demo {
|
||||
background: #6f42c1;
|
||||
color: white;
|
||||
border-color: #6f42c1;
|
||||
.btn-demo {
|
||||
background: rgba(139, 92, 246, 0.15);
|
||||
color: #a78bfa;
|
||||
border-color: rgba(139, 92, 246, 0.3);
|
||||
}
|
||||
|
||||
.btn-demo:hover:not(:disabled) {
|
||||
background: #5a32a3;
|
||||
border-color: #512a97;
|
||||
.btn-demo:hover:not(:disabled) {
|
||||
background: rgba(139, 92, 246, 0.25);
|
||||
border-color: rgba(139, 92, 246, 0.5);
|
||||
box-shadow: 0 4px 12px rgba(139, 92, 246, 0.2);
|
||||
}
|
||||
|
||||
.btn-settings {
|
||||
background: #6c757d;
|
||||
color: white;
|
||||
border-color: #6c757d;
|
||||
.btn-settings {
|
||||
background: rgba(100, 116, 139, 0.15);
|
||||
color: #94a3b8;
|
||||
border-color: rgba(100, 116, 139, 0.3);
|
||||
}
|
||||
|
||||
.btn-settings:hover:not(:disabled) {
|
||||
background: #5a6268;
|
||||
border-color: #545b62;
|
||||
.btn-settings:hover:not(:disabled) {
|
||||
background: rgba(100, 116, 139, 0.25);
|
||||
border-color: rgba(100, 116, 139, 0.5);
|
||||
}
|
||||
|
||||
.btn-trail {
|
||||
background: rgba(0, 212, 255, 0.1);
|
||||
color: #5ec4d4;
|
||||
border-color: rgba(0, 212, 255, 0.25);
|
||||
}
|
||||
|
||||
.btn-trail:hover:not(:disabled) {
|
||||
background: rgba(0, 212, 255, 0.2);
|
||||
border-color: rgba(0, 212, 255, 0.45);
|
||||
box-shadow: 0 4px 12px rgba(0, 212, 255, 0.15);
|
||||
}
|
||||
|
||||
.btn-trail.active {
|
||||
background: rgba(0, 212, 255, 0.2);
|
||||
color: #00d4ff;
|
||||
border-color: rgba(0, 212, 255, 0.5);
|
||||
box-shadow: 0 0 8px rgba(0, 212, 255, 0.2);
|
||||
}
|
||||
|
||||
.mode-select {
|
||||
padding: 5px 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
background: #fff;
|
||||
font-size: 12px;
|
||||
padding: 8px 12px;
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
border-radius: 8px;
|
||||
background: rgba(30, 40, 60, 0.8);
|
||||
color: #b0b8c8;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.mode-select:focus {
|
||||
outline: none;
|
||||
border-color: rgba(139, 92, 246, 0.5);
|
||||
box-shadow: 0 0 0 2px rgba(139, 92, 246, 0.15);
|
||||
}
|
||||
|
||||
.mode-select option {
|
||||
background: #1a2234;
|
||||
color: #c8d0dc;
|
||||
}
|
||||
|
||||
.pose-canvas-container {
|
||||
@@ -410,6 +441,10 @@ export class PoseDetectionCanvas {
|
||||
const demoBtn = document.getElementById(`demo-btn-${this.containerId}`);
|
||||
demoBtn.addEventListener('click', () => this.toggleDemo());
|
||||
|
||||
// Trail toggle button
|
||||
const trailBtn = document.getElementById(`trail-btn-${this.containerId}`);
|
||||
trailBtn.addEventListener('click', () => this.toggleTrail());
|
||||
|
||||
// Settings button
|
||||
const settingsBtn = document.getElementById(`settings-btn-${this.containerId}`);
|
||||
settingsBtn.addEventListener('click', () => this.showSettings());
|
||||
@@ -439,6 +474,7 @@ export class PoseDetectionCanvas {
|
||||
case 'pose_update':
|
||||
this.state.lastPoseData = update.data;
|
||||
this.state.frameCount++;
|
||||
this.updateTrail(update.data);
|
||||
this.renderPoseData(update.data);
|
||||
this.updateStats();
|
||||
this.notifyCallback('onPoseUpdate', update.data);
|
||||
@@ -481,14 +517,40 @@ export class PoseDetectionCanvas {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Render trail before the current frame if enabled
|
||||
if (this.showTrail && this.poseTrail.length > 1) {
|
||||
// The renderer.render() clears the canvas, so we render trail
|
||||
// by hooking into the renderer's canvas context after clear.
|
||||
// We override the render flow: clear, trail, then current.
|
||||
this.renderer.clearCanvas();
|
||||
this.renderTrail(this.renderer.ctx);
|
||||
// Now render current frame without clearing again
|
||||
this.renderCurrentFrameNoClean(poseData);
|
||||
} else {
|
||||
this.renderer.render(poseData, {
|
||||
frameCount: this.state.frameCount,
|
||||
connectionState: this.state.connectionState
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Render error', { error: error.message });
|
||||
this.showError(`Render error: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
renderCurrentFrameNoClean(poseData) {
|
||||
// Call the renderer's render logic without clearing the canvas.
|
||||
// We temporarily stub clearCanvas, render, then restore.
|
||||
const origClear = this.renderer.clearCanvas.bind(this.renderer);
|
||||
this.renderer.clearCanvas = () => {}; // no-op
|
||||
try {
|
||||
this.renderer.render(poseData, {
|
||||
frameCount: this.state.frameCount,
|
||||
connectionState: this.state.connectionState
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Render error', { error: error.message });
|
||||
this.showError(`Render error: ${error.message}`);
|
||||
} finally {
|
||||
this.renderer.clearCanvas = origClear;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -644,6 +706,104 @@ export class PoseDetectionCanvas {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Pose Trail Methods ---
|
||||
|
||||
toggleTrail() {
|
||||
this.showTrail = !this.showTrail;
|
||||
const trailBtn = document.getElementById(`trail-btn-${this.containerId}`);
|
||||
if (trailBtn) {
|
||||
trailBtn.classList.toggle('active', this.showTrail);
|
||||
trailBtn.textContent = this.showTrail ? '\u25CB Trail On' : '\u25CB Trail';
|
||||
}
|
||||
if (!this.showTrail) {
|
||||
this.poseTrail = [];
|
||||
}
|
||||
this.logger.info('Trail toggled', { showTrail: this.showTrail });
|
||||
}
|
||||
|
||||
updateTrail(poseData) {
|
||||
if (!this.showTrail) return;
|
||||
if (!poseData || !poseData.persons || poseData.persons.length === 0) return;
|
||||
|
||||
// Deep clone the keypoints from all persons for this frame
|
||||
const frameKeypoints = poseData.persons.map(person => {
|
||||
if (!person.keypoints) return null;
|
||||
return person.keypoints.map(kp => ({
|
||||
x: kp.x,
|
||||
y: kp.y,
|
||||
confidence: kp.confidence
|
||||
}));
|
||||
}).filter(Boolean);
|
||||
|
||||
if (frameKeypoints.length > 0) {
|
||||
this.poseTrail.push(frameKeypoints);
|
||||
if (this.poseTrail.length > this.maxTrailLength) {
|
||||
this.poseTrail.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
renderTrail(ctx) {
|
||||
if (!this.poseTrail || this.poseTrail.length < 2) return;
|
||||
|
||||
const totalFrames = this.poseTrail.length;
|
||||
|
||||
// Keypoint color palette (same as renderer's body part colors)
|
||||
const kpColors = [
|
||||
'#ff0000', '#ff4500', '#ffa500', '#ffff00', '#adff2f',
|
||||
'#00ff00', '#00ff7f', '#00ffff', '#0080ff', '#0000ff',
|
||||
'#4000ff', '#8000ff', '#ff00ff', '#ff0080', '#ff0040',
|
||||
'#ff8080', '#ffb380'
|
||||
];
|
||||
|
||||
// Render ghosted keypoints and trajectory lines for each frame in the trail
|
||||
// (skip the last frame since it's the current one rendered by the normal pipeline)
|
||||
for (let frameIdx = 0; frameIdx < totalFrames - 1; frameIdx++) {
|
||||
const alpha = 0.1 + (frameIdx / totalFrames) * 0.7;
|
||||
const framePersons = this.poseTrail[frameIdx];
|
||||
const nextFramePersons = this.poseTrail[frameIdx + 1];
|
||||
|
||||
framePersons.forEach((personKeypoints, personIdx) => {
|
||||
if (!personKeypoints) return;
|
||||
|
||||
personKeypoints.forEach((kp, kpIdx) => {
|
||||
if (kp.confidence <= 0.1) return;
|
||||
|
||||
const x = this.renderer.scaleX(kp.x);
|
||||
const y = this.renderer.scaleY(kp.y);
|
||||
const color = kpColors[kpIdx % kpColors.length];
|
||||
|
||||
// Draw ghosted keypoint dot
|
||||
ctx.globalAlpha = alpha * 0.6;
|
||||
ctx.fillStyle = color;
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, 2.5, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
// Draw trajectory line to same keypoint in next frame
|
||||
if (nextFramePersons && nextFramePersons[personIdx]) {
|
||||
const nextKp = nextFramePersons[personIdx][kpIdx];
|
||||
if (nextKp && nextKp.confidence > 0.1) {
|
||||
const nx = this.renderer.scaleX(nextKp.x);
|
||||
const ny = this.renderer.scaleY(nextKp.y);
|
||||
|
||||
ctx.globalAlpha = alpha * 0.4;
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 1;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x, y);
|
||||
ctx.lineTo(nx, ny);
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Reset alpha
|
||||
ctx.globalAlpha = 1.0;
|
||||
}
|
||||
|
||||
// Toggle demo mode
|
||||
toggleDemo() {
|
||||
if (this.demoState && this.demoState.isRunning) {
|
||||
|
||||
@@ -33,6 +33,13 @@ export class SensingTab {
|
||||
_buildDOM() {
|
||||
this.container.innerHTML = `
|
||||
<h2>Live WiFi Sensing</h2>
|
||||
|
||||
<!-- Data-source status banner — updated by _onStateChange -->
|
||||
<div id="sensingSourceBanner" class="sensing-source-banner sensing-source-reconnecting"
|
||||
role="status" aria-live="polite">
|
||||
RECONNECTING...
|
||||
</div>
|
||||
|
||||
<div class="sensing-layout">
|
||||
<!-- 3D viewport -->
|
||||
<div class="sensing-viewport" id="sensingViewport">
|
||||
@@ -98,6 +105,17 @@ export class SensingTab {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Setup info -->
|
||||
<div class="sensing-card">
|
||||
<div class="sensing-card-title">About This Data</div>
|
||||
<p class="sensing-about-text">
|
||||
Metrics are computed from WiFi Channel State Information (CSI).
|
||||
With <strong>1 ESP32</strong> you get presence detection, breathing
|
||||
estimation, and gross motion. Add <strong>3-4+ ESP32 nodes</strong>
|
||||
around the room for spatial resolution and limb-level tracking.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Extra info -->
|
||||
<div class="sensing-card">
|
||||
<div class="sensing-card-title">Details</div>
|
||||
@@ -178,19 +196,35 @@ export class SensingTab {
|
||||
}
|
||||
|
||||
_onStateChange(state) {
|
||||
const dot = this.container.querySelector('#sensingDot');
|
||||
const text = this.container.querySelector('#sensingState');
|
||||
if (!dot || !text) return;
|
||||
const dot = this.container.querySelector('#sensingDot');
|
||||
const text = this.container.querySelector('#sensingState');
|
||||
const banner = this.container.querySelector('#sensingSourceBanner');
|
||||
|
||||
const labels = {
|
||||
disconnected: 'Disconnected',
|
||||
connecting: 'Connecting...',
|
||||
connected: 'Connected',
|
||||
simulated: 'Simulated',
|
||||
};
|
||||
if (dot && text) {
|
||||
const stateLabels = {
|
||||
disconnected: 'Disconnected',
|
||||
connecting: 'Connecting...',
|
||||
connected: 'Connected',
|
||||
reconnecting: 'Reconnecting...',
|
||||
simulated: 'Simulated',
|
||||
};
|
||||
dot.className = 'sensing-dot ' + state;
|
||||
text.textContent = stateLabels[state] || state;
|
||||
}
|
||||
|
||||
dot.className = 'sensing-dot ' + state;
|
||||
text.textContent = labels[state] || state;
|
||||
if (banner) {
|
||||
// Map the service's dataSource to banner text and CSS modifier class.
|
||||
const dataSource = sensingService.dataSource;
|
||||
const bannerConfig = {
|
||||
'live': { text: 'LIVE \u2014 ESP32 HARDWARE', cls: 'sensing-source-live' },
|
||||
'server-simulated': { text: 'SIMULATED \u2014 NO HARDWARE', cls: 'sensing-source-server-sim' },
|
||||
'reconnecting': { text: 'RECONNECTING...', cls: 'sensing-source-reconnecting' },
|
||||
'simulated': { text: 'OFFLINE \u2014 CLIENT SIMULATION', cls: 'sensing-source-simulated' },
|
||||
};
|
||||
const cfg = bannerConfig[dataSource] || bannerConfig.reconnecting;
|
||||
banner.textContent = cfg.text;
|
||||
banner.className = 'sensing-source-banner ' + cfg.cls;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- HUD update --------------------------------------------------------
|
||||
@@ -223,7 +257,8 @@ export class SensingTab {
|
||||
// Details
|
||||
this._setText('valDomFreq', (f.dominant_freq_hz || 0).toFixed(3) + ' Hz');
|
||||
this._setText('valChangePoints', String(f.change_points || 0));
|
||||
this._setText('valSampleRate', data.source === 'simulated' ? 'sim' : 'live');
|
||||
const srcLabel = (data.source === 'simulated' || data.source === 'simulate') ? 'sim' : data.source || 'live';
|
||||
this._setText('valSampleRate', srcLabel);
|
||||
|
||||
// Sparkline
|
||||
this._drawSparkline();
|
||||
|
||||
@@ -55,7 +55,23 @@ export class SettingsPanel {
|
||||
// Advanced settings
|
||||
heartbeatInterval: 30000,
|
||||
maxReconnectAttempts: 10,
|
||||
enableSmoothing: true
|
||||
enableSmoothing: true,
|
||||
|
||||
// Model settings
|
||||
defaultModelPath: 'data/models/',
|
||||
autoLoadModel: false,
|
||||
inferenceDevice: 'CPU',
|
||||
inferenceThreads: 4,
|
||||
progressiveLoading: true,
|
||||
|
||||
// Training settings
|
||||
defaultEpochs: 100,
|
||||
defaultBatchSize: 32,
|
||||
defaultLearningRate: 0.0003,
|
||||
earlyStoppingPatience: 15,
|
||||
checkpointDirectory: 'data/models/',
|
||||
autoExportOnCompletion: true,
|
||||
recordingDirectory: 'data/recordings/'
|
||||
};
|
||||
|
||||
this.callbacks = {
|
||||
@@ -245,6 +261,67 @@ export class SettingsPanel {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Model Settings -->
|
||||
<div class="settings-section">
|
||||
<h4>Model Configuration</h4>
|
||||
<div class="setting-row">
|
||||
<label for="default-model-path-${this.containerId}">Default Model Path:</label>
|
||||
<input type="text" id="default-model-path-${this.containerId}" class="setting-input setting-input-wide" placeholder="data/models/">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="auto-load-model-${this.containerId}">Auto-load Model on Startup:</label>
|
||||
<input type="checkbox" id="auto-load-model-${this.containerId}" class="setting-checkbox">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="inference-device-${this.containerId}">Inference Device:</label>
|
||||
<select id="inference-device-${this.containerId}" class="setting-select">
|
||||
<option value="CPU">CPU</option>
|
||||
<option value="GPU">GPU</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="inference-threads-${this.containerId}">Inference Threads:</label>
|
||||
<input type="number" id="inference-threads-${this.containerId}" class="setting-input" min="1" max="16">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="progressive-loading-${this.containerId}">Progressive Loading:</label>
|
||||
<input type="checkbox" id="progressive-loading-${this.containerId}" class="setting-checkbox">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Training Settings -->
|
||||
<div class="settings-section">
|
||||
<h4>Training Configuration</h4>
|
||||
<div class="setting-row">
|
||||
<label for="default-epochs-${this.containerId}">Default Epochs:</label>
|
||||
<input type="number" id="default-epochs-${this.containerId}" class="setting-input" min="1" max="10000">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="default-batch-size-${this.containerId}">Default Batch Size:</label>
|
||||
<input type="number" id="default-batch-size-${this.containerId}" class="setting-input" min="1" max="512">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="default-learning-rate-${this.containerId}">Default Learning Rate:</label>
|
||||
<input type="number" id="default-learning-rate-${this.containerId}" class="setting-input" min="0.000001" max="1" step="0.0001">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="early-stopping-patience-${this.containerId}">Early Stopping Patience:</label>
|
||||
<input type="number" id="early-stopping-patience-${this.containerId}" class="setting-input" min="1" max="100">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="checkpoint-directory-${this.containerId}">Checkpoint Directory:</label>
|
||||
<input type="text" id="checkpoint-directory-${this.containerId}" class="setting-input setting-input-wide" placeholder="data/models/">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="auto-export-on-completion-${this.containerId}">Auto-export on Completion:</label>
|
||||
<input type="checkbox" id="auto-export-on-completion-${this.containerId}" class="setting-checkbox">
|
||||
</div>
|
||||
<div class="setting-row">
|
||||
<label for="recording-directory-${this.containerId}">Recording Directory:</label>
|
||||
<input type="text" id="recording-directory-${this.containerId}" class="setting-input setting-input-wide" placeholder="data/recordings/">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="settings-toggle">
|
||||
<button class="btn btn-sm" id="toggle-advanced-${this.containerId}">Show Advanced</button>
|
||||
</div>
|
||||
@@ -267,11 +344,12 @@ export class SettingsPanel {
|
||||
const style = document.createElement('style');
|
||||
style.textContent = `
|
||||
.settings-panel {
|
||||
background: #fff;
|
||||
border: 1px solid #ddd;
|
||||
background: #0d1117;
|
||||
border: 1px solid rgba(56, 68, 89, 0.6);
|
||||
border-radius: 8px;
|
||||
font-family: Arial, sans-serif;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
overflow: hidden;
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
.settings-header {
|
||||
@@ -279,13 +357,13 @@ export class SettingsPanel {
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 15px 20px;
|
||||
background: #f8f9fa;
|
||||
border-bottom: 1px solid #ddd;
|
||||
background: rgba(15, 20, 35, 0.95);
|
||||
border-bottom: 1px solid rgba(56, 68, 89, 0.6);
|
||||
}
|
||||
|
||||
.settings-header h3 {
|
||||
margin: 0;
|
||||
color: #333;
|
||||
color: #e0e0e0;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
@@ -297,26 +375,43 @@ export class SettingsPanel {
|
||||
|
||||
.settings-content {
|
||||
padding: 20px;
|
||||
max-height: 400px;
|
||||
max-height: 500px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.settings-content::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
.settings-content::-webkit-scrollbar-track {
|
||||
background: rgba(15, 20, 35, 0.5);
|
||||
}
|
||||
|
||||
.settings-content::-webkit-scrollbar-thumb {
|
||||
background: rgba(56, 68, 89, 0.8);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.settings-content::-webkit-scrollbar-thumb:hover {
|
||||
background: rgba(80, 96, 120, 0.9);
|
||||
}
|
||||
|
||||
.settings-section {
|
||||
margin-bottom: 25px;
|
||||
padding-bottom: 20px;
|
||||
border-bottom: 1px solid #eee;
|
||||
padding: 16px;
|
||||
background: rgba(17, 24, 39, 0.9);
|
||||
border: 1px solid rgba(56, 68, 89, 0.4);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.settings-section:last-child {
|
||||
border-bottom: none;
|
||||
margin-bottom: 0;
|
||||
padding-bottom: 0;
|
||||
}
|
||||
|
||||
.settings-section h4 {
|
||||
margin: 0 0 15px 0;
|
||||
color: #555;
|
||||
font-size: 14px;
|
||||
color: #8899aa;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
@@ -332,7 +427,7 @@ export class SettingsPanel {
|
||||
|
||||
.setting-row label {
|
||||
flex: 1;
|
||||
color: #666;
|
||||
color: #8899aa;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
}
|
||||
@@ -340,9 +435,26 @@ export class SettingsPanel {
|
||||
.setting-input, .setting-select {
|
||||
flex: 0 0 120px;
|
||||
padding: 6px 8px;
|
||||
border: 1px solid #ddd;
|
||||
border: 1px solid rgba(56, 68, 89, 0.6);
|
||||
border-radius: 4px;
|
||||
font-size: 13px;
|
||||
background: rgba(15, 20, 35, 0.8);
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
.setting-input:focus, .setting-select:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
box-shadow: 0 0 0 2px rgba(102, 126, 234, 0.15);
|
||||
}
|
||||
|
||||
.setting-input-wide {
|
||||
flex: 0 0 160px;
|
||||
}
|
||||
|
||||
.setting-select option {
|
||||
background: #1a2234;
|
||||
color: #c8d0dc;
|
||||
}
|
||||
|
||||
.setting-range {
|
||||
@@ -353,41 +465,45 @@ export class SettingsPanel {
|
||||
.setting-value {
|
||||
flex: 0 0 40px;
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
color: #b0b8c8;
|
||||
text-align: center;
|
||||
background: #f8f9fa;
|
||||
background: rgba(15, 20, 35, 0.8);
|
||||
padding: 2px 6px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid #ddd;
|
||||
border: 1px solid rgba(56, 68, 89, 0.6);
|
||||
}
|
||||
|
||||
.setting-checkbox {
|
||||
flex: 0 0 auto;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: #667eea;
|
||||
}
|
||||
|
||||
.setting-color {
|
||||
flex: 0 0 50px;
|
||||
height: 30px;
|
||||
border: 1px solid #ddd;
|
||||
border: 1px solid rgba(56, 68, 89, 0.6);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
background: rgba(15, 20, 35, 0.8);
|
||||
}
|
||||
|
||||
.btn {
|
||||
padding: 6px 12px;
|
||||
border: 1px solid #ddd;
|
||||
border: 1px solid rgba(56, 68, 89, 0.6);
|
||||
border-radius: 4px;
|
||||
background: #fff;
|
||||
background: rgba(30, 40, 60, 0.8);
|
||||
color: #b0b8c8;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.btn:hover {
|
||||
background: #f8f9fa;
|
||||
border-color: #adb5bd;
|
||||
background: rgba(40, 55, 80, 0.9);
|
||||
border-color: rgba(80, 96, 120, 0.8);
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
.btn-sm {
|
||||
@@ -398,32 +514,32 @@ export class SettingsPanel {
|
||||
.settings-toggle {
|
||||
text-align: center;
|
||||
padding-top: 15px;
|
||||
border-top: 1px solid #eee;
|
||||
border-top: 1px solid rgba(56, 68, 89, 0.4);
|
||||
}
|
||||
|
||||
.settings-footer {
|
||||
padding: 10px 20px;
|
||||
background: #f8f9fa;
|
||||
border-top: 1px solid #ddd;
|
||||
background: rgba(15, 20, 35, 0.95);
|
||||
border-top: 1px solid rgba(56, 68, 89, 0.6);
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.settings-status {
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
color: #6b7a8d;
|
||||
}
|
||||
|
||||
.advanced-section {
|
||||
background: #f9f9f9;
|
||||
background: rgba(20, 28, 45, 0.9);
|
||||
margin: 0 -20px 25px -20px;
|
||||
padding: 20px;
|
||||
border: none;
|
||||
border-top: 1px solid #ddd;
|
||||
border-bottom: 1px solid #ddd;
|
||||
border-top: 1px solid rgba(56, 68, 89, 0.4);
|
||||
border-bottom: 1px solid rgba(56, 68, 89, 0.4);
|
||||
}
|
||||
|
||||
.advanced-section h4 {
|
||||
color: #dc3545;
|
||||
color: #ef4444;
|
||||
}
|
||||
`;
|
||||
|
||||
@@ -492,7 +608,9 @@ export class SettingsPanel {
|
||||
const checkboxes = [
|
||||
'auto-reconnect', 'show-keypoints', 'show-skeleton', 'show-bounding-box',
|
||||
'show-confidence', 'show-zones', 'show-debug-info', 'enable-validation',
|
||||
'enable-performance-tracking', 'enable-debug-logging', 'enable-smoothing'
|
||||
'enable-performance-tracking', 'enable-debug-logging', 'enable-smoothing',
|
||||
'auto-load-model', 'progressive-loading',
|
||||
'auto-export-on-completion'
|
||||
];
|
||||
|
||||
checkboxes.forEach(id => {
|
||||
@@ -503,12 +621,14 @@ export class SettingsPanel {
|
||||
});
|
||||
});
|
||||
|
||||
// Number inputs
|
||||
// Number inputs (integers)
|
||||
const numberInputs = [
|
||||
'connection-timeout', 'max-persons', 'max-fps',
|
||||
'heartbeat-interval', 'max-reconnect-attempts'
|
||||
'connection-timeout', 'max-persons', 'max-fps',
|
||||
'heartbeat-interval', 'max-reconnect-attempts',
|
||||
'inference-threads', 'default-epochs', 'default-batch-size',
|
||||
'early-stopping-patience'
|
||||
];
|
||||
|
||||
|
||||
numberInputs.forEach(id => {
|
||||
const input = document.getElementById(`${id}-${this.containerId}`);
|
||||
input?.addEventListener('change', (e) => {
|
||||
@@ -517,6 +637,32 @@ export class SettingsPanel {
|
||||
});
|
||||
});
|
||||
|
||||
// Float number inputs
|
||||
const floatInputs = ['default-learning-rate'];
|
||||
floatInputs.forEach(id => {
|
||||
const input = document.getElementById(`${id}-${this.containerId}`);
|
||||
input?.addEventListener('change', (e) => {
|
||||
const settingKey = this.camelCase(id);
|
||||
this.updateSetting(settingKey, parseFloat(e.target.value));
|
||||
});
|
||||
});
|
||||
|
||||
// Text inputs
|
||||
const textInputs = ['default-model-path', 'checkpoint-directory', 'recording-directory'];
|
||||
textInputs.forEach(id => {
|
||||
const input = document.getElementById(`${id}-${this.containerId}`);
|
||||
input?.addEventListener('change', (e) => {
|
||||
const settingKey = this.camelCase(id);
|
||||
this.updateSetting(settingKey, e.target.value);
|
||||
});
|
||||
});
|
||||
|
||||
// Inference device select
|
||||
const inferenceDeviceSelect = document.getElementById(`inference-device-${this.containerId}`);
|
||||
inferenceDeviceSelect?.addEventListener('change', (e) => {
|
||||
this.updateSetting('inferenceDevice', e.target.value);
|
||||
});
|
||||
|
||||
// Color inputs
|
||||
const colorInputs = ['skeleton-color', 'keypoint-color', 'bounding-box-color'];
|
||||
colorInputs.forEach(id => {
|
||||
@@ -696,7 +842,19 @@ export class SettingsPanel {
|
||||
enableDebugLogging: false,
|
||||
heartbeatInterval: 30000,
|
||||
maxReconnectAttempts: 10,
|
||||
enableSmoothing: true
|
||||
enableSmoothing: true,
|
||||
defaultModelPath: 'data/models/',
|
||||
autoLoadModel: false,
|
||||
inferenceDevice: 'CPU',
|
||||
inferenceThreads: 4,
|
||||
progressiveLoading: true,
|
||||
defaultEpochs: 100,
|
||||
defaultBatchSize: 32,
|
||||
defaultLearningRate: 0.0003,
|
||||
earlyStoppingPatience: 15,
|
||||
checkpointDirectory: 'data/models/',
|
||||
autoExportOnCompletion: true,
|
||||
recordingDirectory: 'data/recordings/'
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
// TrainingPanel Component for WiFi-DensePose UI
|
||||
// Dark-mode panel for training management, CSI recordings, and progress charts.
|
||||
|
||||
import { trainingService } from '../services/training.service.js';
|
||||
|
||||
const TP_STYLES = `
|
||||
.tp-panel{background:rgba(17,24,39,.9);border:1px solid rgba(56,68,89,.6);border-radius:8px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;color:#e0e0e0;overflow:hidden}
|
||||
.tp-header{display:flex;align-items:center;justify-content:space-between;padding:14px 16px;background:rgba(13,17,23,.95);border-bottom:1px solid rgba(56,68,89,.6)}
|
||||
.tp-title{font-size:14px;font-weight:600;color:#e0e0e0}
|
||||
.tp-badge{font-size:11px;font-weight:600;padding:2px 8px;border-radius:10px}
|
||||
.tp-badge-idle{background:rgba(108,117,125,.2);color:#8899aa;border:1px solid rgba(108,117,125,.3)}
|
||||
.tp-badge-active{background:rgba(40,167,69,.2);color:#51cf66;border:1px solid rgba(40,167,69,.3);animation:tp-pulse 1.5s ease-in-out infinite}
|
||||
.tp-badge-done{background:rgba(102,126,234,.2);color:#8ea4f0;border:1px solid rgba(102,126,234,.3)}
|
||||
@keyframes tp-pulse{0%,100%{opacity:1}50%{opacity:.6}}
|
||||
.tp-error{background:rgba(220,53,69,.15);color:#f5a0a8;border:1px solid rgba(220,53,69,.3);border-radius:4px;padding:8px 12px;margin:10px 12px 0;font-size:12px}
|
||||
.tp-section{padding:12px;border-bottom:1px solid rgba(56,68,89,.3)}
|
||||
.tp-section:last-child{border-bottom:none}
|
||||
.tp-section-title{font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.5px;color:#8899aa;margin-bottom:8px}
|
||||
.tp-empty{color:#6b7a8d;font-size:12px;padding:12px 0;text-align:center}
|
||||
.tp-rec-row{display:flex;align-items:center;justify-content:space-between;padding:6px 8px;margin-bottom:4px;background:rgba(13,17,23,.6);border:1px solid rgba(56,68,89,.3);border-radius:4px}
|
||||
.tp-rec-info{display:flex;flex-direction:column;gap:2px}
|
||||
.tp-rec-name{font-size:12px;color:#c8d0dc;font-weight:500}
|
||||
.tp-rec-meta{font-size:10px;color:#6b7a8d}
|
||||
.tp-rec-actions{margin-top:8px}
|
||||
.tp-config-header{display:flex;align-items:center;justify-content:space-between;margin-bottom:6px}
|
||||
.tp-config-form{display:flex;flex-direction:column;gap:6px}
|
||||
.tp-label{font-size:12px;color:#8899aa;display:block;margin-bottom:2px}
|
||||
.tp-input-row{display:flex;justify-content:space-between;align-items:center;gap:8px}
|
||||
.tp-input-row .tp-label{flex:1;margin-bottom:0}
|
||||
.tp-input{width:110px;padding:4px 8px;background:rgba(30,40,60,.8);border:1px solid rgba(56,68,89,.6);border-radius:4px;color:#c8d0dc;font-size:12px}
|
||||
.tp-input:focus{outline:none;border-color:#667eea}
|
||||
.tp-ds-container{display:flex;flex-direction:column;gap:4px;margin-bottom:4px;max-height:100px;overflow-y:auto}
|
||||
.tp-ds-item{display:flex;align-items:center;gap:6px;font-size:12px;color:#c8d0dc;cursor:pointer}
|
||||
.tp-ds-item input{width:14px;height:14px}
|
||||
.tp-train-actions{display:flex;gap:6px;margin-top:10px}
|
||||
.tp-progress-bar{height:6px;background:rgba(30,40,60,.8);border-radius:3px;overflow:hidden;margin-bottom:4px}
|
||||
.tp-progress-fill{height:100%;background:linear-gradient(90deg,#667eea,#764ba2);border-radius:3px;transition:width .3s}
|
||||
.tp-progress-label{font-size:11px;color:#8899aa;text-align:center;margin-bottom:10px}
|
||||
.tp-chart-row{display:flex;gap:8px;margin-bottom:10px;flex-wrap:wrap}
|
||||
.tp-chart-row canvas{border:1px solid rgba(56,68,89,.4);border-radius:4px;flex:1;min-width:120px}
|
||||
.tp-metrics-grid{display:grid;grid-template-columns:1fr 1fr;gap:6px}
|
||||
.tp-metric-cell{background:rgba(13,17,23,.6);border:1px solid rgba(56,68,89,.3);border-radius:4px;padding:6px 8px}
|
||||
.tp-metric-label{font-size:10px;color:#6b7a8d;text-transform:uppercase;letter-spacing:.3px}
|
||||
.tp-metric-value{font-size:13px;color:#c8d0dc;font-weight:500;margin-top:2px}
|
||||
.tp-btn{padding:5px 12px;border-radius:4px;font-size:12px;font-weight:500;cursor:pointer;border:1px solid transparent;transition:all .15s}
|
||||
.tp-btn:disabled{opacity:.5;cursor:not-allowed}
|
||||
.tp-btn-success{background:rgba(40,167,69,.2);color:#51cf66;border-color:rgba(40,167,69,.3)}
|
||||
.tp-btn-success:hover:not(:disabled){background:rgba(40,167,69,.35)}
|
||||
.tp-btn-danger{background:rgba(220,53,69,.2);color:#ff6b6b;border-color:rgba(220,53,69,.3)}
|
||||
.tp-btn-danger:hover:not(:disabled){background:rgba(220,53,69,.35)}
|
||||
.tp-btn-secondary{background:rgba(30,40,60,.8);color:#b0b8c8;border-color:rgba(56,68,89,.6)}
|
||||
.tp-btn-secondary:hover:not(:disabled){background:rgba(40,50,75,.9)}
|
||||
.tp-btn-rec{background:rgba(220,53,69,.15);color:#ff6b6b;border-color:rgba(220,53,69,.3)}
|
||||
.tp-btn-rec:hover:not(:disabled){background:rgba(220,53,69,.3)}
|
||||
.tp-btn-muted{background:transparent;color:#6b7a8d;border-color:rgba(56,68,89,.4);font-size:11px;padding:3px 8px}
|
||||
.tp-btn-muted:hover:not(:disabled){color:#b0b8c8;border-color:rgba(56,68,89,.8)}
|
||||
`;
|
||||
|
||||
export default class TrainingPanel {
|
||||
constructor(container) {
|
||||
this.container = typeof container === 'string'
|
||||
? document.getElementById(container) : container;
|
||||
if (!this.container) throw new Error('TrainingPanel: container element not found');
|
||||
|
||||
this.state = {
|
||||
recordings: [], trainingStatus: null, isRecording: false,
|
||||
configOpen: true, loading: false, error: null
|
||||
};
|
||||
this.config = {
|
||||
epochs: 100, batch_size: 32, learning_rate: 3e-4, patience: 15,
|
||||
selectedRecordings: [], base_model: '', lora_profile_name: ''
|
||||
};
|
||||
this.progressData = { losses: [], pcks: [] };
|
||||
this.unsubscribers = [];
|
||||
this._injectStyles();
|
||||
this.render();
|
||||
this.refresh();
|
||||
this._bindEvents();
|
||||
}
|
||||
|
||||
_bindEvents() {
|
||||
this.unsubscribers.push(
|
||||
trainingService.on('progress', (d) => this._onProgress(d)),
|
||||
trainingService.on('training-started', () => this.refresh()),
|
||||
trainingService.on('training-stopped', () => {
|
||||
trainingService.disconnectProgressStream();
|
||||
this.refresh();
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
_onProgress(data) {
|
||||
if (data.train_loss != null) this.progressData.losses.push(data.train_loss);
|
||||
if (data.val_pck != null) this.progressData.pcks.push(data.val_pck);
|
||||
this._set({ trainingStatus: { ...this.state.trainingStatus, ...data } });
|
||||
}
|
||||
|
||||
// --- Data ---
|
||||
|
||||
async refresh() {
|
||||
this._set({ loading: true, error: null });
|
||||
try {
|
||||
const [recordings, status] = await Promise.all([
|
||||
trainingService.listRecordings().catch(() => []),
|
||||
trainingService.getTrainingStatus().catch(() => null)
|
||||
]);
|
||||
if (status && !status.active) this.progressData = { losses: [], pcks: [] };
|
||||
this._set({ recordings, trainingStatus: status, loading: false });
|
||||
} catch (e) { this._set({ loading: false, error: e.message }); }
|
||||
}
|
||||
|
||||
// --- Actions ---
|
||||
|
||||
async _startRec() {
|
||||
this._set({ loading: true, error: null });
|
||||
try {
|
||||
await trainingService.startRecording({ session_name: `rec_${Date.now()}`, label: 'pose' });
|
||||
this._set({ isRecording: true, loading: false });
|
||||
await this.refresh();
|
||||
} catch (e) { this._set({ loading: false, error: `Recording failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _stopRec() {
|
||||
this._set({ loading: true, error: null });
|
||||
try {
|
||||
await trainingService.stopRecording();
|
||||
this._set({ isRecording: false, loading: false });
|
||||
await this.refresh();
|
||||
} catch (e) { this._set({ loading: false, error: `Stop recording failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _delRec(id) {
|
||||
this._set({ loading: true, error: null });
|
||||
try {
|
||||
await trainingService.deleteRecording(id);
|
||||
this.config.selectedRecordings = this.config.selectedRecordings.filter(r => r !== id);
|
||||
await this.refresh();
|
||||
} catch (e) { this._set({ loading: false, error: `Delete failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _launchTraining(method, extraCfg = {}) {
|
||||
this._set({ loading: true, error: null });
|
||||
this.progressData = { losses: [], pcks: [] };
|
||||
try {
|
||||
trainingService.connectProgressStream();
|
||||
const payload = {
|
||||
dataset_ids: this.config.selectedRecordings,
|
||||
config: {
|
||||
epochs: this.config.epochs,
|
||||
batch_size: this.config.batch_size,
|
||||
learning_rate: this.config.learning_rate,
|
||||
...extraCfg
|
||||
}
|
||||
};
|
||||
await trainingService[method](payload);
|
||||
await this.refresh();
|
||||
} catch (e) { this._set({ loading: false, error: `Training failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
async _stopTraining() {
|
||||
this._set({ loading: true, error: null });
|
||||
try { await trainingService.stopTraining(); await this.refresh(); }
|
||||
catch (e) { this._set({ loading: false, error: `Stop failed: ${e.message}` }); }
|
||||
}
|
||||
|
||||
_set(p) { Object.assign(this.state, p); this.render(); }
|
||||
|
||||
// --- Render ---
|
||||
|
||||
render() {
|
||||
const el = this.container;
|
||||
el.innerHTML = '';
|
||||
const panel = this._el('div', 'tp-panel');
|
||||
panel.appendChild(this._renderHeader());
|
||||
if (this.state.error) panel.appendChild(this._el('div', 'tp-error', this.state.error));
|
||||
panel.appendChild(this._renderRecordings());
|
||||
const ts = this.state.trainingStatus;
|
||||
const active = ts && ts.active;
|
||||
if (active) panel.appendChild(this._renderProgress());
|
||||
else if (ts && !ts.active && this.progressData.losses.length > 0) panel.appendChild(this._renderComplete());
|
||||
else panel.appendChild(this._renderConfig());
|
||||
el.appendChild(panel);
|
||||
if (active) requestAnimationFrame(() => this._drawCharts());
|
||||
}
|
||||
|
||||
_renderHeader() {
|
||||
const h = this._el('div', 'tp-header');
|
||||
h.appendChild(this._el('span', 'tp-title', 'Training'));
|
||||
const ts = this.state.trainingStatus;
|
||||
let cls = 'tp-badge tp-badge-idle', txt = 'Idle';
|
||||
if (ts && ts.active) { cls = 'tp-badge tp-badge-active'; txt = 'Training'; }
|
||||
else if (ts && !ts.active && this.progressData.losses.length > 0) { cls = 'tp-badge tp-badge-done'; txt = 'Completed'; }
|
||||
h.appendChild(this._el('span', cls, txt));
|
||||
return h;
|
||||
}
|
||||
|
||||
_renderRecordings() {
|
||||
const s = this._el('div', 'tp-section');
|
||||
s.appendChild(this._el('div', 'tp-section-title', 'CSI Recordings'));
|
||||
if (this.state.recordings.length === 0 && !this.state.loading) {
|
||||
s.appendChild(this._el('div', 'tp-empty', 'Start recording CSI data to train a model'));
|
||||
} else {
|
||||
this.state.recordings.forEach(rec => {
|
||||
const row = this._el('div', 'tp-rec-row');
|
||||
const info = this._el('div', 'tp-rec-info');
|
||||
info.appendChild(this._el('span', 'tp-rec-name', rec.name || rec.id));
|
||||
const parts = [];
|
||||
if (rec.frame_count != null) parts.push(rec.frame_count + ' frames');
|
||||
if (rec.file_size_bytes != null) parts.push(this._fmtB(rec.file_size_bytes));
|
||||
if (rec.started_at && rec.ended_at) parts.push(Math.round((new Date(rec.ended_at) - new Date(rec.started_at)) / 1000) + 's');
|
||||
info.appendChild(this._el('span', 'tp-rec-meta', parts.join(' / ')));
|
||||
row.appendChild(info);
|
||||
const del = this._btn('Delete', 'tp-btn tp-btn-muted', () => this._delRec(rec.id));
|
||||
del.disabled = this.state.loading;
|
||||
row.appendChild(del);
|
||||
s.appendChild(row);
|
||||
});
|
||||
}
|
||||
const acts = this._el('div', 'tp-rec-actions');
|
||||
if (this.state.isRecording) {
|
||||
const b = this._btn('Stop Recording', 'tp-btn tp-btn-danger', () => this._stopRec());
|
||||
b.disabled = this.state.loading; acts.appendChild(b);
|
||||
} else {
|
||||
const b = this._btn('Start Recording', 'tp-btn tp-btn-rec', () => this._startRec());
|
||||
b.disabled = this.state.loading; acts.appendChild(b);
|
||||
}
|
||||
s.appendChild(acts);
|
||||
return s;
|
||||
}
|
||||
|
||||
_renderConfig() {
|
||||
const s = this._el('div', 'tp-section');
|
||||
const hdr = this._el('div', 'tp-config-header');
|
||||
hdr.appendChild(this._el('span', 'tp-section-title', 'Training Configuration'));
|
||||
hdr.appendChild(this._btn(this.state.configOpen ? 'Collapse' : 'Expand', 'tp-btn tp-btn-muted',
|
||||
() => { this.state.configOpen = !this.state.configOpen; this.render(); }));
|
||||
s.appendChild(hdr);
|
||||
if (!this.state.configOpen) return s;
|
||||
|
||||
const form = this._el('div', 'tp-config-form');
|
||||
if (this.state.recordings.length > 0) {
|
||||
form.appendChild(this._el('label', 'tp-label', 'Datasets'));
|
||||
const dc = this._el('div', 'tp-ds-container');
|
||||
this.state.recordings.forEach(rec => {
|
||||
const lb = this._el('label', 'tp-ds-item');
|
||||
const cb = document.createElement('input');
|
||||
cb.type = 'checkbox';
|
||||
cb.checked = this.config.selectedRecordings.includes(rec.id);
|
||||
cb.addEventListener('change', () => {
|
||||
if (cb.checked) { if (!this.config.selectedRecordings.includes(rec.id)) this.config.selectedRecordings.push(rec.id); }
|
||||
else { this.config.selectedRecordings = this.config.selectedRecordings.filter(r => r !== rec.id); }
|
||||
});
|
||||
lb.appendChild(cb);
|
||||
lb.appendChild(this._el('span', null, rec.name || rec.id));
|
||||
dc.appendChild(lb);
|
||||
});
|
||||
form.appendChild(dc);
|
||||
}
|
||||
const ir = (l, t, v, fn) => {
|
||||
const r = this._el('div', 'tp-input-row');
|
||||
r.appendChild(this._el('label', 'tp-label', l));
|
||||
const inp = document.createElement('input');
|
||||
inp.type = t; inp.className = 'tp-input'; inp.value = v;
|
||||
inp.addEventListener('change', () => fn(inp.value));
|
||||
r.appendChild(inp); return r;
|
||||
};
|
||||
form.appendChild(ir('Epochs', 'number', this.config.epochs, v => { this.config.epochs = parseInt(v) || 100; }));
|
||||
form.appendChild(ir('Batch Size', 'number', this.config.batch_size, v => { this.config.batch_size = parseInt(v) || 32; }));
|
||||
form.appendChild(ir('Learning Rate', 'text', this.config.learning_rate, v => { this.config.learning_rate = parseFloat(v) || 3e-4; }));
|
||||
form.appendChild(ir('Early Stop Patience', 'number', this.config.patience, v => { this.config.patience = parseInt(v) || 15; }));
|
||||
form.appendChild(ir('Base Model (opt.)', 'text', this.config.base_model, v => { this.config.base_model = v; }));
|
||||
form.appendChild(ir('LoRA Profile (opt.)', 'text', this.config.lora_profile_name, v => { this.config.lora_profile_name = v; }));
|
||||
s.appendChild(form);
|
||||
|
||||
const acts = this._el('div', 'tp-train-actions');
|
||||
const btns = [
|
||||
this._btn('Start Training', 'tp-btn tp-btn-success', () => this._launchTraining('startTraining', { patience: this.config.patience, base_model: this.config.base_model || undefined })),
|
||||
this._btn('Pretrain', 'tp-btn tp-btn-secondary', () => this._launchTraining('startPretraining')),
|
||||
this._btn('LoRA', 'tp-btn tp-btn-secondary', () => this._launchTraining('startLoraTraining', { base_model: this.config.base_model || undefined, profile_name: this.config.lora_profile_name || 'default' }))
|
||||
];
|
||||
btns.forEach(b => { b.disabled = this.state.loading; acts.appendChild(b); });
|
||||
s.appendChild(acts);
|
||||
return s;
|
||||
}
|
||||
|
||||
_renderProgress() {
|
||||
const ts = this.state.trainingStatus || {};
|
||||
const s = this._el('div', 'tp-section');
|
||||
s.appendChild(this._el('div', 'tp-section-title', 'Training Progress'));
|
||||
|
||||
const pct = ts.total_epochs ? Math.round((ts.epoch / ts.total_epochs) * 100) : 0;
|
||||
const bar = this._el('div', 'tp-progress-bar');
|
||||
const fill = this._el('div', 'tp-progress-fill');
|
||||
fill.style.width = pct + '%';
|
||||
bar.appendChild(fill); s.appendChild(bar);
|
||||
s.appendChild(this._el('div', 'tp-progress-label', `Epoch ${ts.epoch ?? 0} / ${ts.total_epochs ?? '?'} (${pct}%)`));
|
||||
|
||||
const cr = this._el('div', 'tp-chart-row');
|
||||
const lc = document.createElement('canvas'); lc.id = 'tp-loss-chart'; lc.width = 260; lc.height = 140;
|
||||
const pc = document.createElement('canvas'); pc.id = 'tp-pck-chart'; pc.width = 260; pc.height = 140;
|
||||
cr.appendChild(lc); cr.appendChild(pc); s.appendChild(cr);
|
||||
|
||||
const g = this._el('div', 'tp-metrics-grid');
|
||||
const mc = (l, v) => { const c = this._el('div', 'tp-metric-cell'); c.appendChild(this._el('div', 'tp-metric-label', l)); c.appendChild(this._el('div', 'tp-metric-value', v)); return c; };
|
||||
g.appendChild(mc('Loss', ts.train_loss != null ? ts.train_loss.toFixed(4) : '--'));
|
||||
g.appendChild(mc('PCK', ts.val_pck != null ? (ts.val_pck * 100).toFixed(1) + '%' : '--'));
|
||||
g.appendChild(mc('OKS', ts.val_oks != null ? ts.val_oks.toFixed(3) : '--'));
|
||||
g.appendChild(mc('LR', ts.lr != null ? ts.lr.toExponential(1) : '--'));
|
||||
g.appendChild(mc('Best PCK', ts.best_pck != null ? (ts.best_pck * 100).toFixed(1) + '% (e' + (ts.best_epoch ?? '?') + ')' : '--'));
|
||||
g.appendChild(mc('Patience', ts.patience_remaining != null ? String(ts.patience_remaining) : '--'));
|
||||
g.appendChild(mc('ETA', ts.eta_secs != null ? this._fmtEta(ts.eta_secs) : '--'));
|
||||
g.appendChild(mc('Phase', ts.phase || '--'));
|
||||
s.appendChild(g);
|
||||
|
||||
const stop = this._btn('Stop Training', 'tp-btn tp-btn-danger', () => this._stopTraining());
|
||||
stop.disabled = this.state.loading; stop.style.marginTop = '10px'; s.appendChild(stop);
|
||||
return s;
|
||||
}
|
||||
|
||||
_renderComplete() {
|
||||
const ts = this.state.trainingStatus || {};
|
||||
const s = this._el('div', 'tp-section');
|
||||
s.appendChild(this._el('div', 'tp-section-title', 'Training Complete'));
|
||||
const g = this._el('div', 'tp-metrics-grid');
|
||||
const mc = (l, v) => { const c = this._el('div', 'tp-metric-cell'); c.appendChild(this._el('div', 'tp-metric-label', l)); c.appendChild(this._el('div', 'tp-metric-value', v)); return c; };
|
||||
const losses = this.progressData.losses;
|
||||
g.appendChild(mc('Final Loss', losses.length > 0 ? losses[losses.length - 1].toFixed(4) : '--'));
|
||||
g.appendChild(mc('Best PCK', ts.best_pck != null ? (ts.best_pck * 100).toFixed(1) + '%' : '--'));
|
||||
g.appendChild(mc('Best Epoch', ts.best_epoch != null ? String(ts.best_epoch) : '--'));
|
||||
g.appendChild(mc('Total Epochs', String(losses.length)));
|
||||
s.appendChild(g);
|
||||
const acts = this._el('div', 'tp-train-actions');
|
||||
acts.appendChild(this._btn('New Training', 'tp-btn tp-btn-secondary', () => {
|
||||
this.progressData = { losses: [], pcks: [] }; this._set({ trainingStatus: null });
|
||||
}));
|
||||
s.appendChild(acts);
|
||||
return s;
|
||||
}
|
||||
|
||||
// --- Chart drawing ---
|
||||
|
||||
_drawCharts() {
|
||||
this._drawChart('tp-loss-chart', this.progressData.losses, { color: '#ff6b6b', label: 'Loss', yMin: 0, yMax: null });
|
||||
this._drawChart('tp-pck-chart', this.progressData.pcks, { color: '#51cf66', label: 'PCK', yMin: 0, yMax: 1 });
|
||||
}
|
||||
|
||||
_drawChart(id, data, opts) {
|
||||
const cv = document.getElementById(id);
|
||||
if (!cv) return;
|
||||
const ctx = cv.getContext('2d'), w = cv.width, h = cv.height;
|
||||
const p = { t: 20, r: 10, b: 24, l: 44 };
|
||||
ctx.fillStyle = '#0d1117'; ctx.fillRect(0, 0, w, h);
|
||||
ctx.fillStyle = '#8899aa'; ctx.font = '11px -apple-system,sans-serif'; ctx.fillText(opts.label, p.l, 14);
|
||||
if (!data.length) { ctx.fillStyle = '#6b7a8d'; ctx.fillText('No data', w / 2 - 20, h / 2); return; }
|
||||
const pw = w - p.l - p.r, ph = h - p.t - p.b;
|
||||
let yMin = opts.yMin ?? Math.min(...data), yMax = opts.yMax ?? Math.max(...data);
|
||||
if (yMax === yMin) yMax = yMin + 1;
|
||||
ctx.strokeStyle = 'rgba(255,255,255,.08)'; ctx.lineWidth = 1;
|
||||
for (let i = 0; i <= 4; i++) {
|
||||
const y = p.t + (ph / 4) * i;
|
||||
ctx.beginPath(); ctx.moveTo(p.l, y); ctx.lineTo(w - p.r, y); ctx.stroke();
|
||||
const v = yMax - ((yMax - yMin) / 4) * i;
|
||||
ctx.fillStyle = '#6b7a8d'; ctx.font = '9px sans-serif'; ctx.fillText(v.toFixed(v >= 1 ? 2 : 3), 2, y + 3);
|
||||
}
|
||||
const xl = Math.min(data.length, 5);
|
||||
for (let i = 0; i < xl; i++) {
|
||||
const idx = Math.round((data.length - 1) * (i / (xl - 1 || 1)));
|
||||
ctx.fillStyle = '#6b7a8d'; ctx.fillText(String(idx + 1), p.l + (pw * idx) / (data.length - 1 || 1) - 4, h - 4);
|
||||
}
|
||||
ctx.strokeStyle = opts.color; ctx.lineWidth = 1.5; ctx.beginPath();
|
||||
data.forEach((v, i) => {
|
||||
const x = p.l + (pw * i) / (data.length - 1 || 1);
|
||||
const y = p.t + ph - ((v - yMin) / (yMax - yMin)) * ph;
|
||||
i === 0 ? ctx.moveTo(x, y) : ctx.lineTo(x, y);
|
||||
});
|
||||
ctx.stroke();
|
||||
if (data.length > 0) {
|
||||
const ly = p.t + ph - ((data[data.length - 1] - yMin) / (yMax - yMin)) * ph;
|
||||
ctx.fillStyle = opts.color; ctx.beginPath(); ctx.arc(p.l + pw, ly, 3, 0, Math.PI * 2); ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
_el(tag, cls, txt) {
|
||||
const e = document.createElement(tag);
|
||||
if (cls) e.className = cls;
|
||||
if (txt != null) e.textContent = txt;
|
||||
return e;
|
||||
}
|
||||
|
||||
_btn(txt, cls, fn) {
|
||||
const b = document.createElement('button');
|
||||
b.className = cls; b.textContent = txt;
|
||||
b.addEventListener('click', fn); return b;
|
||||
}
|
||||
|
||||
_fmtB(b) { return b < 1024 ? b + ' B' : b < 1048576 ? (b / 1024).toFixed(1) + ' KB' : (b / 1048576).toFixed(1) + ' MB'; }
|
||||
_fmtEta(s) { return s < 60 ? Math.round(s) + 's' : s < 3600 ? Math.round(s / 60) + 'm' : (s / 3600).toFixed(1) + 'h'; }
|
||||
|
||||
_injectStyles() {
|
||||
if (document.getElementById('training-panel-styles')) return;
|
||||
const s = document.createElement('style');
|
||||
s.id = 'training-panel-styles';
|
||||
s.textContent = TP_STYLES;
|
||||
document.head.appendChild(s);
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.unsubscribers.forEach(fn => fn());
|
||||
this.unsubscribers = [];
|
||||
trainingService.disconnectProgressStream();
|
||||
if (this.container) this.container.innerHTML = '';
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.destroy();
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@
|
||||
<button class="nav-tab" data-tab="performance">Performance</button>
|
||||
<button class="nav-tab" data-tab="applications">Applications</button>
|
||||
<button class="nav-tab" data-tab="sensing">Sensing</button>
|
||||
<button class="nav-tab" data-tab="training">Training</button>
|
||||
</nav>
|
||||
|
||||
<!-- Dashboard Tab -->
|
||||
@@ -67,6 +68,11 @@
|
||||
<span class="status-text">-</span>
|
||||
<span class="status-message"></span>
|
||||
</div>
|
||||
<div class="component-status" data-component="datasource" id="dashboard-datasource">
|
||||
<span class="component-name">Data Source</span>
|
||||
<span class="status-text">-</span>
|
||||
<span class="status-message"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -482,6 +488,18 @@
|
||||
|
||||
<!-- Sensing Tab -->
|
||||
<section id="sensing" class="tab-content"></section>
|
||||
|
||||
<!-- Training Tab -->
|
||||
<section id="training" class="tab-content">
|
||||
<div class="tab-header">
|
||||
<h2>Model Training</h2>
|
||||
<p>Record CSI data, train pose estimation models, and manage .rvf files</p>
|
||||
</div>
|
||||
<div id="training-container" style="display: flex; gap: 20px; flex-wrap: wrap;">
|
||||
<div id="training-panel-container" style="flex: 1; min-width: 400px;"></div>
|
||||
<div id="model-panel-container" style="flex: 1; min-width: 350px; max-width: 450px;"></div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<!-- Error Toast -->
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
EXPO_PUBLIC_DEFAULT_SERVER_URL=http://192.168.1.100:8080
|
||||
@@ -0,0 +1,26 @@
|
||||
module.exports = {
|
||||
root: true,
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module',
|
||||
ecmaFeatures: {
|
||||
jsx: true,
|
||||
},
|
||||
},
|
||||
plugins: ['@typescript-eslint', 'react', 'react-hooks'],
|
||||
extends: [
|
||||
'eslint:recommended',
|
||||
'plugin:react/recommended',
|
||||
'plugin:react-hooks/recommended',
|
||||
'plugin:@typescript-eslint/recommended',
|
||||
],
|
||||
settings: {
|
||||
react: {
|
||||
version: 'detect',
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
'react/react-in-jsx-scope': 'off',
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,41 @@
|
||||
# Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files
|
||||
|
||||
# dependencies
|
||||
node_modules/
|
||||
|
||||
# Expo
|
||||
.expo/
|
||||
dist/
|
||||
web-build/
|
||||
expo-env.d.ts
|
||||
|
||||
# Native
|
||||
.kotlin/
|
||||
*.orig.*
|
||||
*.jks
|
||||
*.p8
|
||||
*.p12
|
||||
*.key
|
||||
*.mobileprovision
|
||||
|
||||
# Metro
|
||||
.metro-health-check*
|
||||
|
||||
# debug
|
||||
npm-debug.*
|
||||
yarn-debug.*
|
||||
yarn-error.*
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# local env files
|
||||
.env*.local
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
|
||||
# generated native folders
|
||||
/ios
|
||||
/android
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all"
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { useEffect } from 'react';
|
||||
import { NavigationContainer, DarkTheme } from '@react-navigation/native';
|
||||
import { GestureHandlerRootView } from 'react-native-gesture-handler';
|
||||
import { StatusBar } from 'expo-status-bar';
|
||||
import { SafeAreaProvider } from 'react-native-safe-area-context';
|
||||
import { apiService } from '@/services/api.service';
|
||||
import { rssiService } from '@/services/rssi.service';
|
||||
import { wsService } from '@/services/ws.service';
|
||||
import { ThemeProvider } from './src/theme/ThemeContext';
|
||||
import { usePoseStore } from './src/stores/poseStore';
|
||||
import { useSettingsStore } from './src/stores/settingsStore';
|
||||
import { RootNavigator } from './src/navigation/RootNavigator';
|
||||
|
||||
export default function App() {
|
||||
const serverUrl = useSettingsStore((state) => state.serverUrl);
|
||||
const rssiScanEnabled = useSettingsStore((state) => state.rssiScanEnabled);
|
||||
|
||||
useEffect(() => {
|
||||
apiService.setBaseUrl(serverUrl);
|
||||
const unsubscribe = wsService.subscribe(usePoseStore.getState().handleFrame);
|
||||
wsService.connect(serverUrl);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
wsService.disconnect();
|
||||
};
|
||||
}, [serverUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!rssiScanEnabled) {
|
||||
rssiService.stopScanning();
|
||||
return;
|
||||
}
|
||||
|
||||
const unsubscribe = rssiService.subscribe(() => {
|
||||
// Consumers can subscribe elsewhere for RSSI events.
|
||||
});
|
||||
rssiService.startScanning(2000);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
rssiService.stopScanning();
|
||||
};
|
||||
}, [rssiScanEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
(globalThis as { __appStartTime?: number }).__appStartTime = Date.now();
|
||||
}, []);
|
||||
|
||||
const navigationTheme = {
|
||||
...DarkTheme,
|
||||
colors: {
|
||||
...DarkTheme.colors,
|
||||
background: '#0A0E1A',
|
||||
card: '#0D1117',
|
||||
text: '#E2E8F0',
|
||||
border: '#1E293B',
|
||||
primary: '#32B8C6',
|
||||
},
|
||||
};
|
||||
|
||||
return (
|
||||
<GestureHandlerRootView style={{ flex: 1 }}>
|
||||
<SafeAreaProvider>
|
||||
<ThemeProvider>
|
||||
<NavigationContainer theme={navigationTheme}>
|
||||
<RootNavigator />
|
||||
</NavigationContainer>
|
||||
</ThemeProvider>
|
||||
</SafeAreaProvider>
|
||||
<StatusBar style="light" />
|
||||
</GestureHandlerRootView>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
# WiFi-DensePose Mobile
|
||||
|
||||
**See through walls from your phone.** Real-time WiFi sensing, vital signs, and disaster response — in a cross-platform mobile app.
|
||||
|
||||
WiFi-DensePose Mobile is a React Native / Expo companion app for the [WiFi-DensePose](../../README.md) sensing platform. It connects to a WiFi sensing server over WebSocket, renders live 3D Gaussian splat visualizations of detected humans, displays breathing and heart rate in real time, and provides a full WiFi-MAT disaster triage dashboard — all from a single codebase that runs on iOS, Android, and Web.
|
||||
|
||||
> | Screen | What It Shows |
|
||||
> |--------|---------------|
|
||||
> | **Live** | 3D Gaussian splat body rendering with FPS counter, signal strength, confidence HUD |
|
||||
> | **Vitals** | Breathing rate (6-30 BPM) and heart rate (40-120 BPM) arc gauges with sparkline history |
|
||||
> | **Zones** | SVG floor plan with occupancy grid, zone legend, presence heatmap |
|
||||
> | **MAT** | Mass casualty assessment: survivor counter, triage alerts, zone management |
|
||||
> | **Settings** | Server URL, theme picker, RSSI-only toggle, alert sound control |
|
||||
|
||||
```bash
|
||||
# Quick start — web preview in 30 seconds
|
||||
cd ui/mobile
|
||||
npm install
|
||||
npx expo start --web
|
||||
```
|
||||
|
||||
<!-- Screenshot placeholder: replace with actual app screenshots -->
|
||||
<!--  -->
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
| | Feature | Details |
|
||||
|---|---------|---------|
|
||||
| **3D Live View** | Gaussian splat rendering | Three.js via WebView (native) or iframe (web), real-time pose overlay |
|
||||
| **Vital Signs** | Breathing + heart rate | Arc gauge components with sparkline 60-sample history, confidence indicators |
|
||||
| **Disaster Response** | WiFi-MAT dashboard | Survivor detection, START triage classification, priority alerts, zone scan tracking |
|
||||
| **Floor Plan** | SVG occupancy grid | Zone-level presence visualization, color-coded density, interactive legend |
|
||||
| **Cross-Platform** | iOS, Android, Web | Expo SDK 55, React Native 0.83, single codebase with platform-specific modules |
|
||||
| **Offline Capable** | Automatic simulation fallback | When the sensing server is unreachable, generates synthetic data so the UI stays functional |
|
||||
| **RSSI Mode** | No CSI hardware needed | Toggle RSSI-only scanning for coarse presence detection on consumer WiFi devices |
|
||||
| **Dark Theme** | Cyan accent (#32B8C6) | Dark-first design system with consistent color tokens, spacing scale, and monospace typography |
|
||||
| **Persistent State** | Zustand + AsyncStorage | Settings, connection preferences, and theme survive app restarts |
|
||||
| **Platform WiFi** | Native RSSI scanning | Android: `react-native-wifi-reborn`, iOS: stub (requires entitlement), Web: synthetic values |
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
| Requirement | Version | Notes |
|
||||
|-------------|---------|-------|
|
||||
| Node.js | 18+ | LTS recommended |
|
||||
| npm | 9+ | Ships with Node.js 18+ |
|
||||
| Expo CLI | Latest | Installed automatically via `npx` |
|
||||
| iOS Simulator | Xcode 15+ | macOS only; optional for iOS development |
|
||||
| Android Emulator | API 33+ | Android Studio; optional for Android development |
|
||||
| WiFi-DensePose Server | Any | Optional — app falls back to simulated data without a server |
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Web (fastest)
|
||||
|
||||
```bash
|
||||
cd ui/mobile
|
||||
npm install
|
||||
npx expo start --web
|
||||
```
|
||||
|
||||
Open `http://localhost:8081` in your browser. The app starts in simulation mode with synthetic pose and vital sign data.
|
||||
|
||||
### Android
|
||||
|
||||
```bash
|
||||
cd ui/mobile
|
||||
npm install
|
||||
npx expo start --android
|
||||
```
|
||||
|
||||
Requires Android Studio with an emulator running, or a physical device with Expo Go installed.
|
||||
|
||||
### iOS
|
||||
|
||||
```bash
|
||||
cd ui/mobile
|
||||
npm install
|
||||
npx expo start --ios
|
||||
```
|
||||
|
||||
Requires Xcode with a simulator, or a physical device with Expo Go. RSSI scanning on iOS requires the `com.apple.developer.networking.wifi-info` entitlement.
|
||||
|
||||
---
|
||||
|
||||
## Connecting to a Sensing Server
|
||||
|
||||
The app connects to the WiFi-DensePose sensing server over WebSocket for live data. Configure the server URL in the **Settings** tab.
|
||||
|
||||
| Server Location | URL | Notes |
|
||||
|----------------|-----|-------|
|
||||
| Local dev server | `http://localhost:3000` | Default; sensing WS auto-connects on port 3001 |
|
||||
| Docker container | `http://host.docker.internal:3000` | From emulator connecting to host Docker |
|
||||
| ESP32 mesh | `http://<esp32-ip>:3000` | Direct connection to ESP32 aggregator |
|
||||
| Remote server | `https://your-server.example.com` | TLS supported; WebSocket upgrades to `wss://` |
|
||||
|
||||
When the server is unreachable, the app automatically falls back to **simulation mode** after exhausting reconnect attempts (exponential backoff). A yellow `SIM` badge appears in the connection banner. Reconnection resumes automatically when the server becomes available.
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><strong>Architecture</strong></summary>
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
ui/mobile/
|
||||
App.tsx Root component (providers, navigation, services)
|
||||
app.config.ts Expo configuration
|
||||
index.ts Entry point
|
||||
src/
|
||||
components/
|
||||
ConnectionBanner.tsx Server status banner (connected/simulated/disconnected)
|
||||
ErrorBoundary.tsx Crash boundary with fallback UI
|
||||
GaugeArc.tsx SVG arc gauge for vital sign display
|
||||
HudOverlay.tsx Heads-up display overlay
|
||||
LoadingSpinner.tsx Themed loading indicator
|
||||
ModeBadge.tsx LIVE / SIM / RSSI mode indicator
|
||||
OccupancyGrid.tsx Grid-based occupancy visualization
|
||||
SignalBar.tsx RSSI signal strength bars
|
||||
SparklineChart.tsx Mini sparkline for metric history
|
||||
StatusDot.tsx Connection status indicator dot
|
||||
ThemedText.tsx Text component with theme presets
|
||||
ThemedView.tsx View component with theme background
|
||||
constants/
|
||||
api.ts REST API path constants
|
||||
simulation.ts Simulation tick interval, defaults
|
||||
websocket.ts WS path, reconnect delays, max attempts
|
||||
hooks/
|
||||
usePoseStream.ts Subscribe to live or simulated sensing frames
|
||||
useRssiScanner.ts Platform RSSI scanning hook
|
||||
useServerReachability.ts HTTP health check polling
|
||||
useTheme.ts Dark/light/system theme resolution
|
||||
useWebViewBridge.ts WebView message bridge for Gaussian viewer
|
||||
navigation/
|
||||
MainTabs.tsx Bottom tab navigator (5 tabs with lazy loading)
|
||||
RootNavigator.tsx Root stack navigator
|
||||
types.ts Navigation param list types
|
||||
screens/
|
||||
LiveScreen/
|
||||
index.tsx 3D Gaussian splat view with HUD overlay
|
||||
GaussianSplatWebView.tsx Native WebView renderer (Three.js)
|
||||
GaussianSplatWebView.web.tsx Web iframe renderer
|
||||
LiveHUD.tsx FPS, RSSI, confidence, person count overlay
|
||||
useGaussianBridge.ts WebView message protocol
|
||||
VitalsScreen/
|
||||
index.tsx Breathing + heart rate dashboard
|
||||
BreathingGauge.tsx Arc gauge for breathing BPM
|
||||
HeartRateGauge.tsx Arc gauge for heart rate BPM
|
||||
MetricCard.tsx Vital sign metric card with sparkline
|
||||
ZonesScreen/
|
||||
index.tsx Floor plan occupancy view
|
||||
FloorPlanSvg.tsx SVG floor plan renderer
|
||||
useOccupancyGrid.ts Grid computation from sensing frames
|
||||
ZoneLegend.tsx Color-coded zone legend
|
||||
MATScreen/
|
||||
index.tsx Mass casualty assessment dashboard
|
||||
AlertCard.tsx Single triage alert card
|
||||
AlertList.tsx Scrollable alert list with priority sorting
|
||||
MatWebView.tsx MAT visualization WebView
|
||||
SurvivorCounter.tsx Survivor count by triage status
|
||||
useMatBridge.ts MAT WebView message protocol
|
||||
SettingsScreen/
|
||||
index.tsx App settings panel
|
||||
ServerUrlInput.tsx Server URL text input with validation
|
||||
RssiToggle.tsx RSSI-only mode switch
|
||||
ThemePicker.tsx Dark / light / system theme selector
|
||||
services/
|
||||
ws.service.ts WebSocket client with auto-reconnect + simulation fallback
|
||||
api.service.ts REST client (Axios) with retry logic
|
||||
rssi.service.ts Platform-agnostic RSSI scanner interface
|
||||
rssi.service.android.ts Android: react-native-wifi-reborn integration
|
||||
rssi.service.ios.ts iOS: stub (requires entitlement)
|
||||
rssi.service.web.ts Web: synthetic RSSI values
|
||||
simulation.service.ts Generates synthetic SensingFrame data
|
||||
stores/
|
||||
poseStore.ts Pose frames, connection status, frame history (Zustand)
|
||||
matStore.ts MAT survivors, zones, alerts, disaster events (Zustand)
|
||||
settingsStore.ts Server URL, theme, RSSI toggle (Zustand + persist)
|
||||
theme/
|
||||
colors.ts Color tokens (bg, surface, accent, danger, etc.)
|
||||
spacing.ts 4px-based spacing scale
|
||||
typography.ts Font families and size presets
|
||||
ThemeContext.tsx React context provider for theme
|
||||
index.ts Theme barrel export
|
||||
types/
|
||||
sensing.ts SensingFrame, SensingNode, VitalsData, Classification
|
||||
mat.ts Survivor, Alert, ScanZone, TriageStatus, DisasterType
|
||||
api.ts PoseStatus, ZoneConfig, HistoricalFrames, ApiError
|
||||
navigation.ts Navigation param lists
|
||||
utils/
|
||||
colorMap.ts Value-to-color mapping for heatmaps
|
||||
formatters.ts Number and date formatting utilities
|
||||
ringBuffer.ts Fixed-size circular buffer for frame history
|
||||
urlValidator.ts Server URL validation
|
||||
e2e/ Maestro end-to-end test specs
|
||||
assets/ App icons and images
|
||||
```
|
||||
|
||||
### Data Flow
|
||||
|
||||
```
|
||||
WiFi Sensing Server (Rust/Axum)
|
||||
|
|
||||
| WebSocket (ws://host:3001/ws/sensing)
|
||||
v
|
||||
ws.service.ts -----> [auto-reconnect with exponential backoff]
|
||||
| |
|
||||
| SensingFrame | (server unreachable)
|
||||
v v
|
||||
poseStore.ts simulation.service.ts
|
||||
| |
|
||||
| Zustand state | synthetic SensingFrame
|
||||
v v
|
||||
usePoseStream.ts <----------+
|
||||
|
|
||||
+---> LiveScreen (3D Gaussian splat + HUD)
|
||||
+---> VitalsScreen (breathing + heart rate gauges)
|
||||
+---> ZonesScreen (floor plan occupancy grid)
|
||||
|
||||
api.service.ts -----> REST API (GET /api/pose/status, /zones, /frames)
|
||||
|
|
||||
v
|
||||
matStore.ts -----> MATScreen (survivor counter, alerts, zones)
|
||||
|
||||
rssi.service.ts -----> Platform WiFi scan (Android / iOS / Web)
|
||||
|
|
||||
v
|
||||
useRssiScanner.ts -----> LiveScreen HUD (signal bars)
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><strong>Screens</strong></summary>
|
||||
|
||||
### Live
|
||||
|
||||
The primary visualization screen. Renders a 3D Gaussian splat representation of detected humans using Three.js. On native platforms, the renderer runs inside a WebView; on web, it uses an iframe. A heads-up display overlays connection status, FPS, RSSI signal strength, detection confidence, and person count. Supports three modes: **LIVE** (connected to server), **SIM** (simulation fallback), and **RSSI** (RSSI-only scanning).
|
||||
|
||||
### Vitals
|
||||
|
||||
Displays real-time breathing rate and heart rate extracted from CSI signal processing. Each vital sign is shown as an animated arc gauge (`GaugeArc` component) with the current BPM value, a 60-sample sparkline history (`SparklineChart`), and a confidence percentage. Normal ranges: breathing 6-30 BPM, heart rate 40-120 BPM.
|
||||
|
||||
### Zones
|
||||
|
||||
A floor plan view that maps WiFi sensing coverage to physical space. Uses SVG rendering (`react-native-svg`) to draw zones with color-coded occupancy density. The `useOccupancyGrid` hook computes grid cell values from incoming sensing frames. A legend shows the color scale from empty to high-density zones.
|
||||
|
||||
### MAT
|
||||
|
||||
Mass Casualty Assessment Tool for disaster response. Displays a survivor counter grouped by START triage classification (Immediate / Delayed / Minor / Deceased), a scrollable alert list sorted by priority, and zone scan progress. Each alert card shows the survivor location, recommended action, and triage color. The MAT tab badge shows the active alert count.
|
||||
|
||||
### Settings
|
||||
|
||||
Configuration panel with four controls:
|
||||
- **Server URL** — text input with URL validation; changes trigger WebSocket reconnect
|
||||
- **Theme** — dark / light / system picker
|
||||
- **RSSI Scanning** — toggle for platform-native WiFi RSSI scanning
|
||||
- **Alert Sound** — toggle for MAT alert audio notifications
|
||||
|
||||
All settings persist across app restarts via Zustand with AsyncStorage.
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><strong>API Integration</strong></summary>
|
||||
|
||||
### WebSocket Protocol
|
||||
|
||||
The app connects to the sensing server's WebSocket endpoint for real-time data streaming.
|
||||
|
||||
**Endpoint:** `ws://<host>:3001/ws/sensing`
|
||||
|
||||
**Frame format** (`SensingFrame`):
|
||||
|
||||
```typescript
|
||||
interface SensingFrame {
|
||||
type?: string;
|
||||
timestamp?: number;
|
||||
source?: string; // "live" | "simulated"
|
||||
tick?: number;
|
||||
nodes: SensingNode[]; // Per-node RSSI, position, amplitude
|
||||
features: FeatureSet; // mean_rssi, variance, motion_band_power, etc.
|
||||
classification: Classification; // motion_level, presence, confidence
|
||||
signal_field: SignalField; // 3D voxel grid values
|
||||
vital_signs?: VitalsData; // breathing_bpm, hr_proxy_bpm, confidence
|
||||
}
|
||||
```
|
||||
|
||||
The WebSocket service (`ws.service.ts`) handles:
|
||||
- Automatic reconnection with exponential backoff (1s, 2s, 4s, 8s, 16s)
|
||||
- Fallback to simulation after max reconnect attempts
|
||||
- Protocol upgrade (`http:` to `ws:`, `https:` to `wss:`)
|
||||
- Port mapping (HTTP 3000 maps to WS 3001)
|
||||
|
||||
### REST API
|
||||
|
||||
The REST client (`api.service.ts`) provides:
|
||||
|
||||
| Method | Path | Returns |
|
||||
|--------|------|---------|
|
||||
| `GET` | `/api/pose/status` | `PoseStatus` — server health and capabilities |
|
||||
| `GET` | `/api/pose/zones` | `ZoneConfig[]` — configured sensing zones |
|
||||
| `GET` | `/api/pose/frames?limit=N` | `HistoricalFrames` — recent frame history |
|
||||
|
||||
All requests use Axios with a 5-second timeout and automatic retry (2 attempts).
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit Tests
|
||||
|
||||
```bash
|
||||
cd ui/mobile
|
||||
npm test
|
||||
```
|
||||
|
||||
Runs the Jest test suite via `jest-expo`. Tests cover:
|
||||
|
||||
| Category | Files | What Is Tested |
|
||||
|----------|-------|----------------|
|
||||
| Components | 7 | `ConnectionBanner`, `GaugeArc`, `HudOverlay`, `OccupancyGrid`, `SignalBar`, `SparklineChart`, `StatusDot` |
|
||||
| Screens | 5 | `LiveScreen`, `VitalsScreen`, `ZonesScreen`, `MATScreen`, `SettingsScreen` |
|
||||
| Services | 4 | `ws.service`, `api.service`, `rssi.service`, `simulation.service` |
|
||||
| Stores | 3 | `poseStore`, `matStore`, `settingsStore` |
|
||||
| Hooks | 3 | `usePoseStream`, `useRssiScanner`, `useServerReachability` |
|
||||
| Utils | 3 | `colorMap`, `ringBuffer`, `urlValidator` |
|
||||
|
||||
### End-to-End Tests (Maestro)
|
||||
|
||||
```bash
|
||||
# Install Maestro CLI
|
||||
curl -Ls https://get.maestro.mobile.dev | bash
|
||||
|
||||
# Run all e2e specs
|
||||
maestro test e2e/
|
||||
```
|
||||
|
||||
Maestro YAML specs cover each screen:
|
||||
|
||||
| Spec | What It Verifies |
|
||||
|------|-----------------|
|
||||
| `live_screen.yaml` | 3D viewer loads, HUD elements visible, mode badge displays |
|
||||
| `vitals_screen.yaml` | Breathing and heart rate gauges render with values |
|
||||
| `zones_screen.yaml` | Floor plan SVG renders, zone legend visible |
|
||||
| `mat_screen.yaml` | Survivor counter displays, alert list populates |
|
||||
| `settings_screen.yaml` | URL input editable, theme picker works, toggles respond |
|
||||
| `offline_fallback.yaml` | App transitions to SIM mode when server unreachable |
|
||||
|
||||
---
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology | Version |
|
||||
|-------|-----------|---------|
|
||||
| Framework | Expo | 55 |
|
||||
| UI | React Native | 0.83 |
|
||||
| Language | TypeScript | 5.9 |
|
||||
| Navigation | React Navigation | 7.x |
|
||||
| State | Zustand | 5.x |
|
||||
| HTTP | Axios | 1.x |
|
||||
| SVG | react-native-svg | 15.x |
|
||||
| WebView | react-native-webview | 13.x |
|
||||
| WiFi | react-native-wifi-reborn | 4.x |
|
||||
| Charts | Victory Native | 41.x |
|
||||
| Animations | react-native-reanimated | 4.x |
|
||||
| Testing | Jest + jest-expo | 30.x |
|
||||
| E2E | Maestro | Latest |
|
||||
|
||||
---
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch from `main`
|
||||
3. Make changes in the `ui/mobile/` directory
|
||||
4. Run `npm test` and verify all tests pass
|
||||
5. Run `npx expo start --web` to verify the app renders correctly
|
||||
6. Submit a pull request
|
||||
|
||||
Follow the project's existing patterns:
|
||||
- Components go in `src/components/`
|
||||
- Screen-specific components go in `src/screens/<ScreenName>/`
|
||||
- Platform-specific files use the `.android.ts` / `.ios.ts` / `.web.ts` suffix convention
|
||||
- All state management uses Zustand stores in `src/stores/`
|
||||
- All types go in `src/types/`
|
||||
|
||||
---
|
||||
|
||||
## Credits
|
||||
|
||||
Mobile app by [@MaTriXy](https://github.com/MaTriXy) — original scaffold, screen architecture, and cross-platform service layer.
|
||||
|
||||
Built on the [WiFi-DensePose](../../README.md) sensing platform.
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
[MIT](../../LICENSE)
|
||||
@@ -0,0 +1,12 @@
|
||||
export default {
|
||||
name: 'WiFi-DensePose',
|
||||
slug: 'wifi-densepose',
|
||||
version: '1.0.0',
|
||||
ios: {
|
||||
bundleIdentifier: 'com.ruvnet.wifidensepose',
|
||||
},
|
||||
android: {
|
||||
package: 'com.ruvnet.wifidensepose',
|
||||
},
|
||||
// Use expo-env and app-level defaults from the project configuration when available.
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"expo": {
|
||||
"name": "mobile",
|
||||
"slug": "mobile",
|
||||
"version": "1.0.0",
|
||||
"orientation": "portrait",
|
||||
"icon": "./assets/icon.png",
|
||||
"userInterfaceStyle": "light",
|
||||
"splash": {
|
||||
"image": "./assets/splash-icon.png",
|
||||
"resizeMode": "contain",
|
||||
"backgroundColor": "#ffffff"
|
||||
},
|
||||
"ios": {
|
||||
"supportsTablet": true
|
||||
},
|
||||
"android": {
|
||||
"adaptiveIcon": {
|
||||
"backgroundColor": "#E6F4FE",
|
||||
"foregroundImage": "./assets/android-icon-foreground.png",
|
||||
"backgroundImage": "./assets/android-icon-background.png",
|
||||
"monochromeImage": "./assets/android-icon-monochrome.png"
|
||||
},
|
||||
"predictiveBackGestureEnabled": false
|
||||
},
|
||||
"web": {
|
||||
"favicon": "./assets/favicon.png"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 77 KiB |
|
After Width: | Height: | Size: 4.0 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 384 KiB |
|
After Width: | Height: | Size: 17 KiB |
@@ -0,0 +1,9 @@
|
||||
module.exports = function (api) {
|
||||
api.cache(true);
|
||||
return {
|
||||
presets: ['babel-preset-expo'],
|
||||
plugins: [
|
||||
'react-native-reanimated/plugin'
|
||||
]
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"cli": {
|
||||
"version": ">= 4.0.0"
|
||||
},
|
||||
"build": {
|
||||
"development": {
|
||||
"developmentClient": true,
|
||||
"distribution": "internal"
|
||||
},
|
||||
"preview": {
|
||||
"distribution": "internal"
|
||||
},
|
||||
"production": {
|
||||
"autoIncrement": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { registerRootComponent } from 'expo';
|
||||
import App from './App';
|
||||
|
||||
registerRootComponent(App);
|
||||
@@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
preset: 'jest-expo',
|
||||
setupFilesAfterEnv: ['<rootDir>/jest.setup.ts'],
|
||||
testPathIgnorePatterns: ['/src/__tests__/'],
|
||||
transformIgnorePatterns: [
|
||||
'node_modules/(?!(expo|expo-.+|react-native|@react-native|react-native-webview|react-native-reanimated|react-native-svg|react-native-safe-area-context|react-native-screens|@react-navigation|@expo|@unimodules|expo-modules-core)/)',
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
jest.mock('@react-native-async-storage/async-storage', () =>
|
||||
require('@react-native-async-storage/async-storage/jest/async-storage-mock')
|
||||
);
|
||||
|
||||
jest.mock('react-native-wifi-reborn', () => ({
|
||||
loadWifiList: jest.fn(async () => []),
|
||||
}));
|
||||
|
||||
jest.mock('react-native-reanimated', () =>
|
||||
require('react-native-reanimated/mock')
|
||||
);
|
||||
|
||||
jest.mock('react-native-webview', () => {
|
||||
const React = require('react');
|
||||
const { View } = require('react-native');
|
||||
|
||||
const MockWebView = (props: unknown) => React.createElement(View, props);
|
||||
|
||||
return {
|
||||
__esModule: true,
|
||||
default: MockWebView,
|
||||
WebView: MockWebView,
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
const { getDefaultConfig } = require('expo/metro-config');
|
||||
|
||||
const config = getDefaultConfig(__dirname);
|
||||
|
||||
// Force CJS resolution for packages that use import.meta (not supported in Hermes script mode)
|
||||
config.resolver = {
|
||||
...config.resolver,
|
||||
unstable_enablePackageExports: false,
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"name": "mobile",
|
||||
"version": "1.0.0",
|
||||
"main": "index.ts",
|
||||
"scripts": {
|
||||
"start": "expo start",
|
||||
"android": "expo start --android",
|
||||
"ios": "expo start --ios",
|
||||
"web": "expo start --web",
|
||||
"test": "jest",
|
||||
"lint": "eslint ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@expo/vector-icons": "^15.0.2",
|
||||
"@react-native-async-storage/async-storage": "2.2.0",
|
||||
"@react-navigation/bottom-tabs": "^7.15.3",
|
||||
"@react-navigation/native": "^7.1.31",
|
||||
"@types/three": "^0.183.1",
|
||||
"axios": "^1.13.6",
|
||||
"expo": "~55.0.4",
|
||||
"expo-status-bar": "~55.0.4",
|
||||
"react": "19.2.0",
|
||||
"react-dom": "19.2.0",
|
||||
"react-native": "0.83.2",
|
||||
"react-native-gesture-handler": "~2.30.0",
|
||||
"react-native-reanimated": "4.2.1",
|
||||
"react-native-safe-area-context": "~5.6.2",
|
||||
"react-native-screens": "~4.23.0",
|
||||
"react-native-svg": "15.15.3",
|
||||
"react-native-web": "^0.21.0",
|
||||
"react-native-webview": "13.16.0",
|
||||
"react-native-wifi-reborn": "^4.13.6",
|
||||
"three": "^0.183.2",
|
||||
"victory-native": "^41.20.2",
|
||||
"zustand": "^5.0.11"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/jest-native": "^5.4.3",
|
||||
"@testing-library/react-native": "^13.3.3",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/react": "~19.2.2",
|
||||
"@typescript-eslint/eslint-plugin": "^8.56.1",
|
||||
"@typescript-eslint/parser": "^8.56.1",
|
||||
"babel-preset-expo": "^55.0.10",
|
||||
"eslint": "^10.0.2",
|
||||
"jest": "^30.2.0",
|
||||
"jest-expo": "^55.0.9",
|
||||
"prettier": "^3.8.1",
|
||||
"react-native-worklets": "^0.7.4",
|
||||
"typescript": "~5.9.2"
|
||||
},
|
||||
"private": true
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import React, { PropsWithChildren } from 'react';
|
||||
import { render, type RenderOptions } from '@testing-library/react-native';
|
||||
import { NavigationContainer } from '@react-navigation/native';
|
||||
import { GestureHandlerRootView } from 'react-native-gesture-handler';
|
||||
import { SafeAreaProvider } from 'react-native-safe-area-context';
|
||||
import { ThemeProvider } from '@/theme/ThemeContext';
|
||||
|
||||
type TestProvidersProps = PropsWithChildren<object>;
|
||||
|
||||
const TestProviders = ({ children }: TestProvidersProps) => (
|
||||
<GestureHandlerRootView style={{ flex: 1 }}>
|
||||
<SafeAreaProvider>
|
||||
<ThemeProvider>{children}</ThemeProvider>
|
||||
</SafeAreaProvider>
|
||||
</GestureHandlerRootView>
|
||||
);
|
||||
|
||||
const TestProvidersWithNavigation = ({ children }: TestProvidersProps) => (
|
||||
<TestProviders>
|
||||
<NavigationContainer>{children}</NavigationContainer>
|
||||
</TestProviders>
|
||||
);
|
||||
|
||||
interface RenderWithProvidersOptions extends Omit<RenderOptions, 'wrapper'> {
|
||||
withNavigation?: boolean;
|
||||
}
|
||||
|
||||
export const renderWithProviders = (
|
||||
ui: React.ReactElement,
|
||||
{ withNavigation, ...options }: RenderWithProvidersOptions = {},
|
||||
) => {
|
||||
return render(ui, {
|
||||
...options,
|
||||
wrapper: withNavigation ? TestProvidersWithNavigation : TestProviders,
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
describe('placeholder', () => {
|
||||
it('passes', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,585 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta
|
||||
name="viewport"
|
||||
content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"
|
||||
/>
|
||||
<meta
|
||||
http-equiv="Content-Security-Policy"
|
||||
content="default-src 'self'; script-src 'self' https://cdnjs.cloudflare.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'"
|
||||
/>
|
||||
<title>WiFi DensePose Splat Viewer</title>
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#gaussian-splat-root {
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
background: #0a0e1a;
|
||||
touch-action: none;
|
||||
}
|
||||
|
||||
#gaussian-splat-root {
|
||||
position: relative;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="gaussian-splat-root"></div>
|
||||
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r165/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.165.0/examples/js/controls/OrbitControls.js"></script>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const postMessageToRN = (message) => {
|
||||
if (!window.ReactNativeWebView || typeof window.ReactNativeWebView.postMessage !== 'function') {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
window.ReactNativeWebView.postMessage(JSON.stringify(message));
|
||||
} catch (error) {
|
||||
console.error('Failed to post RN message', error);
|
||||
}
|
||||
};
|
||||
|
||||
const postError = (message) => {
|
||||
postMessageToRN({
|
||||
type: 'ERROR',
|
||||
payload: {
|
||||
message: typeof message === 'string' ? message : 'Unknown bridge error',
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// Use global THREE from CDN
|
||||
const getThree = () => window.THREE;
|
||||
|
||||
// ---- Custom Splat Shaders --------------------------------------------
|
||||
|
||||
const SPLAT_VERTEX = `
|
||||
attribute float splatSize;
|
||||
attribute vec3 splatColor;
|
||||
attribute float splatOpacity;
|
||||
|
||||
varying vec3 vColor;
|
||||
varying float vOpacity;
|
||||
|
||||
void main() {
|
||||
vColor = splatColor;
|
||||
vOpacity = splatOpacity;
|
||||
|
||||
vec4 mvPosition = modelViewMatrix * vec4(position, 1.0);
|
||||
gl_PointSize = splatSize * (300.0 / -mvPosition.z);
|
||||
gl_Position = projectionMatrix * mvPosition;
|
||||
}
|
||||
`;
|
||||
|
||||
const SPLAT_FRAGMENT = `
|
||||
varying vec3 vColor;
|
||||
varying float vOpacity;
|
||||
|
||||
void main() {
|
||||
// Circular soft-edge disc
|
||||
float dist = length(gl_PointCoord - vec2(0.5));
|
||||
if (dist > 0.5) discard;
|
||||
float alpha = smoothstep(0.5, 0.2, dist) * vOpacity;
|
||||
gl_FragColor = vec4(vColor, alpha);
|
||||
}
|
||||
`;
|
||||
|
||||
// ---- Color helpers ---------------------------------------------------
|
||||
|
||||
/** Map a scalar 0-1 to blue -> green -> red gradient */
|
||||
function valueToColor(v) {
|
||||
const clamped = Math.max(0, Math.min(1, v));
|
||||
// blue(0) -> cyan(0.25) -> green(0.5) -> yellow(0.75) -> red(1)
|
||||
let r;
|
||||
let g;
|
||||
let b;
|
||||
if (clamped < 0.5) {
|
||||
const t = clamped * 2;
|
||||
r = 0;
|
||||
g = t;
|
||||
b = 1 - t;
|
||||
} else {
|
||||
const t = (clamped - 0.5) * 2;
|
||||
r = t;
|
||||
g = 1 - t;
|
||||
b = 0;
|
||||
}
|
||||
return [r, g, b];
|
||||
}
|
||||
|
||||
// ---- GaussianSplatRenderer -------------------------------------------
|
||||
|
||||
class GaussianSplatRenderer {
|
||||
/** @param {HTMLElement} container - DOM element to attach the renderer to */
|
||||
constructor(container, opts = {}) {
|
||||
const THREE = getThree();
|
||||
if (!THREE) {
|
||||
throw new Error('Three.js not loaded');
|
||||
}
|
||||
|
||||
this.container = container;
|
||||
this.width = opts.width || container.clientWidth || 800;
|
||||
this.height = opts.height || 500;
|
||||
|
||||
// Scene
|
||||
this.scene = new THREE.Scene();
|
||||
this.scene.background = new THREE.Color(0x0a0e1a);
|
||||
|
||||
// Camera — perspective looking down at the room
|
||||
this.camera = new THREE.PerspectiveCamera(45, this.width / this.height, 0.1, 200);
|
||||
this.camera.position.set(0, 10, 12);
|
||||
this.camera.lookAt(0, 0, 0);
|
||||
|
||||
// Renderer
|
||||
this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
|
||||
this.renderer.setSize(this.width, this.height);
|
||||
this.renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||
container.appendChild(this.renderer.domElement);
|
||||
|
||||
// Lights
|
||||
const ambient = new THREE.AmbientLight(0x9ec7ff, 0.35);
|
||||
this.scene.add(ambient);
|
||||
|
||||
const directional = new THREE.DirectionalLight(0x9ec7ff, 0.65);
|
||||
directional.position.set(4, 10, 6);
|
||||
directional.castShadow = false;
|
||||
this.scene.add(directional);
|
||||
|
||||
// Grid & room
|
||||
this._createRoom(THREE);
|
||||
|
||||
// Signal field splats (20x20 = 400 points on the floor plane)
|
||||
this.gridSize = 20;
|
||||
this._createFieldSplats(THREE);
|
||||
|
||||
// Node markers (ESP32 / router positions)
|
||||
this._createNodeMarkers(THREE);
|
||||
|
||||
// Body disruption blob
|
||||
this._createBodyBlob(THREE);
|
||||
|
||||
// Orbit controls for drag + pinch zoom
|
||||
this.controls = new THREE.OrbitControls(this.camera, this.renderer.domElement);
|
||||
this.controls.target.set(0, 0, 0);
|
||||
this.controls.minDistance = 6;
|
||||
this.controls.maxDistance = 40;
|
||||
this.controls.enableDamping = true;
|
||||
this.controls.dampingFactor = 0.08;
|
||||
this.controls.update();
|
||||
|
||||
// Animation state
|
||||
this._animFrame = null;
|
||||
this._lastData = null;
|
||||
this._fpsFrames = [];
|
||||
this._lastFpsReport = 0;
|
||||
|
||||
// Start render loop
|
||||
this._animate();
|
||||
}
|
||||
|
||||
// ---- Scene setup ---------------------------------------------------
|
||||
|
||||
_createRoom(THREE) {
|
||||
// Floor grid (on y = 0), 20 units
|
||||
const grid = new THREE.GridHelper(20, 20, 0x1a3a4a, 0x0d1f28);
|
||||
grid.position.y = 0;
|
||||
this.scene.add(grid);
|
||||
|
||||
// Room boundary wireframe
|
||||
const boxGeo = new THREE.BoxGeometry(20, 6, 20);
|
||||
const edges = new THREE.EdgesGeometry(boxGeo);
|
||||
const line = new THREE.LineSegments(
|
||||
edges,
|
||||
new THREE.LineBasicMaterial({ color: 0x1a4a5a, opacity: 0.3, transparent: true }),
|
||||
);
|
||||
line.position.y = 3;
|
||||
this.scene.add(line);
|
||||
}
|
||||
|
||||
_createFieldSplats(THREE) {
|
||||
const count = this.gridSize * this.gridSize;
|
||||
|
||||
const positions = new Float32Array(count * 3);
|
||||
const sizes = new Float32Array(count);
|
||||
const colors = new Float32Array(count * 3);
|
||||
const opacities = new Float32Array(count);
|
||||
|
||||
// Lay splats on the floor plane (y = 0.05 to sit just above grid)
|
||||
for (let iz = 0; iz < this.gridSize; iz++) {
|
||||
for (let ix = 0; ix < this.gridSize; ix++) {
|
||||
const idx = iz * this.gridSize + ix;
|
||||
positions[idx * 3 + 0] = (ix - this.gridSize / 2) + 0.5; // x
|
||||
positions[idx * 3 + 1] = 0.05; // y
|
||||
positions[idx * 3 + 2] = (iz - this.gridSize / 2) + 0.5; // z
|
||||
|
||||
sizes[idx] = 1.5;
|
||||
colors[idx * 3] = 0.1;
|
||||
colors[idx * 3 + 1] = 0.2;
|
||||
colors[idx * 3 + 2] = 0.6;
|
||||
opacities[idx] = 0.15;
|
||||
}
|
||||
}
|
||||
|
||||
const geo = new THREE.BufferGeometry();
|
||||
geo.setAttribute('position', new THREE.BufferAttribute(positions, 3));
|
||||
geo.setAttribute('splatSize', new THREE.BufferAttribute(sizes, 1));
|
||||
geo.setAttribute('splatColor', new THREE.BufferAttribute(colors, 3));
|
||||
geo.setAttribute('splatOpacity', new THREE.BufferAttribute(opacities, 1));
|
||||
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
vertexShader: SPLAT_VERTEX,
|
||||
fragmentShader: SPLAT_FRAGMENT,
|
||||
transparent: true,
|
||||
depthWrite: false,
|
||||
blending: THREE.AdditiveBlending,
|
||||
});
|
||||
|
||||
this.fieldPoints = new THREE.Points(geo, mat);
|
||||
this.scene.add(this.fieldPoints);
|
||||
}
|
||||
|
||||
_createNodeMarkers(THREE) {
|
||||
// Router at center — green sphere
|
||||
const routerGeo = new THREE.SphereGeometry(0.3, 16, 16);
|
||||
const routerMat = new THREE.MeshBasicMaterial({ color: 0x00ff88, transparent: true, opacity: 0.8 });
|
||||
this.routerMarker = new THREE.Mesh(routerGeo, routerMat);
|
||||
this.routerMarker.position.set(0, 0.5, 0);
|
||||
this.scene.add(this.routerMarker);
|
||||
|
||||
// ESP32 node — cyan sphere (default position, updated from data)
|
||||
const nodeGeo = new THREE.SphereGeometry(0.25, 16, 16);
|
||||
const nodeMat = new THREE.MeshBasicMaterial({ color: 0x00ccff, transparent: true, opacity: 0.8 });
|
||||
this.nodeMarker = new THREE.Mesh(nodeGeo, nodeMat);
|
||||
this.nodeMarker.position.set(2, 0.5, 1.5);
|
||||
this.scene.add(this.nodeMarker);
|
||||
}
|
||||
|
||||
_createBodyBlob(THREE) {
|
||||
// A cluster of splats representing body disruption
|
||||
const count = 64;
|
||||
const positions = new Float32Array(count * 3);
|
||||
const sizes = new Float32Array(count);
|
||||
const colors = new Float32Array(count * 3);
|
||||
const opacities = new Float32Array(count);
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
// Random sphere distribution
|
||||
const theta = Math.random() * Math.PI * 2;
|
||||
const phi = Math.acos(2 * Math.random() - 1);
|
||||
const r = Math.random() * 1.5;
|
||||
positions[i * 3] = r * Math.sin(phi) * Math.cos(theta);
|
||||
positions[i * 3 + 1] = r * Math.cos(phi) + 2;
|
||||
positions[i * 3 + 2] = r * Math.sin(phi) * Math.sin(theta);
|
||||
|
||||
sizes[i] = 2 + Math.random() * 3;
|
||||
colors[i * 3] = 0.2;
|
||||
colors[i * 3 + 1] = 0.8;
|
||||
colors[i * 3 + 2] = 0.3;
|
||||
opacities[i] = 0.0; // hidden until presence detected
|
||||
}
|
||||
|
||||
const geo = new THREE.BufferGeometry();
|
||||
geo.setAttribute('position', new THREE.BufferAttribute(positions, 3));
|
||||
geo.setAttribute('splatSize', new THREE.BufferAttribute(sizes, 1));
|
||||
geo.setAttribute('splatColor', new THREE.BufferAttribute(colors, 3));
|
||||
geo.setAttribute('splatOpacity', new THREE.BufferAttribute(opacities, 1));
|
||||
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
vertexShader: SPLAT_VERTEX,
|
||||
fragmentShader: SPLAT_FRAGMENT,
|
||||
transparent: true,
|
||||
depthWrite: false,
|
||||
blending: THREE.AdditiveBlending,
|
||||
});
|
||||
|
||||
this.bodyBlob = new THREE.Points(geo, mat);
|
||||
this.scene.add(this.bodyBlob);
|
||||
}
|
||||
|
||||
// ---- Data update --------------------------------------------------
|
||||
|
||||
/**
|
||||
* Update the visualization with new sensing data.
|
||||
* @param {object} data - sensing_update JSON from ws_server
|
||||
*/
|
||||
update(data) {
|
||||
this._lastData = data;
|
||||
if (!data) return;
|
||||
|
||||
const features = data.features || {};
|
||||
const classification = data.classification || {};
|
||||
const signalField = data.signal_field || {};
|
||||
const nodes = data.nodes || [];
|
||||
|
||||
// -- Update signal field splats ------------------------------------
|
||||
if (signalField.values && this.fieldPoints) {
|
||||
const geo = this.fieldPoints.geometry;
|
||||
const clr = geo.attributes.splatColor.array;
|
||||
const sizes = geo.attributes.splatSize.array;
|
||||
const opac = geo.attributes.splatOpacity.array;
|
||||
const vals = signalField.values;
|
||||
const count = Math.min(vals.length, this.gridSize * this.gridSize);
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const v = vals[i];
|
||||
const [r, g, b] = valueToColor(v);
|
||||
clr[i * 3] = r;
|
||||
clr[i * 3 + 1] = g;
|
||||
clr[i * 3 + 2] = b;
|
||||
sizes[i] = 1.0 + v * 4.0;
|
||||
opac[i] = 0.1 + v * 0.6;
|
||||
}
|
||||
|
||||
geo.attributes.splatColor.needsUpdate = true;
|
||||
geo.attributes.splatSize.needsUpdate = true;
|
||||
geo.attributes.splatOpacity.needsUpdate = true;
|
||||
}
|
||||
|
||||
// -- Update body blob ----------------------------------------------
|
||||
if (this.bodyBlob) {
|
||||
const bGeo = this.bodyBlob.geometry;
|
||||
const bOpac = bGeo.attributes.splatOpacity.array;
|
||||
const bClr = bGeo.attributes.splatColor.array;
|
||||
const bSize = bGeo.attributes.splatSize.array;
|
||||
|
||||
const presence = classification.presence || false;
|
||||
const motionLvl = classification.motion_level || 'absent';
|
||||
const confidence = classification.confidence || 0;
|
||||
const breathing = features.breathing_band_power || 0;
|
||||
|
||||
// Breathing pulsation
|
||||
const breathPulse = 1.0 + Math.sin(Date.now() * 0.004) * Math.min(breathing * 3, 0.4);
|
||||
|
||||
for (let i = 0; i < bOpac.length; i++) {
|
||||
if (presence) {
|
||||
bOpac[i] = confidence * 0.4;
|
||||
|
||||
// Color by motion level
|
||||
if (motionLvl === 'active') {
|
||||
bClr[i * 3] = 1.0;
|
||||
bClr[i * 3 + 1] = 0.2;
|
||||
bClr[i * 3 + 2] = 0.1;
|
||||
} else {
|
||||
bClr[i * 3] = 0.1;
|
||||
bClr[i * 3 + 1] = 0.8;
|
||||
bClr[i * 3 + 2] = 0.4;
|
||||
}
|
||||
|
||||
bSize[i] = (2 + Math.random() * 2) * breathPulse;
|
||||
} else {
|
||||
bOpac[i] = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
bGeo.attributes.splatOpacity.needsUpdate = true;
|
||||
bGeo.attributes.splatColor.needsUpdate = true;
|
||||
bGeo.attributes.splatSize.needsUpdate = true;
|
||||
}
|
||||
|
||||
// -- Update node positions -----------------------------------------
|
||||
if (nodes.length > 0 && nodes[0].position && this.nodeMarker) {
|
||||
const pos = nodes[0].position;
|
||||
this.nodeMarker.position.set(pos[0], 0.5, pos[2]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Render loop -------------------------------------------------
|
||||
|
||||
_animate() {
|
||||
this._animFrame = requestAnimationFrame(() => this._animate());
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
// Gentle router glow pulse
|
||||
if (this.routerMarker) {
|
||||
const pulse = 0.6 + 0.3 * Math.sin(now * 0.003);
|
||||
this.routerMarker.material.opacity = pulse;
|
||||
}
|
||||
|
||||
this.controls.update();
|
||||
this.renderer.render(this.scene, this.camera);
|
||||
|
||||
this._fpsFrames.push(now);
|
||||
while (this._fpsFrames.length > 0 && this._fpsFrames[0] < now - 1000) {
|
||||
this._fpsFrames.shift();
|
||||
}
|
||||
|
||||
if (now - this._lastFpsReport >= 1000) {
|
||||
const fps = this._fpsFrames.length;
|
||||
this._lastFpsReport = now;
|
||||
postMessageToRN({
|
||||
type: 'FPS_TICK',
|
||||
payload: { fps },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Resize / cleanup --------------------------------------------
|
||||
|
||||
resize(width, height) {
|
||||
if (!width || !height) return;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.camera.aspect = width / height;
|
||||
this.camera.updateProjectionMatrix();
|
||||
this.renderer.setSize(width, height);
|
||||
}
|
||||
|
||||
dispose() {
|
||||
if (this._animFrame) {
|
||||
cancelAnimationFrame(this._animFrame);
|
||||
}
|
||||
|
||||
this.controls?.dispose();
|
||||
this.renderer.dispose();
|
||||
if (this.renderer.domElement.parentNode) {
|
||||
this.renderer.domElement.parentNode.removeChild(this.renderer.domElement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Expose renderer constructor for debugging/interop
|
||||
window.GaussianSplatRenderer = GaussianSplatRenderer;
|
||||
|
||||
let renderer = null;
|
||||
let pendingFrame = null;
|
||||
let pendingResize = null;
|
||||
|
||||
const postSafeReady = () => {
|
||||
postMessageToRN({ type: 'READY' });
|
||||
};
|
||||
|
||||
const routeMessage = (event) => {
|
||||
let raw = event.data;
|
||||
if (typeof raw === 'object' && raw != null && 'data' in raw) {
|
||||
raw = raw.data;
|
||||
}
|
||||
|
||||
let message = raw;
|
||||
if (typeof raw === 'string') {
|
||||
try {
|
||||
message = JSON.parse(raw);
|
||||
} catch (err) {
|
||||
postError('Failed to parse RN message payload');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!message || typeof message !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'FRAME_UPDATE') {
|
||||
const payload = message.payload || null;
|
||||
if (!payload) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!renderer) {
|
||||
pendingFrame = payload;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
renderer.update(payload);
|
||||
} catch (error) {
|
||||
postError((error && error.message) || 'Failed to update frame');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'RESIZE') {
|
||||
const dims = message.payload || {};
|
||||
const w = Number(dims.width);
|
||||
const h = Number(dims.height);
|
||||
if (!Number.isFinite(w) || !Number.isFinite(h) || !w || !h) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!renderer) {
|
||||
pendingResize = { width: w, height: h };
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
renderer.resize(w, h);
|
||||
} catch (error) {
|
||||
postError((error && error.message) || 'Failed to resize renderer');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'DISPOSE') {
|
||||
if (!renderer) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
renderer.dispose();
|
||||
} catch (error) {
|
||||
postError((error && error.message) || 'Failed to dispose renderer');
|
||||
}
|
||||
renderer = null;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const buildRenderer = () => {
|
||||
const container = document.getElementById('gaussian-splat-root');
|
||||
if (!container) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
renderer = new GaussianSplatRenderer(container, {
|
||||
width: container.clientWidth || window.innerWidth,
|
||||
height: container.clientHeight || window.innerHeight,
|
||||
});
|
||||
|
||||
if (pendingFrame) {
|
||||
renderer.update(pendingFrame);
|
||||
pendingFrame = null;
|
||||
}
|
||||
|
||||
if (pendingResize) {
|
||||
renderer.resize(pendingResize.width, pendingResize.height);
|
||||
pendingResize = null;
|
||||
}
|
||||
|
||||
postSafeReady();
|
||||
} catch (error) {
|
||||
renderer = null;
|
||||
postError((error && error.message) || 'Failed to initialize renderer');
|
||||
}
|
||||
};
|
||||
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', buildRenderer);
|
||||
} else {
|
||||
buildRenderer();
|
||||
}
|
||||
|
||||
window.addEventListener('message', routeMessage);
|
||||
window.addEventListener('resize', () => {
|
||||
if (!renderer) {
|
||||
pendingResize = {
|
||||
width: window.innerWidth,
|
||||
height: window.innerHeight,
|
||||
};
|
||||
return;
|
||||
}
|
||||
renderer.resize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,505 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>MAT Dashboard</title>
|
||||
<style>
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: #0a0e1a;
|
||||
color: #e5e7eb;
|
||||
font-family: 'Courier New', 'Consolas', monospace;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
#status {
|
||||
color: #6dd4df;
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
#mapCanvas {
|
||||
flex: 1;
|
||||
width: 100%;
|
||||
border: 1px solid #1e293b;
|
||||
border-radius: 8px;
|
||||
min-height: 180px;
|
||||
background: #0a0e1a;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app">
|
||||
<div id="status">Initializing MAT dashboard...</div>
|
||||
<canvas id="mapCanvas"></canvas>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const TRIAGE = {
|
||||
Immediate: 0,
|
||||
Delayed: 1,
|
||||
Minimal: 2,
|
||||
Expectant: 3,
|
||||
Unknown: 4,
|
||||
};
|
||||
|
||||
const TRIAGE_COLOR = ['#ff0000', '#ffcc00', '#00cc00', '#111111', '#888888'];
|
||||
const PRIORITY = { Critical: 0, High: 1, Medium: 2, Low: 3 };
|
||||
|
||||
const toRgba = (status) => TRIAGE_COLOR[status] || TRIAGE_COLOR[4];
|
||||
const safeId = () =>
|
||||
typeof crypto !== 'undefined' && crypto.randomUUID
|
||||
? crypto.randomUUID()
|
||||
: `id-${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
|
||||
|
||||
const isNumber = (value) => typeof value === 'number' && Number.isFinite(value);
|
||||
|
||||
class MatDashboard {
|
||||
constructor() {
|
||||
this.event = null;
|
||||
this.zones = new Map();
|
||||
this.survivors = new Map();
|
||||
this.alerts = new Map();
|
||||
this.motionVector = { x: 0, y: 0 };
|
||||
}
|
||||
|
||||
createEvent(type, lat, lon, name) {
|
||||
const eventId = safeId();
|
||||
this.event = {
|
||||
event_id: eventId,
|
||||
disaster_type: type,
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
description: name,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.zones.clear();
|
||||
this.survivors.clear();
|
||||
this.alerts.clear();
|
||||
return eventId;
|
||||
}
|
||||
|
||||
addRectangleZone(name, x, y, w, h) {
|
||||
const id = safeId();
|
||||
this.zones.set(id, {
|
||||
id,
|
||||
name,
|
||||
zone_type: 'rectangle',
|
||||
status: 0,
|
||||
scan_count: 0,
|
||||
detection_count: 0,
|
||||
x,
|
||||
y,
|
||||
width: w,
|
||||
height: h,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
addCircleZone(name, cx, cy, radius) {
|
||||
const id = safeId();
|
||||
this.zones.set(id, {
|
||||
id,
|
||||
name,
|
||||
zone_type: 'circle',
|
||||
status: 0,
|
||||
scan_count: 0,
|
||||
detection_count: 0,
|
||||
center_x: cx,
|
||||
center_y: cy,
|
||||
radius,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
addZoneFromPayload(payload) {
|
||||
if (!payload || typeof payload !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
const source = payload;
|
||||
const type = source.zone_type || source.type || 'rectangle';
|
||||
const name = source.name || `Zone-${safeId().slice(0, 4)}`;
|
||||
|
||||
if (type === 'circle' || source.center_x !== undefined) {
|
||||
const cx = isNumber(source.center_x) ? source.center_x : 120;
|
||||
const cy = isNumber(source.center_y) ? source.center_y : 120;
|
||||
const radius = isNumber(source.radius) ? source.radius : 50;
|
||||
return this.addCircleZone(name, cx, cy, radius);
|
||||
}
|
||||
|
||||
const x = isNumber(source.x) ? source.x : 40;
|
||||
const y = isNumber(source.y) ? source.y : 40;
|
||||
const width = isNumber(source.width) ? source.width : 100;
|
||||
const height = isNumber(source.height) ? source.height : 100;
|
||||
return this.addRectangleZone(name, x, y, width, height);
|
||||
}
|
||||
|
||||
inferTriage(vitalSigns, confidence) {
|
||||
const breathing = isNumber(vitalSigns?.breathing_rate) ? vitalSigns.breathing_rate : 14;
|
||||
const heart = isNumber(vitalSigns?.heart_rate)
|
||||
? vitalSigns.heart_rate
|
||||
: isNumber(vitalSigns?.hr)
|
||||
? vitalSigns.hr
|
||||
: 70;
|
||||
|
||||
if (!isNumber(confidence) || confidence > 0.82) {
|
||||
if (breathing < 10 || breathing > 35 || heart > 150) {
|
||||
return TRIAGE.Immediate;
|
||||
}
|
||||
if (breathing >= 8 && breathing <= 34) {
|
||||
return TRIAGE.Delayed;
|
||||
}
|
||||
}
|
||||
|
||||
if (breathing >= 6 && breathing <= 28 && heart > 45 && heart < 180) {
|
||||
return TRIAGE.Minimal;
|
||||
}
|
||||
|
||||
return TRIAGE.Expectant;
|
||||
}
|
||||
|
||||
locateZoneForPoint(x, y) {
|
||||
for (const [id, zone] of this.zones.entries()) {
|
||||
if (zone.zone_type === 'circle') {
|
||||
const dx = x - zone.center_x;
|
||||
const dy = y - zone.center_y;
|
||||
const inside = Math.sqrt(dx * dx + dy * dy) <= zone.radius;
|
||||
if (inside) {
|
||||
return id;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (x >= zone.x && x <= zone.x + zone.width && y >= zone.y && y <= zone.y + zone.height) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
return this.zones.size > 0 ? this.zones.keys().next().value : safeId();
|
||||
}
|
||||
|
||||
processSurvivorDetection(zone, confidence = 0.6, vital_signs = {}) {
|
||||
const zoneKey =
|
||||
typeof zone === 'string'
|
||||
? [...this.zones.values()].find((entry) => entry.id === zone || entry.name === zone)
|
||||
: null;
|
||||
|
||||
const selectedZone =
|
||||
zoneKey
|
||||
|| (this.zones.size > 0
|
||||
? [...this.zones.values()][Math.floor(Math.random() * Math.max(1, this.zones.size))]
|
||||
: null);
|
||||
|
||||
const bounds = this._pickPointInZone(selectedZone);
|
||||
const triageStatus = this.inferTriage(vital_signs, confidence);
|
||||
const breathingRate = isNumber(vital_signs?.breathing_rate)
|
||||
? vital_signs.breathing_rate
|
||||
: 10 + confidence * 28;
|
||||
const heartRate = isNumber(vital_signs?.heart_rate)
|
||||
? vital_signs.heart_rate
|
||||
: isNumber(vital_signs?.hr)
|
||||
? vital_signs.hr
|
||||
: 55 + confidence * 60;
|
||||
|
||||
const id = safeId();
|
||||
const zone_id = this.locateZoneForPoint(bounds.x, bounds.y);
|
||||
|
||||
const survivor = {
|
||||
id,
|
||||
zone_id,
|
||||
x: bounds.x,
|
||||
y: bounds.y,
|
||||
depth: -Math.abs(isNumber(vital_signs.depth) ? vital_signs.depth : Math.random() * 3),
|
||||
triage_status: triageStatus,
|
||||
triage_color: toRgba(triageStatus),
|
||||
confidence,
|
||||
breathing_rate: breathingRate,
|
||||
heart_rate: heartRate,
|
||||
first_detected: new Date().toISOString(),
|
||||
last_updated: new Date().toISOString(),
|
||||
is_deteriorating: false,
|
||||
};
|
||||
|
||||
this.survivors.set(id, survivor);
|
||||
if (selectedZone) {
|
||||
selectedZone.detection_count = (selectedZone.detection_count || 0) + 1;
|
||||
}
|
||||
|
||||
if (typeof this.postMessage === 'function') {
|
||||
this.postMessage({
|
||||
type: 'SURVIVOR_DETECTED',
|
||||
payload: survivor,
|
||||
});
|
||||
}
|
||||
|
||||
this.generateAlerts();
|
||||
return id;
|
||||
}
|
||||
|
||||
_pickPointInZone(zone) {
|
||||
if (!zone) {
|
||||
return {
|
||||
x: 220 + Math.random() * 80,
|
||||
y: 120 + Math.random() * 80,
|
||||
};
|
||||
}
|
||||
|
||||
if (zone.zone_type === 'circle') {
|
||||
const angle = Math.random() * Math.PI * 2;
|
||||
const radius = Math.random() * (zone.radius || 20);
|
||||
return {
|
||||
x: Math.max(10, Math.min(560, zone.center_x + Math.cos(angle) * radius)),
|
||||
y: Math.max(10, Math.min(280, zone.center_y + Math.sin(angle) * radius)),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
x: Math.max(zone.x || 5, Math.min((zone.x || 5) + (zone.width || 40), (zone.x || 5) + Math.random() * (zone.width || 40))),
|
||||
y: Math.max(zone.y || 5, Math.min((zone.y || 5) + (zone.height || 40), (zone.y || 5) + Math.random() * (zone.height || 40))),
|
||||
};
|
||||
}
|
||||
|
||||
generateAlerts() {
|
||||
for (const survivor of this.survivors.values()) {
|
||||
if ((survivor.triage_status !== TRIAGE.Immediate && survivor.triage_status !== TRIAGE.Delayed)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const alertId = `alert-${survivor.id}`;
|
||||
if (this.alerts.has(alertId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const priority =
|
||||
survivor.triage_status === TRIAGE.Immediate ? PRIORITY.Critical : PRIORITY.High;
|
||||
const message =
|
||||
survivor.triage_status === TRIAGE.Immediate
|
||||
? `Immediate rescue required at (${survivor.x.toFixed(0)}, ${survivor.y.toFixed(0)})`
|
||||
: `High-priority rescue needed at (${survivor.x.toFixed(0)}, ${survivor.y.toFixed(0)})`;
|
||||
const alert = {
|
||||
id: alertId,
|
||||
survivor_id: survivor.id,
|
||||
priority,
|
||||
title: survivor.triage_status === TRIAGE.Immediate ? 'URGENT' : 'HIGH',
|
||||
message,
|
||||
recommended_action: survivor.triage_status === TRIAGE.Immediate ? 'Dispatch now' : 'Coordinate rescue',
|
||||
triage_status: survivor.triage_status,
|
||||
location_x: survivor.x,
|
||||
location_y: survivor.y,
|
||||
created_at: new Date().toISOString(),
|
||||
priority_color: survivor.triage_status === TRIAGE.Immediate ? '#ff0000' : '#ff8c00',
|
||||
};
|
||||
|
||||
this.alerts.set(alertId, alert);
|
||||
if (typeof this.postMessage === 'function') {
|
||||
this.postMessage({
|
||||
type: 'ALERT_GENERATED',
|
||||
payload: alert,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processFrame(frame) {
|
||||
const motion = Number(frame?.features?.motion_band_power || 0);
|
||||
const xDelta = isNumber(motion) ? (motion - 0.1) * 4 : 0;
|
||||
const yDelta = isNumber(frame?.features?.breathing_band_power || 0)
|
||||
? (frame.features.breathing_band_power - 0.1) * 3
|
||||
: 0;
|
||||
this.motionVector = { x: xDelta || 0, y: yDelta || 0 };
|
||||
|
||||
for (const survivor of this.survivors.values()) {
|
||||
const jitterX = (Math.random() - 0.5) * 2;
|
||||
const jitterY = (Math.random() - 0.5) * 2;
|
||||
survivor.x = Math.max(5, Math.min(560, survivor.x + this.motionVector.x + jitterX));
|
||||
survivor.y = Math.max(5, Math.min(280, survivor.y + this.motionVector.y + jitterY));
|
||||
survivor.last_updated = new Date().toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
renderZones(ctx) {
|
||||
for (const zone of this.zones.values()) {
|
||||
const fill = 'rgba(0, 150, 255, 0.3)';
|
||||
ctx.strokeStyle = '#0096ff';
|
||||
ctx.fillStyle = fill;
|
||||
ctx.lineWidth = 2;
|
||||
|
||||
if (zone.zone_type === 'circle') {
|
||||
ctx.beginPath();
|
||||
ctx.arc(zone.center_x, zone.center_y, zone.radius, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
ctx.stroke();
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.font = '12px monospace';
|
||||
ctx.fillText(zone.name, zone.center_x - 22, zone.center_y);
|
||||
} else {
|
||||
ctx.fillRect(zone.x, zone.y, zone.width, zone.height);
|
||||
ctx.strokeRect(zone.x, zone.y, zone.width, zone.height);
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.font = '12px monospace';
|
||||
ctx.fillText(zone.name, zone.x + 4, zone.y + 14);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
renderSurvivors(ctx) {
|
||||
for (const survivor of this.survivors.values()) {
|
||||
const radius = survivor.is_deteriorating ? 11 : 9;
|
||||
|
||||
if (survivor.triage_status === TRIAGE.Immediate) {
|
||||
ctx.fillStyle = 'rgba(255, 0, 0, 0.26)';
|
||||
ctx.beginPath();
|
||||
ctx.arc(survivor.x, survivor.y, radius + 6, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
ctx.fillStyle = survivor.triage_color || toRgba(TRIAGE.Minimal);
|
||||
ctx.font = 'bold 18px monospace';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.textBaseline = 'middle';
|
||||
ctx.fillText('✦', survivor.x, survivor.y);
|
||||
ctx.strokeStyle = '#ffffff';
|
||||
ctx.lineWidth = 1.5;
|
||||
ctx.beginPath();
|
||||
ctx.arc(survivor.x, survivor.y, radius, 0, Math.PI * 2);
|
||||
ctx.stroke();
|
||||
|
||||
if (survivor.depth < 0) {
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.font = '9px monospace';
|
||||
ctx.fillText(`${Math.abs(survivor.depth).toFixed(1)}m`, survivor.x + radius + 4, survivor.y + 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
render(ctx, width, height) {
|
||||
ctx.clearRect(0, 0, width, height);
|
||||
ctx.fillStyle = '#0a0e1a';
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
|
||||
ctx.strokeStyle = '#1f2a3d';
|
||||
ctx.lineWidth = 1;
|
||||
const grid = 40;
|
||||
for (let x = 0; x <= width; x += grid) {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x, 0);
|
||||
ctx.lineTo(x, height);
|
||||
ctx.stroke();
|
||||
}
|
||||
for (let y = 0; y <= height; y += grid) {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(0, y);
|
||||
ctx.lineTo(width, y);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
this.renderZones(ctx);
|
||||
this.renderSurvivors(ctx);
|
||||
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.font = '12px monospace';
|
||||
const stats = {
|
||||
survivors: this.survivors.size,
|
||||
alerts: this.alerts.size,
|
||||
};
|
||||
ctx.fillText(`Survivors: ${stats.survivors}`, 12, 20);
|
||||
ctx.fillText(`Alerts: ${stats.alerts}`, 12, 36);
|
||||
}
|
||||
|
||||
postMessage(message) {
|
||||
if (typeof window.ReactNativeWebView !== 'undefined' && window.ReactNativeWebView.postMessage) {
|
||||
window.ReactNativeWebView.postMessage(JSON.stringify(message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dashboard = new MatDashboard();
|
||||
const canvas = document.getElementById('mapCanvas');
|
||||
const ctx = canvas.getContext('2d');
|
||||
const status = document.getElementById('status');
|
||||
|
||||
const resize = () => {
|
||||
canvas.width = Math.max(200, Math.floor(canvas.parentElement.clientWidth - 2));
|
||||
canvas.height = Math.max(180, Math.floor(canvas.parentElement.clientHeight - 20));
|
||||
};
|
||||
|
||||
const startup = () => {
|
||||
dashboard.createEvent('earthquake', 37.7749, -122.4194, 'Training Scenario');
|
||||
dashboard.addRectangleZone('Zone A', 60, 45, 170, 120);
|
||||
dashboard.addCircleZone('Zone B', 300, 170, 70);
|
||||
dashboard.processSurvivorDetection('Zone A', 0.94, { breathing_rate: 11, hr: 128 });
|
||||
dashboard.processSurvivorDetection('Zone A', 0.88, { breathing_rate: 16, hr: 118 });
|
||||
dashboard.processSurvivorDetection('Zone B', 0.71, { breathing_rate: 9, hr: 142 });
|
||||
status.textContent = 'MAT dashboard ready';
|
||||
dashboard.postMessage({ type: 'READY' });
|
||||
};
|
||||
|
||||
const loop = () => {
|
||||
if (dashboard.zones.size > 0) {
|
||||
dashboard.render(ctx, canvas.width, canvas.height);
|
||||
}
|
||||
requestAnimationFrame(loop);
|
||||
};
|
||||
|
||||
window.addEventListener('resize', resize);
|
||||
|
||||
window.addEventListener('message', (evt) => {
|
||||
let incoming = evt.data;
|
||||
try {
|
||||
if (typeof incoming === 'string') {
|
||||
incoming = JSON.parse(incoming);
|
||||
}
|
||||
} catch {
|
||||
incoming = null;
|
||||
}
|
||||
|
||||
if (!incoming || typeof incoming !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (incoming.type === 'CREATE_EVENT') {
|
||||
const payload = incoming.payload || {};
|
||||
dashboard.createEvent(
|
||||
payload.type || payload.disaster_type || 'earthquake',
|
||||
payload.latitude || 0,
|
||||
payload.longitude || 0,
|
||||
payload.name || payload.description || 'Disaster Event',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (incoming.type === 'ADD_ZONE') {
|
||||
dashboard.addZoneFromPayload(incoming.payload || {});
|
||||
return;
|
||||
}
|
||||
|
||||
if (incoming.type === 'FRAME_UPDATE') {
|
||||
dashboard.processFrame(incoming.payload || {});
|
||||
}
|
||||
});
|
||||
|
||||
resize();
|
||||
startup();
|
||||
loop();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,70 @@
|
||||
import { StyleSheet, View } from 'react-native';
|
||||
import { ThemedText } from './ThemedText';
|
||||
|
||||
type ConnectionState = 'connected' | 'simulated' | 'disconnected';
|
||||
|
||||
type ConnectionBannerProps = {
|
||||
status: ConnectionState;
|
||||
};
|
||||
|
||||
const resolveState = (status: ConnectionState) => {
|
||||
if (status === 'connected') {
|
||||
return {
|
||||
label: 'LIVE STREAM',
|
||||
backgroundColor: '#0F6B2A',
|
||||
textColor: '#E2FFEA',
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 'disconnected') {
|
||||
return {
|
||||
label: 'DISCONNECTED',
|
||||
backgroundColor: '#8A1E2A',
|
||||
textColor: '#FFE3E7',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
label: 'SIMULATED DATA',
|
||||
backgroundColor: '#9A5F0C',
|
||||
textColor: '#FFF3E1',
|
||||
};
|
||||
};
|
||||
|
||||
export const ConnectionBanner = ({ status }: ConnectionBannerProps) => {
|
||||
const state = resolveState(status);
|
||||
|
||||
return (
|
||||
<View
|
||||
style={[
|
||||
styles.banner,
|
||||
{
|
||||
backgroundColor: state.backgroundColor,
|
||||
borderBottomColor: state.textColor,
|
||||
},
|
||||
]}
|
||||
>
|
||||
<ThemedText preset="labelMd" style={[styles.text, { color: state.textColor }]}>
|
||||
{state.label}
|
||||
</ThemedText>
|
||||
</View>
|
||||
);
|
||||
};
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
banner: {
|
||||
position: 'absolute',
|
||||
left: 0,
|
||||
right: 0,
|
||||
top: 0,
|
||||
zIndex: 100,
|
||||
paddingVertical: 6,
|
||||
borderBottomWidth: 2,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
},
|
||||
text: {
|
||||
letterSpacing: 2,
|
||||
fontWeight: '700',
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Component, ErrorInfo, ReactNode } from 'react';
|
||||
import { Button, StyleSheet, View } from 'react-native';
|
||||
import { ThemedText } from './ThemedText';
|
||||
import { ThemedView } from './ThemedView';
|
||||
|
||||
type ErrorBoundaryProps = {
|
||||
children: ReactNode;
|
||||
};
|
||||
|
||||
type ErrorBoundaryState = {
|
||||
hasError: boolean;
|
||||
error?: Error;
|
||||
};
|
||||
|
||||
export class ErrorBoundary extends Component<ErrorBoundaryProps, ErrorBoundaryState> {
|
||||
constructor(props: ErrorBoundaryProps) {
|
||||
super(props);
|
||||
this.state = { hasError: false };
|
||||
}
|
||||
|
||||
static getDerivedStateFromError(error: Error): ErrorBoundaryState {
|
||||
return { hasError: true, error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error, errorInfo: ErrorInfo) {
|
||||
console.error('ErrorBoundary caught an error', error, errorInfo);
|
||||
}
|
||||
|
||||
handleRetry = () => {
|
||||
this.setState({ hasError: false, error: undefined });
|
||||
};
|
||||
|
||||
render() {
|
||||
if (this.state.hasError) {
|
||||
return (
|
||||
<ThemedView style={styles.container}>
|
||||
<ThemedText preset="displayMd">Something went wrong</ThemedText>
|
||||
<ThemedText preset="bodySm" style={styles.message}>
|
||||
{this.state.error?.message ?? 'An unexpected error occurred.'}
|
||||
</ThemedText>
|
||||
<View style={styles.buttonWrap}>
|
||||
<Button title="Retry" onPress={this.handleRetry} />
|
||||
</View>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
container: {
|
||||
flex: 1,
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
padding: 20,
|
||||
gap: 12,
|
||||
},
|
||||
message: {
|
||||
textAlign: 'center',
|
||||
},
|
||||
buttonWrap: {
|
||||
marginTop: 8,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import { useEffect, useMemo } from 'react';
|
||||
import { StyleSheet, View } from 'react-native';
|
||||
import Animated, { interpolateColor, useAnimatedProps, useSharedValue, withSpring } from 'react-native-reanimated';
|
||||
import Svg, { Circle, G, Text as SvgText } from 'react-native-svg';
|
||||
|
||||
type GaugeArcProps = {
|
||||
value: number;
|
||||
min?: number;
|
||||
max: number;
|
||||
label: string;
|
||||
unit: string;
|
||||
color: string;
|
||||
colorTo?: string;
|
||||
size?: number;
|
||||
};
|
||||
|
||||
const AnimatedCircle = Animated.createAnimatedComponent(Circle);
|
||||
|
||||
const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value));
|
||||
|
||||
export const GaugeArc = ({ value, min = 0, max, label, unit, color, colorTo, size = 140 }: GaugeArcProps) => {
|
||||
const radius = (size - 20) / 2;
|
||||
const circumference = 2 * Math.PI * radius;
|
||||
const arcLength = circumference * 0.75;
|
||||
const strokeWidth = 12;
|
||||
const progress = useSharedValue(0);
|
||||
|
||||
const normalized = useMemo(() => {
|
||||
const span = max - min;
|
||||
const safeSpan = span > 0 ? span : 1;
|
||||
return clamp((value - min) / safeSpan, 0, 1);
|
||||
}, [value, min, max]);
|
||||
|
||||
const displayValue = useMemo(() => {
|
||||
if (!Number.isFinite(value)) {
|
||||
return '--';
|
||||
}
|
||||
return `${Math.max(min, Math.min(max, value)).toFixed(1)} ${unit}`;
|
||||
}, [max, min, unit, value]);
|
||||
|
||||
useEffect(() => {
|
||||
progress.value = withSpring(normalized, {
|
||||
damping: 16,
|
||||
stiffness: 140,
|
||||
mass: 1,
|
||||
});
|
||||
}, [normalized, progress]);
|
||||
|
||||
const animatedStroke = useAnimatedProps(() => {
|
||||
const dashOffset = arcLength - arcLength * progress.value;
|
||||
const strokeColor = colorTo ? interpolateColor(progress.value, [0, 1], [color, colorTo]) : color;
|
||||
|
||||
return {
|
||||
strokeDashoffset: dashOffset,
|
||||
stroke: strokeColor,
|
||||
};
|
||||
});
|
||||
|
||||
return (
|
||||
<View style={styles.wrapper}>
|
||||
<Svg width={size} height={size} viewBox={`0 0 ${size} ${size}`}>
|
||||
<G transform={`rotate(-135 ${size / 2} ${size / 2})`}>
|
||||
<Circle
|
||||
cx={size / 2}
|
||||
cy={size / 2}
|
||||
r={radius}
|
||||
strokeWidth={strokeWidth}
|
||||
stroke="#1E293B"
|
||||
fill="none"
|
||||
strokeDasharray={`${arcLength} ${circumference}`}
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<AnimatedCircle
|
||||
cx={size / 2}
|
||||
cy={size / 2}
|
||||
r={radius}
|
||||
strokeWidth={strokeWidth}
|
||||
stroke={color}
|
||||
fill="none"
|
||||
strokeDasharray={`${arcLength} ${circumference}`}
|
||||
strokeLinecap="round"
|
||||
animatedProps={animatedStroke}
|
||||
/>
|
||||
</G>
|
||||
<SvgText
|
||||
x={size / 2}
|
||||
y={size / 2 - 8}
|
||||
fill="#E2E8F0"
|
||||
fontSize={Math.round(size * 0.16)}
|
||||
fontFamily="Courier New"
|
||||
fontWeight="700"
|
||||
textAnchor="middle"
|
||||
>
|
||||
{displayValue}
|
||||
</SvgText>
|
||||
<SvgText
|
||||
x={size / 2}
|
||||
y={size / 2 + 18}
|
||||
fill="#94A3B8"
|
||||
fontSize={Math.round(size * 0.085)}
|
||||
fontFamily="Courier New"
|
||||
textAnchor="middle"
|
||||
letterSpacing="0.6"
|
||||
>
|
||||
{label}
|
||||
</SvgText>
|
||||
</Svg>
|
||||
</View>
|
||||
);
|
||||
};
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
wrapper: {
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
},
|
||||
});
|
||||