mirror of
https://github.com/ruvnet/RuView
synced 2026-07-22 17:23:19 +00:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2dfb4884be | |||
| e7904786f0 | |||
| 9a078e4ac8 | |||
| 0e39faac73 | |||
| ad41a89960 | |||
| e3021c777c | |||
| b4c2f7d20b | |||
| aea9892aed | |||
| 347ad4bb11 | |||
| 5d7fccce79 | |||
| cbedbce9e3 | |||
| 7343bdc4dd | |||
| 21b2b3352f | |||
| e11d569a39 | |||
| 36e70bf229 | |||
| f06d0c6ab5 |
@@ -0,0 +1,58 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
# Keep all third-party GitHub Actions on verified, pinned commit SHAs.
|
||||||
|
# Pairs with the SHA pinning in security-scan.yml and ci.yml so that
|
||||||
|
# future bumps stay automated and reviewable rather than drifting back
|
||||||
|
# to mutable @master / @main refs. See issue #442.
|
||||||
|
- package-ecosystem: github-actions
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- github-actions
|
||||||
|
|
||||||
|
# Mobile app npm deps. Includes the @xmldom/xmldom, node-forge, and
|
||||||
|
# picomatch advisories from #442 plus axios and any future surface.
|
||||||
|
- package-ecosystem: npm
|
||||||
|
directory: /ui/mobile
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- mobile
|
||||||
|
|
||||||
|
# Desktop UI npm deps. Direct vite devDep currently has a HIGH advisory
|
||||||
|
# (dev-server-only path traversal); track future bumps automatically.
|
||||||
|
- package-ecosystem: npm
|
||||||
|
directory: /v2/crates/wifi-densepose-desktop/ui
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- desktop
|
||||||
|
|
||||||
|
# Python deps used by v1/ and the FastAPI service. requirements.txt is
|
||||||
|
# only loosely pinned; let Dependabot surface upstream CVE bumps.
|
||||||
|
- package-ecosystem: pip
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- python
|
||||||
|
|
||||||
|
# Rust workspace (15+ crates). cargo audit is not currently wired into
|
||||||
|
# any workflow, so Dependabot is the primary automated bump path.
|
||||||
|
- package-ecosystem: cargo
|
||||||
|
directory: /v2
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- rust
|
||||||
@@ -255,7 +255,7 @@ jobs:
|
|||||||
docker stop test-container
|
docker stop test-container
|
||||||
|
|
||||||
- name: Run container security scan
|
- name: Run container security scan
|
||||||
uses: aquasecurity/trivy-action@master
|
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||||
with:
|
with:
|
||||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
|
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||||
format: 'sarif'
|
format: 'sarif'
|
||||||
|
|||||||
@@ -98,6 +98,32 @@ jobs:
|
|||||||
echo "Flash image integrity verified"
|
echo "Flash image integrity verified"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
- name: Verify embedded version string matches version.txt (fixes #505)
|
||||||
|
working-directory: firmware/esp32-csi-node
|
||||||
|
run: |
|
||||||
|
EXPECTED=$(cat version.txt | tr -d '[:space:]')
|
||||||
|
BIN=build/esp32-csi-node.bin
|
||||||
|
# Extract version from ESP-IDF app_desc: magic 0xABCD5432 at offset 0
|
||||||
|
# followed by version string at offset 16, null-terminated, max 32 chars.
|
||||||
|
EMBEDDED=$(python3 -c "
|
||||||
|
import struct, sys
|
||||||
|
data = open('$BIN','rb').read()
|
||||||
|
magic = struct.pack('<I', 0xABCD5432)
|
||||||
|
i = data.find(magic)
|
||||||
|
if i < 0:
|
||||||
|
sys.exit('app_desc magic not found')
|
||||||
|
ver = data[i+16:i+48].split(b'\\x00',1)[0].decode('ascii','replace')
|
||||||
|
print(ver)
|
||||||
|
" 2>&1)
|
||||||
|
echo "Expected version: $EXPECTED"
|
||||||
|
echo "Embedded version: $EMBEDDED"
|
||||||
|
if [ "$EMBEDDED" != "$EXPECTED" ]; then
|
||||||
|
echo "::error::Version string mismatch! version.txt='$EXPECTED' but binary reports '$EMBEDDED'."
|
||||||
|
echo "::error::Ensure version.txt is updated before building and tagging."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Version string verified: $EMBEDDED"
|
||||||
|
|
||||||
- name: Stage release binaries with variant-specific names
|
- name: Stage release binaries with variant-specific names
|
||||||
working-directory: firmware/esp32-csi-node
|
working-directory: firmware/esp32-csi-node
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
name: Point Cloud Viewer → GitHub Pages
|
||||||
|
|
||||||
|
# Publishes the live 3D point cloud viewer to gh-pages/pointcloud/.
|
||||||
|
# The viewer defaults to a synthetic in-browser demo; users can append
|
||||||
|
# ?backend=<url> or ?backend=auto to point it at a real ruview-pointcloud
|
||||||
|
# server (CORS-permitting host required). See ADR-094.
|
||||||
|
#
|
||||||
|
# Uses keep_files: true to preserve the existing observatory/, pose-fusion/,
|
||||||
|
# nvsim/, and root index.html demos already on gh-pages.
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- 'v2/crates/wifi-densepose-pointcloud/src/viewer.html'
|
||||||
|
- '.github/workflows/pointcloud-pages.yml'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: pointcloud-pages
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout main
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Stage viewer for Pages
|
||||||
|
run: |
|
||||||
|
mkdir -p _site/pointcloud
|
||||||
|
cp v2/crates/wifi-densepose-pointcloud/src/viewer.html _site/pointcloud/index.html
|
||||||
|
# Drop a tiny README so direct browsers of the directory get context.
|
||||||
|
cat > _site/pointcloud/README.md <<'EOF'
|
||||||
|
# RuView — Live 3D Point Cloud Viewer
|
||||||
|
|
||||||
|
Hosted at: https://ruvnet.github.io/RuView/pointcloud/
|
||||||
|
|
||||||
|
## Modes
|
||||||
|
|
||||||
|
- Default — synthetic in-browser demo (no backend, no network calls).
|
||||||
|
- `?backend=auto` — fetch from `/api/splats` on the same origin
|
||||||
|
(only works when the viewer is served by `ruview-pointcloud serve`).
|
||||||
|
- `?backend=<url>` — fetch from `<url>/api/splats`. The intended
|
||||||
|
local-ESP32 use is `?backend=http://127.0.0.1:9880`: run
|
||||||
|
`ruview-pointcloud serve --bind 127.0.0.1:9880` on the same
|
||||||
|
machine with your ESP32 streaming CSI to UDP port 3333, then
|
||||||
|
visit the URL above. The local server's CorsLayer permits
|
||||||
|
requests from `https://ruvnet.github.io`, and modern browsers
|
||||||
|
permit HTTPS→127.0.0.1 mixed-content as a trustworthy origin.
|
||||||
|
The "📡 Connect ESP32" button in the viewer prompts for this
|
||||||
|
URL and persists it in localStorage.
|
||||||
|
- `?live=1` — require a live backend; show an offline message instead
|
||||||
|
of falling back to the synthetic demo.
|
||||||
|
|
||||||
|
See ADR-094 for the deployment design.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Deploy to gh-pages/pointcloud/
|
||||||
|
uses: peaceiris/actions-gh-pages@v4
|
||||||
|
with:
|
||||||
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
publish_dir: ./_site/pointcloud
|
||||||
|
destination_dir: pointcloud
|
||||||
|
# CRITICAL: preserves observatory/, pose-fusion/, nvsim/, and root
|
||||||
|
# index.html already on gh-pages.
|
||||||
|
keep_files: true
|
||||||
|
commit_message: 'deploy(pointcloud): ${{ github.sha }}'
|
||||||
|
user_name: 'github-actions[bot]'
|
||||||
|
user_email: 'github-actions[bot]@users.noreply.github.com'
|
||||||
@@ -111,7 +111,7 @@ jobs:
|
|||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Run Snyk vulnerability scan
|
- name: Run Snyk vulnerability scan
|
||||||
uses: snyk/actions/python@master
|
uses: snyk/actions/python@9adf32b1121593767fc3c057af55b55db032dc04 # v1.0.0
|
||||||
env:
|
env:
|
||||||
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
||||||
with:
|
with:
|
||||||
@@ -163,7 +163,7 @@ jobs:
|
|||||||
cache-to: type=gha,mode=max
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
- name: Run Trivy vulnerability scanner
|
- name: Run Trivy vulnerability scanner
|
||||||
uses: aquasecurity/trivy-action@master
|
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||||
with:
|
with:
|
||||||
image-ref: 'wifi-densepose:scan'
|
image-ref: 'wifi-densepose:scan'
|
||||||
format: 'sarif'
|
format: 'sarif'
|
||||||
@@ -221,7 +221,7 @@ jobs:
|
|||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Run Checkov IaC scan
|
- name: Run Checkov IaC scan
|
||||||
uses: bridgecrewio/checkov-action@master
|
uses: bridgecrewio/checkov-action@99bb2caf247dfd9f03cf984373bc6043d4e32ebf # v12.1347.0
|
||||||
with:
|
with:
|
||||||
directory: .
|
directory: .
|
||||||
framework: kubernetes,dockerfile,terraform,ansible
|
framework: kubernetes,dockerfile,terraform,ansible
|
||||||
@@ -238,7 +238,7 @@ jobs:
|
|||||||
category: checkov
|
category: checkov
|
||||||
|
|
||||||
- name: Run Terrascan IaC scan
|
- name: Run Terrascan IaC scan
|
||||||
uses: tenable/terrascan-action@main
|
uses: tenable/terrascan-action@3a6e87da8e244513bd77b631e624552643f794c6 # v1.4.1
|
||||||
with:
|
with:
|
||||||
iac_type: 'k8s'
|
iac_type: 'k8s'
|
||||||
iac_version: 'v1'
|
iac_version: 'v1'
|
||||||
@@ -247,7 +247,7 @@ jobs:
|
|||||||
sarif_upload: true
|
sarif_upload: true
|
||||||
|
|
||||||
- name: Run KICS IaC scan
|
- name: Run KICS IaC scan
|
||||||
uses: checkmarx/kics-github-action@master
|
uses: checkmarx/kics-github-action@05aa5eb70eede1355220f4ca5238d96b397e30a6 # v2.1.20
|
||||||
with:
|
with:
|
||||||
path: '.'
|
path: '.'
|
||||||
output_path: kics-results
|
output_path: kics-results
|
||||||
@@ -277,7 +277,7 @@ jobs:
|
|||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Run TruffleHog secret scan
|
- name: Run TruffleHog secret scan
|
||||||
uses: trufflesecurity/trufflehog@main
|
uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2
|
||||||
with:
|
with:
|
||||||
path: ./
|
path: ./
|
||||||
base: main
|
base: main
|
||||||
|
|||||||
+5
-1
@@ -167,7 +167,11 @@ firing cleanly, HEALTH mesh packets sent.
|
|||||||
Kconfig surface added under "Adaptive Controller (ADR-081)".
|
Kconfig surface added under "Adaptive Controller (ADR-081)".
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- **`provision.py` esptool v5 compat** (#391) — Stale `write_flash` (underscore) syntax in the dry-run manual-flash hint now uses `write-flash` (hyphenated) for esptool >= 5.x. The primary flash command was already correct.
|
- **Firmware: SPI flash cache crash under high CSI callback pressure** (RuView#396, #397) — ESP32-S3 nodes crashed in `cache_ll_l1_resume_icache` / `wDev_ProcessFiq` after ~2400 callbacks when the promiscuous filter admitted DATA frames at 100–500 Hz. Fixed by narrowing the filter mask to `WIFI_PROMIS_FILTER_MASK_MGMT` (~10 Hz beacons), adding a 50 Hz early callback rate gate (`CSI_MIN_PROCESS_INTERVAL_US`) that drops excess callbacks before any processing work, and enabling `CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y` as defense-in-depth. Stability validated with a 4-min-per-node soak.
|
||||||
|
- **Firmware: `filter_mac` / `node_id` clobber by WiFi driver init** (#232, #375, #385, #386, #390, #397) — `g_nvs_config` can be corrupted during `wifi_init_sta()` on some devices (confirmed on `80:b5:4e:c1:be:b8`), reverting `node_id` to the Kconfig default and producing garbage MAC-filter reads in the CSI callback (100–500 Hz). New `csi_collector_set_node_id()` API called from `app_main()` **before** `wifi_init_sta()` captures both fields into module-local statics (`s_node_id`, `s_filter_mac`, `s_filter_mac_set`). `csi_collector_init()` now runs a canary that distinguishes "early≠g_nvs_config" (corruption confirmed) from a no-op match. All CSI runtime paths use the defensive copies exclusively.
|
||||||
|
- **Firmware: `edge_processing` sample rate mismatch** (#397) — `estimate_bpm_zero_crossing()` was called with a hard-coded `sample_rate = 20.0f`, but MGMT-only promiscuous delivers ~10 Hz. Breathing and heart-rate reports were 2× too high. Corrected to `10.0f` with an explicit comment tying it to the callback rate.
|
||||||
|
- **`provision.py` esptool command form** (#391, #397) — ESP-IDF v5.4 bundles `esptool 4.10.0`, which only accepts `write_flash` (underscore). Standalone `pip install esptool` v5.x accepts both forms but prefers `write-flash`. #391 switched to `write-flash` which broke the documented ESP-IDF Python venv flow; #397 reverts to `write_flash` (works with both esptool 4.x and 5.x) with an inline comment warning future maintainers not to "re-fix" it.
|
||||||
|
- **`provision.py` esptool v5 dry-run hint** (#391) — Stale `write_flash` (underscore) syntax in the dry-run manual-flash hint now uses `write-flash` (hyphenated) for esptool >= 5.x. The primary flash command was already correct.
|
||||||
- **`provision.py` silent NVS wipe** (#391) — The script replaces the entire `csi_cfg` NVS namespace on every run, so partial invocations were silently erasing WiFi credentials and causing `Retrying WiFi connection (10/10)` in the field. Now refuses to run without `--ssid`, `--password`, and `--target-ip` unless `--force-partial` is passed. `--force-partial` prints a warning listing which keys will be wiped.
|
- **`provision.py` silent NVS wipe** (#391) — The script replaces the entire `csi_cfg` NVS namespace on every run, so partial invocations were silently erasing WiFi credentials and causing `Retrying WiFi connection (10/10)` in the field. Now refuses to run without `--ssid`, `--password`, and `--target-ip` unless `--force-partial` is passed. `--force-partial` prints a warning listing which keys will be wiped.
|
||||||
- **Firmware: defensive `node_id` capture** (#232, #375, #385, #386, #390) — Users on multi-node deployments reported `node_id` reverting to the Kconfig default (`1`) in UDP frames and in the `csi_collector` init log, despite NVS loading the correct value. The root cause (memory corruption of `g_nvs_config`) has not been definitively isolated, but the UDP frame header is now tamper-proof: `csi_collector_init()` captures `g_nvs_config.node_id` into a module-local `s_node_id` once, and `csi_serialize_frame()` plus all other consumers (`edge_processing.c`, `wasm_runtime.c`, `display_ui.c`, `swarm_bridge_init`) read it via the new `csi_collector_get_node_id()` accessor. A canary logs `WARN` if `g_nvs_config.node_id` diverges from `s_node_id` at end-of-init, helping isolate the upstream corruption path. Validated on attached ESP32-S3 (COM8): NVS `node_id=2` propagates through boot log, capture log, init log, and byte[4] of every UDP frame.
|
- **Firmware: defensive `node_id` capture** (#232, #375, #385, #386, #390) — Users on multi-node deployments reported `node_id` reverting to the Kconfig default (`1`) in UDP frames and in the `csi_collector` init log, despite NVS loading the correct value. The root cause (memory corruption of `g_nvs_config`) has not been definitively isolated, but the UDP frame header is now tamper-proof: `csi_collector_init()` captures `g_nvs_config.node_id` into a module-local `s_node_id` once, and `csi_serialize_frame()` plus all other consumers (`edge_processing.c`, `wasm_runtime.c`, `display_ui.c`, `swarm_bridge_init`) read it via the new `csi_collector_get_node_id()` accessor. A canary logs `WARN` if `g_nvs_config.node_id` diverges from `s_node_id` at end-of-init, helping isolate the upstream corruption path. Validated on attached ESP32-S3 (COM8): NVS `node_id=2` propagates through boot log, capture log, init log, and byte[4] of every UDP frame.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,203 @@
|
|||||||
|
# ADR-094: Live 3D Point Cloud Viewer — GitHub Pages Deployment with Optional Real-Data Stream
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|---|---|
|
||||||
|
| **Status** | Proposed (2026-04-29) |
|
||||||
|
| **Date** | 2026-04-29 |
|
||||||
|
| **Authors** | ruv |
|
||||||
|
| **Related** | ADR-092 (nvsim dashboard Pages deployment), ADR-059 (live ESP32 CSI pipeline), ADR-079 (camera ground-truth training) |
|
||||||
|
| **Branch** | `feat/pointcloud-pages-demo` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Context
|
||||||
|
|
||||||
|
The `wifi-densepose-pointcloud` crate ships a Three.js-based viewer
|
||||||
|
(`v2/crates/wifi-densepose-pointcloud/src/viewer.html`) that renders the
|
||||||
|
fused camera-depth + WiFi CSI + mmWave point cloud produced by the
|
||||||
|
`ruview-pointcloud serve` binary. Today the viewer is local-only:
|
||||||
|
|
||||||
|
- It is served by the Axum binary on `127.0.0.1:9880`.
|
||||||
|
- It polls `/api/splats` every 500 ms expecting a backend on the same
|
||||||
|
origin.
|
||||||
|
- There is no GitHub Pages deployment, so the README's
|
||||||
|
"▶ Live 3D Point Cloud" link points at the moved-content section in
|
||||||
|
`docs/readme-details.md`, not at a hosted demo. The two sibling demos
|
||||||
|
(Live Observatory, Dual-Modal Pose Fusion) are already hosted at
|
||||||
|
`https://ruvnet.github.io/RuView/` and `…/pose-fusion.html`.
|
||||||
|
|
||||||
|
This is an asymmetry: a first-time visitor can preview the WiFi pose
|
||||||
|
demo and the Observatory in one click, but cannot preview the point
|
||||||
|
cloud without cloning the repo, building Rust, plugging in an ESP32,
|
||||||
|
and pointing a webcam at themselves. That gap suppresses the most
|
||||||
|
visually compelling demonstration of the v0.7+ sensor-fusion work.
|
||||||
|
|
||||||
|
A naive fix — drop the static HTML at `gh-pages/pointcloud/` — does
|
||||||
|
not work because the viewer's `fetch("/api/splats")` will 404 on Pages
|
||||||
|
and the canvas will hang at "Loading…". A second naive fix — bake in a
|
||||||
|
fixed sample dataset — solves the loading state but loses the live-data
|
||||||
|
story entirely, and forks the viewer into a "demo build" and a "real
|
||||||
|
build" that drift apart.
|
||||||
|
|
||||||
|
## 2. Decision
|
||||||
|
|
||||||
|
Ship **one** viewer that auto-selects its transport from URL parameters,
|
||||||
|
and publish it to `gh-pages/pointcloud/` alongside the other demos:
|
||||||
|
|
||||||
|
1. **Default mode** — when the viewer is opened with no query parameters
|
||||||
|
on `https://ruvnet.github.io/RuView/pointcloud/`, present a "▶ Enable
|
||||||
|
camera" CTA. On click the viewer requests webcam access, runs
|
||||||
|
**MediaPipe Face Mesh** in-browser (~30 fps, 478 refined landmarks),
|
||||||
|
and renders the visitor's own face as a point cloud — the closest
|
||||||
|
browser equivalent of the local pipeline's depth-backprojected face
|
||||||
|
geometry that motivated this ADR (`I could see the outline of my face
|
||||||
|
in points`). The viewer mirrors x to match selfie convention and
|
||||||
|
maps Face Mesh's relative-z to the same world-coordinate range the
|
||||||
|
live `/api/splats` payload uses, so a single render path drives both.
|
||||||
|
Badge reads `● DEMO Your Face (MediaPipe)`. If the user denies
|
||||||
|
camera permission, dismisses the prompt, or visits on a device
|
||||||
|
without a webcam, the viewer falls back automatically to a
|
||||||
|
procedural scaffold (floor grid, walls, breathing figure, 17-keypoint
|
||||||
|
skeleton). All processing is client-side; no frames leave the
|
||||||
|
browser. ~480-500 splats from the face plus ~110 floor/wall context
|
||||||
|
splats.
|
||||||
|
2. **Auto mode** (`?backend=auto`) — fetch from `/api/splats` on the same
|
||||||
|
origin. This is the local-development case (`ruview-pointcloud serve`
|
||||||
|
serves the viewer and the API together). On any failure (404, network
|
||||||
|
error, CORS), fall back silently to synthetic-demo rendering so the
|
||||||
|
tab never dies.
|
||||||
|
3. **Remote mode** (`?backend=<url>`) — fetch from `<url>/api/splats`.
|
||||||
|
This is the **integrated-ESP32** path: the user runs
|
||||||
|
`ruview-pointcloud serve --bind 127.0.0.1:9880` locally with an
|
||||||
|
ESP32-S3 streaming CSI to UDP port 3333, then opens
|
||||||
|
`https://ruvnet.github.io/RuView/pointcloud/?backend=http://127.0.0.1:9880`.
|
||||||
|
The hosted Pages viewer becomes a thin client for the local Rust
|
||||||
|
fusion pipeline (camera depth + WiFi CSI + mmWave) without a clone
|
||||||
|
or rebuild. The viewer also exposes a "📡 Connect ESP32" button that
|
||||||
|
prompts for the URL, persists it in `localStorage`, and reloads
|
||||||
|
with the query param.
|
||||||
|
|
||||||
|
For this to work the local server must answer the browser's CORS
|
||||||
|
preflight. `stream.rs` therefore installs a `tower_http` `CorsLayer`
|
||||||
|
that allows three origin classes:
|
||||||
|
|
||||||
|
- `https://ruvnet.github.io` — the published Pages demo.
|
||||||
|
- `http://localhost:*` and `http://127.0.0.1:*` — developer running
|
||||||
|
the bundled `viewer.html` directly.
|
||||||
|
- `null` — `file://` origins.
|
||||||
|
|
||||||
|
Mixed-content (HTTPS Pages → HTTP loopback) is permitted because
|
||||||
|
modern browsers (Chrome 94+, Firefox 116+, Safari 16.4+) classify
|
||||||
|
`127.0.0.1` and `localhost` as "potentially trustworthy" origins.
|
||||||
|
Any other origin (a public hostname, etc.) is denied — this is not
|
||||||
|
a wildcard CORS posture. Badge reads `● REMOTE <url>`. Same silent
|
||||||
|
demo fallback on failure.
|
||||||
|
4. **Strict-live mode** (`?live=1`) — disable the demo fallback. If the
|
||||||
|
chosen transport fails, replace the info panel with an explicit offline
|
||||||
|
message (`● OFFLINE — Live backend required but unreachable`). Useful
|
||||||
|
for embedding the viewer in a status page or kiosk.
|
||||||
|
|
||||||
|
The synthetic frame returned by the in-browser generator matches the
|
||||||
|
JSON shape of the live `/api/splats` payload exactly (`splats`, `count`,
|
||||||
|
`frame`, `live`, `pipeline.{skeleton,vitals,…}`), so a single render path
|
||||||
|
drives both modes. There is no demo build vs real build — only one HTML
|
||||||
|
file, one render path, and one set of bugs.
|
||||||
|
|
||||||
|
A new GitHub Actions workflow (`.github/workflows/pointcloud-pages.yml`)
|
||||||
|
copies the viewer to `gh-pages/pointcloud/index.html` on every push to
|
||||||
|
`main` that touches the viewer, using `peaceiris/actions-gh-pages@v4`
|
||||||
|
with `keep_files: true` to preserve the existing observatory, pose-fusion,
|
||||||
|
and nvsim deployments.
|
||||||
|
|
||||||
|
## 3. Consequences
|
||||||
|
|
||||||
|
### Positive
|
||||||
|
|
||||||
|
- **First-click demo.** Visitors clicking the README's
|
||||||
|
"▶ Live 3D Point Cloud" link land on a working Three.js scene in <1 s,
|
||||||
|
no toolchain required. Matches the parity of the other two demos.
|
||||||
|
- **Real-data on demand.** Users with their own `ruview-pointcloud serve`
|
||||||
|
host can use the same hosted viewer URL with
|
||||||
|
`?backend=https://their-host.example.com` — no clone, no rebuild. The
|
||||||
|
hosted demo doubles as a thin client for self-hosted backends.
|
||||||
|
- **Single render path.** Synthetic frames flow through the same
|
||||||
|
`handleData → updateSplats → drawSkeleton` pipeline as live frames, so
|
||||||
|
visual regressions surface in the demo and the live build at the same
|
||||||
|
time. This is the same dual-transport pattern ADR-092 chose for nvsim.
|
||||||
|
- **No backend deploy required.** Pages serves static HTML; the demo
|
||||||
|
works without standing up an Axum host on the public internet, and
|
||||||
|
there is no per-visitor CSI/camera plumbing to provision.
|
||||||
|
- **Preserves existing deployments.** `keep_files: true` plus the
|
||||||
|
`pointcloud/` destination means observatory/, pose-fusion/, nvsim/,
|
||||||
|
and the root index.html on gh-pages are untouched.
|
||||||
|
|
||||||
|
### Negative / tradeoffs
|
||||||
|
|
||||||
|
- **Face mesh ≠ CSI.** Browser webcam + MediaPipe gives real face
|
||||||
|
geometry but does not produce CSI-derived pose. Visitors who want to
|
||||||
|
see the *WiFi-driven* path still need `?backend=<their-host>`. The
|
||||||
|
procedural fallback is not WiFi-driven either; it is purely visual
|
||||||
|
scaffolding. We accept this — the goal of the hosted demo is to
|
||||||
|
convey the *shape* of what the local pipeline produces (a point
|
||||||
|
cloud of the user) rather than reproduce the WiFi physics in the
|
||||||
|
browser. The latter is a future ADR (WASM port of the fusion crate).
|
||||||
|
- **CORS burden on remote mode.** Users who want to share their backend
|
||||||
|
must add `Access-Control-Allow-Origin: https://ruvnet.github.io` (or
|
||||||
|
`*`) to their `ruview-pointcloud serve` config. We document this in the
|
||||||
|
workflow's generated README; we do **not** add a public proxy.
|
||||||
|
- **Synthetic generator lives in the viewer.** ~80 LOC of procedural JS
|
||||||
|
is now part of `viewer.html`. Acceptable: the file is already the
|
||||||
|
client-side render bundle, and the generator is bounded and inert
|
||||||
|
(deterministic, no I/O, no eval).
|
||||||
|
- **No replay-from-recording in this ADR.** A future ADR may add a
|
||||||
|
`?recording=<url>.jsonl` mode that replays captured frames at native
|
||||||
|
rate; that is out of scope here.
|
||||||
|
|
||||||
|
### Neutral
|
||||||
|
|
||||||
|
- The local-dev experience is unchanged. `ruview-pointcloud serve` still
|
||||||
|
serves `viewer.html` from the bundled asset and the viewer still hits
|
||||||
|
`/api/splats` because `?backend` defaults to `auto`. Nothing in the
|
||||||
|
Rust crate changes — this is HTML + workflow only.
|
||||||
|
|
||||||
|
## 4. Implementation
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `v2/crates/wifi-densepose-pointcloud/src/viewer.html` | Add URL-param transport selector (`backend`, `live`), synthetic frame generator, demo-fallback path, transport-aware mode badge. ~120 LOC added, no removed behavior. |
|
||||||
|
| `.github/workflows/pointcloud-pages.yml` | New workflow: stage viewer to `_site/pointcloud/index.html`, deploy to `gh-pages/pointcloud/` with `keep_files: true`. Triggers on viewer changes and on manual dispatch. |
|
||||||
|
| `README.md` | Already updated — `▶ Live 3D Point Cloud` link will be retargeted to `https://ruvnet.github.io/RuView/pointcloud/` once the first deploy succeeds. (Tracked separately, not blocking this ADR.) |
|
||||||
|
| `docs/adr/README.md` | ADR index — add ADR-094 row. |
|
||||||
|
|
||||||
|
## 5. Acceptance Gates
|
||||||
|
|
||||||
|
This ADR is **Implemented** when all of the following hold:
|
||||||
|
|
||||||
|
1. Pushing to `main` with a viewer change triggers
|
||||||
|
`pointcloud-pages.yml`, which deploys to `gh-pages/pointcloud/` in
|
||||||
|
under 60 seconds.
|
||||||
|
2. `https://ruvnet.github.io/RuView/pointcloud/` loads, shows the
|
||||||
|
"Enable camera" CTA, and on accept renders the visitor's face as a
|
||||||
|
point cloud with badge `● DEMO Your Face (MediaPipe)` and non-zero
|
||||||
|
splat + frame counts. On camera denial, falls back to the
|
||||||
|
procedural scene with badge `● DEMO Synthetic`.
|
||||||
|
3. Existing demos at `https://ruvnet.github.io/RuView/` and
|
||||||
|
`…/pose-fusion.html` and `…/nvsim/` are still reachable after the
|
||||||
|
first deploy (smoke-tested manually).
|
||||||
|
4. `https://ruvnet.github.io/RuView/pointcloud/?live=1` shows the
|
||||||
|
`● OFFLINE` panel (because no same-origin backend exists on Pages).
|
||||||
|
5. `https://ruvnet.github.io/RuView/pointcloud/?backend=https://example.invalid`
|
||||||
|
falls back to demo within one poll interval (~500 ms) without
|
||||||
|
throwing in the console.
|
||||||
|
6. Running `./target/release/ruview-pointcloud serve` locally and
|
||||||
|
opening `http://127.0.0.1:9880/` (which serves the same HTML) still
|
||||||
|
shows live-mode rendering with the `● LIVE Local Backend` badge.
|
||||||
|
|
||||||
|
## 6. Out of Scope
|
||||||
|
|
||||||
|
- Replaying recorded JSONL frames in the browser (future ADR).
|
||||||
|
- WASM-side execution of the fusion pipeline in the browser (would
|
||||||
|
require porting the camera + mmWave path; deferred).
|
||||||
|
- Authentication / signed splats payloads — backend-side concern,
|
||||||
|
unaffected by this client-side change.
|
||||||
|
- Hosting a public CORS proxy for users without their own backend.
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -25,13 +25,20 @@
|
|||||||
/* ADR-060: Access the global NVS config for MAC filter and channel override. */
|
/* ADR-060: Access the global NVS config for MAC filter and channel override. */
|
||||||
extern nvs_config_t g_nvs_config;
|
extern nvs_config_t g_nvs_config;
|
||||||
|
|
||||||
/* Defensive fix (#232, #375, #385, #386, #390): capture node_id at init-time
|
/* Defensive fix (#232, #375, #385, #386, #390): capture NVS config fields into
|
||||||
* into a module-local static. Using the global g_nvs_config.node_id directly
|
* module-local statics BEFORE wifi_init_sta() runs, because WiFi driver init
|
||||||
* at every callback is vulnerable to any memory corruption that clobbers the
|
* can corrupt g_nvs_config (confirmed on device 80:b5:4e:c1:be:b8).
|
||||||
* struct (which users have reported reverting node_id to the Kconfig default
|
* main.c calls csi_collector_set_node_id() immediately after nvs_config_load(),
|
||||||
* of 1). The local copy is set once at csi_collector_init() and then used
|
* and all runtime paths use the local copies exclusively. */
|
||||||
* exclusively by csi_serialize_frame(). */
|
|
||||||
static uint8_t s_node_id = 1;
|
static uint8_t s_node_id = 1;
|
||||||
|
static bool s_node_id_early_set = false;
|
||||||
|
|
||||||
|
/* Defensive copy of MAC filter config — the CSI callback fires at 100-500 Hz
|
||||||
|
* and reads filter_mac_set + filter_mac on every invocation. If wifi_init_sta()
|
||||||
|
* corrupts g_nvs_config, the callback would read garbage, potentially causing
|
||||||
|
* LoadProhibited panics (observed: Core 0 panic after ~2400 callbacks). */
|
||||||
|
static uint8_t s_filter_mac[6] = {0};
|
||||||
|
static bool s_filter_mac_set = false;
|
||||||
|
|
||||||
/* ADR-057: Build-time guard — fail early if CSI is not enabled in sdkconfig.
|
/* ADR-057: Build-time guard — fail early if CSI is not enabled in sdkconfig.
|
||||||
* Without this, the firmware compiles but crashes at runtime with:
|
* Without this, the firmware compiles but crashes at runtime with:
|
||||||
@@ -60,6 +67,24 @@ static uint32_t s_rate_skip = 0;
|
|||||||
#define CSI_MIN_SEND_INTERVAL_US (20 * 1000)
|
#define CSI_MIN_SEND_INTERVAL_US (20 * 1000)
|
||||||
static int64_t s_last_send_us = 0;
|
static int64_t s_last_send_us = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimum interval between processing ANY CSI callback in microseconds.
|
||||||
|
* Promiscuous MGMT+DATA can fire 100-500+ times/sec. At rates above ~50 Hz,
|
||||||
|
* the WiFi FIQ handler (wDev_ProcessFiq) races with SPI flash cache operations,
|
||||||
|
* causing Core 0 LoadProhibited panics in cache_ll_l1_resume_icache.
|
||||||
|
*
|
||||||
|
* This early gate drops excess callbacks BEFORE any processing (serialization,
|
||||||
|
* UDP, edge enqueue), keeping the effective callback rate at ~50 Hz while
|
||||||
|
* preserving the full MGMT+DATA promiscuous filter and HT-LTF/STBC CSI quality.
|
||||||
|
*
|
||||||
|
* The WiFi hardware still captures all frames and the CSI data is generated,
|
||||||
|
* but we simply discard the excess in software. This reduces the time spent
|
||||||
|
* in callback context per second, giving the WiFi ISR more headroom.
|
||||||
|
*/
|
||||||
|
#define CSI_MIN_PROCESS_INTERVAL_US (20 * 1000) /* 50 Hz */
|
||||||
|
static int64_t s_last_process_us = 0;
|
||||||
|
static uint32_t s_early_drop = 0;
|
||||||
|
|
||||||
/* ---- ADR-029: Channel-hop state ---- */
|
/* ---- ADR-029: Channel-hop state ---- */
|
||||||
|
|
||||||
/** Channel hop table (populated from NVS at boot or via set_hop_table). */
|
/** Channel hop table (populated from NVS at boot or via set_hop_table). */
|
||||||
@@ -165,9 +190,20 @@ static void wifi_csi_callback(void *ctx, wifi_csi_info_t *info)
|
|||||||
{
|
{
|
||||||
(void)ctx;
|
(void)ctx;
|
||||||
|
|
||||||
/* ADR-060: MAC address filtering — drop frames from non-matching sources. */
|
/* Early rate gate: drop excess callbacks to ~50 Hz to prevent
|
||||||
if (g_nvs_config.filter_mac_set) {
|
* SPI flash cache crash in WiFi ISR (wDev_ProcessFiq). */
|
||||||
if (memcmp(info->mac, g_nvs_config.filter_mac, 6) != 0) {
|
int64_t now_us = esp_timer_get_time();
|
||||||
|
if ((now_us - s_last_process_us) < CSI_MIN_PROCESS_INTERVAL_US) {
|
||||||
|
s_early_drop++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
s_last_process_us = now_us;
|
||||||
|
|
||||||
|
/* ADR-060: MAC address filtering — drop frames from non-matching sources.
|
||||||
|
* Uses defensively-copied s_filter_mac instead of g_nvs_config (which can
|
||||||
|
* be corrupted by wifi_init_sta — same root cause as the node_id clobber). */
|
||||||
|
if (s_filter_mac_set) {
|
||||||
|
if (memcmp(info->mac, s_filter_mac, 6) != 0) {
|
||||||
return; /* Source MAC doesn't match filter — skip frame. */
|
return; /* Source MAC doesn't match filter — skip frame. */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -222,14 +258,60 @@ static void wifi_promiscuous_cb(void *buf, wifi_promiscuous_pkt_type_t type)
|
|||||||
(void)type;
|
(void)type;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void csi_collector_set_node_id(uint8_t node_id)
|
||||||
|
{
|
||||||
|
s_node_id = node_id;
|
||||||
|
s_node_id_early_set = true;
|
||||||
|
ESP_LOGI(TAG, "Early capture node_id=%u (before WiFi init, #232/#390)",
|
||||||
|
(unsigned)node_id);
|
||||||
|
|
||||||
|
/* Also capture MAC filter config now — same struct, same corruption risk.
|
||||||
|
* The CSI callback reads filter_mac_set on every invocation (100-500 Hz),
|
||||||
|
* so a corrupted value could cause erratic filtering or crash. */
|
||||||
|
s_filter_mac_set = (g_nvs_config.filter_mac_set != 0);
|
||||||
|
if (s_filter_mac_set) {
|
||||||
|
memcpy(s_filter_mac, g_nvs_config.filter_mac, 6);
|
||||||
|
ESP_LOGI(TAG, "Early capture filter_mac=%02x:%02x:%02x:%02x:%02x:%02x",
|
||||||
|
s_filter_mac[0], s_filter_mac[1], s_filter_mac[2],
|
||||||
|
s_filter_mac[3], s_filter_mac[4], s_filter_mac[5]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void csi_collector_init(void)
|
void csi_collector_init(void)
|
||||||
{
|
{
|
||||||
/* Capture node_id into module-local static at init time. After this point
|
if (!s_node_id_early_set) {
|
||||||
* csi_serialize_frame() uses s_node_id exclusively, isolating the UDP
|
/* Fallback: no early capture — use current g_nvs_config (may be clobbered). */
|
||||||
* frame node_id field from any memory corruption of g_nvs_config. */
|
s_node_id = g_nvs_config.node_id;
|
||||||
s_node_id = g_nvs_config.node_id;
|
ESP_LOGW(TAG, "Late capture node_id=%u (no early set_node_id call)",
|
||||||
ESP_LOGI(TAG, "Captured node_id=%u at init (defensive copy for #232/#375/#385/#390)",
|
(unsigned)s_node_id);
|
||||||
(unsigned)s_node_id);
|
} else if (g_nvs_config.node_id != s_node_id) {
|
||||||
|
/* Canary: early capture disagrees with current g_nvs_config — corruption
|
||||||
|
* happened between nvs_config_load() and here (likely wifi_init_sta). */
|
||||||
|
ESP_LOGW(TAG, "node_id clobber CONFIRMED: early=%u g_nvs_config=%u "
|
||||||
|
"(WiFi init likely corrupted struct, using early value)",
|
||||||
|
(unsigned)s_node_id, (unsigned)g_nvs_config.node_id);
|
||||||
|
} else {
|
||||||
|
ESP_LOGI(TAG, "node_id=%u verified (early capture matches g_nvs_config)",
|
||||||
|
(unsigned)s_node_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Canary for filter_mac: check if WiFi init corrupted the filter fields. */
|
||||||
|
if (s_node_id_early_set) {
|
||||||
|
bool mac_set_now = (g_nvs_config.filter_mac_set != 0);
|
||||||
|
if (mac_set_now != s_filter_mac_set) {
|
||||||
|
ESP_LOGW(TAG, "filter_mac_set clobber CONFIRMED: early=%d g_nvs_config=%d",
|
||||||
|
(int)s_filter_mac_set, (int)mac_set_now);
|
||||||
|
} else if (s_filter_mac_set &&
|
||||||
|
memcmp(s_filter_mac, g_nvs_config.filter_mac, 6) != 0) {
|
||||||
|
ESP_LOGW(TAG, "filter_mac clobber CONFIRMED: bytes differ after WiFi init");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/* No early capture — grab filter config now (may already be corrupted). */
|
||||||
|
s_filter_mac_set = (g_nvs_config.filter_mac_set != 0);
|
||||||
|
if (s_filter_mac_set) {
|
||||||
|
memcpy(s_filter_mac, g_nvs_config.filter_mac, 6);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* ADR-060: Determine the CSI channel.
|
/* ADR-060: Determine the CSI channel.
|
||||||
* Priority: 1) NVS override (--channel), 2) connected AP channel, 3) Kconfig default. */
|
* Priority: 1) NVS override (--channel), 2) connected AP channel, 3) Kconfig default. */
|
||||||
@@ -260,12 +342,19 @@ void csi_collector_init(void)
|
|||||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous(true));
|
ESP_ERROR_CHECK(esp_wifi_set_promiscuous(true));
|
||||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_rx_cb(wifi_promiscuous_cb));
|
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_rx_cb(wifi_promiscuous_cb));
|
||||||
|
|
||||||
|
/* MGMT-only promiscuous filter + active probe injection (RuView#396).
|
||||||
|
*
|
||||||
|
* DATA frames cause 100-500+ WiFi HW interrupts/sec which crashes Core 0
|
||||||
|
* in wDev_ProcessFiq (SPI flash cache race in ESP-IDF WiFi blob).
|
||||||
|
* MGMT-only gives ~10 Hz (beacons). Probe request injection at 10 Hz
|
||||||
|
* adds ~10 Hz probe responses from APs → ~20 Hz total, matching the
|
||||||
|
* edge processing designed sample rate of 20 Hz. */
|
||||||
wifi_promiscuous_filter_t filt = {
|
wifi_promiscuous_filter_t filt = {
|
||||||
.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT | WIFI_PROMIS_FILTER_MASK_DATA,
|
.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT,
|
||||||
};
|
};
|
||||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_filter(&filt));
|
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_filter(&filt));
|
||||||
|
|
||||||
ESP_LOGI(TAG, "Promiscuous mode enabled for CSI capture");
|
ESP_LOGI(TAG, "Promiscuous mode enabled (MGMT-only, RuView#396)");
|
||||||
|
|
||||||
wifi_csi_config_t csi_config = {
|
wifi_csi_config_t csi_config = {
|
||||||
.lltf_en = true,
|
.lltf_en = true,
|
||||||
@@ -290,16 +379,6 @@ void csi_collector_init(void)
|
|||||||
|
|
||||||
ESP_LOGI(TAG, "CSI collection initialized (node_id=%u, channel=%u)",
|
ESP_LOGI(TAG, "CSI collection initialized (node_id=%u, channel=%u)",
|
||||||
(unsigned)s_node_id, (unsigned)csi_channel);
|
(unsigned)s_node_id, (unsigned)csi_channel);
|
||||||
|
|
||||||
/* Clobber-detection canary: if g_nvs_config.node_id no longer matches the
|
|
||||||
* value we captured, something corrupted the struct between nvs_config_load
|
|
||||||
* and here. This is the historic #232/#375 symptom. */
|
|
||||||
if (g_nvs_config.node_id != s_node_id) {
|
|
||||||
ESP_LOGW(TAG, "node_id clobber detected: captured=%u but g_nvs_config=%u "
|
|
||||||
"(frames will use captured value %u). Please report to #390.",
|
|
||||||
(unsigned)s_node_id, (unsigned)g_nvs_config.node_id,
|
|
||||||
(unsigned)s_node_id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Accessor for other modules that need the authoritative runtime node_id. */
|
/* Accessor for other modules that need the authoritative runtime node_id. */
|
||||||
|
|||||||
@@ -30,14 +30,24 @@
|
|||||||
void csi_collector_init(void);
|
void csi_collector_init(void);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the runtime node_id captured at csi_collector_init().
|
* Capture node_id BEFORE wifi_init_sta() or any other heavy init.
|
||||||
*
|
*
|
||||||
* This is a defensive copy of g_nvs_config.node_id taken at init time. Other
|
* Must be called from app_main() immediately after nvs_config_load().
|
||||||
* modules (edge_processing, wasm_runtime, display_ui) should prefer this
|
* WiFi driver initialization can corrupt g_nvs_config.node_id (confirmed
|
||||||
* accessor over reading g_nvs_config.node_id directly, because the global
|
* on device 80:b5:4e:c1:be:b8, NVS=3 but post-WiFi reads as 1).
|
||||||
* struct can be clobbered by memory corruption (see #232, #375, #385, #390).
|
* This early capture shields s_node_id from that corruption window.
|
||||||
*
|
*
|
||||||
* @return Node ID (0-255) as loaded from NVS or Kconfig default at boot.
|
* @param node_id Value from g_nvs_config.node_id, read right after NVS load.
|
||||||
|
*/
|
||||||
|
void csi_collector_set_node_id(uint8_t node_id);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the runtime node_id (early capture if available, otherwise init-time).
|
||||||
|
*
|
||||||
|
* Other modules (edge_processing, wasm_runtime, display_ui) should prefer
|
||||||
|
* this accessor over reading g_nvs_config.node_id directly.
|
||||||
|
*
|
||||||
|
* @return Node ID (0-255) as loaded from NVS at boot.
|
||||||
*/
|
*/
|
||||||
uint8_t csi_collector_get_node_id(void);
|
uint8_t csi_collector_get_node_id(void);
|
||||||
|
|
||||||
|
|||||||
@@ -714,8 +714,11 @@ static void process_frame(const edge_ring_slot_t *slot)
|
|||||||
s_frame_count++;
|
s_frame_count++;
|
||||||
s_latest_rssi = slot->rssi;
|
s_latest_rssi = slot->rssi;
|
||||||
|
|
||||||
/* Assumed CSI sample rate (~20 Hz for typical ESP32 CSI). */
|
/* CSI sample rate. MGMT-only promiscuous filter (RuView#396, csi_collector.c)
|
||||||
const float sample_rate = 20.0f;
|
* yields ~10 Hz from beacons; keep this value aligned with csi_collector's
|
||||||
|
* effective callback rate or estimate_bpm_zero_crossing() reports the wrong
|
||||||
|
* BPM (2× rate mismatch → 2× wrong breathing/HR). */
|
||||||
|
const float sample_rate = 10.0f;
|
||||||
|
|
||||||
/* --- Step 1-2: Phase extraction + unwrapping per subcarrier --- */
|
/* --- Step 1-2: Phase extraction + unwrapping per subcarrier --- */
|
||||||
float phases[EDGE_MAX_SUBCARRIERS];
|
float phases[EDGE_MAX_SUBCARRIERS];
|
||||||
|
|||||||
@@ -140,6 +140,11 @@ void app_main(void)
|
|||||||
/* Load runtime config (NVS overrides Kconfig defaults) */
|
/* Load runtime config (NVS overrides Kconfig defaults) */
|
||||||
nvs_config_load(&g_nvs_config);
|
nvs_config_load(&g_nvs_config);
|
||||||
|
|
||||||
|
/* Capture node_id IMMEDIATELY — before wifi_init_sta() can corrupt
|
||||||
|
* g_nvs_config. See #232/#375/#390: WiFi driver init clobbers the struct
|
||||||
|
* on some devices, reverting node_id to the Kconfig default of 1. */
|
||||||
|
csi_collector_set_node_id(g_nvs_config.node_id);
|
||||||
|
|
||||||
const esp_app_desc_t *app_desc = esp_app_get_description();
|
const esp_app_desc_t *app_desc = esp_app_get_description();
|
||||||
ESP_LOGI(TAG, "ESP32-S3 CSI Node (ADR-018) — v%s — Node ID: %d",
|
ESP_LOGI(TAG, "ESP32-S3 CSI Node (ADR-018) — v%s — Node ID: %d",
|
||||||
app_desc->version, g_nvs_config.node_id);
|
app_desc->version, g_nvs_config.node_id);
|
||||||
|
|||||||
@@ -155,7 +155,10 @@ def flash_nvs(port, baud, nvs_bin):
|
|||||||
"--chip", "esp32s3",
|
"--chip", "esp32s3",
|
||||||
"--port", port,
|
"--port", port,
|
||||||
"--baud", str(baud),
|
"--baud", str(baud),
|
||||||
"write-flash",
|
# Keep underscore form — ESP-IDF v5.4 bundles esptool 4.10.0 which only
|
||||||
|
# accepts "write_flash". pip's esptool >=5.x accepts both (hyphenated
|
||||||
|
# form preferred) but keeps underscore working. Do not "correct" this.
|
||||||
|
"write_flash",
|
||||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||||
]
|
]
|
||||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
||||||
|
|||||||
@@ -32,6 +32,5 @@ CONFIG_LWIP_SO_RCVBUF=y
|
|||||||
# FreeRTOS: increase task stack for CSI processing
|
# FreeRTOS: increase task stack for CSI processing
|
||||||
CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
|
CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
|
||||||
|
|
||||||
# ADR-081: adaptive_controller runs emit_feature_state + stream_sender
|
# Extra WiFi IRAM placement (defense-in-depth for RuView#396 SPI cache race)
|
||||||
# network I/O inside Timer Svc callbacks, exceeding the 2 KiB default.
|
CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y
|
||||||
CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=8192
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
0.6.2
|
0.6.4
|
||||||
|
|||||||
Generated
+10
-46
@@ -5127,9 +5127,9 @@
|
|||||||
"license": "BSD-3-Clause"
|
"license": "BSD-3-Clause"
|
||||||
},
|
},
|
||||||
"node_modules/@xmldom/xmldom": {
|
"node_modules/@xmldom/xmldom": {
|
||||||
"version": "0.8.11",
|
"version": "0.8.13",
|
||||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
|
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
|
||||||
"integrity": "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==",
|
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=10.0.0"
|
"node": ">=10.0.0"
|
||||||
@@ -5310,18 +5310,6 @@
|
|||||||
"node": ">= 8"
|
"node": ">= 8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/anymatch/node_modules/picomatch": {
|
|
||||||
"version": "2.3.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
|
||||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=8.6"
|
|
||||||
},
|
|
||||||
"funding": {
|
|
||||||
"url": "https://github.com/sponsors/jonschlinkert"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/arg": {
|
"node_modules/arg": {
|
||||||
"version": "5.0.2",
|
"version": "5.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
||||||
@@ -11935,18 +11923,6 @@
|
|||||||
"node": ">=8"
|
"node": ">=8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/jest-util/node_modules/picomatch": {
|
|
||||||
"version": "2.3.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
|
||||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=8.6"
|
|
||||||
},
|
|
||||||
"funding": {
|
|
||||||
"url": "https://github.com/sponsors/jonschlinkert"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/jest-validate": {
|
"node_modules/jest-validate": {
|
||||||
"version": "29.7.0",
|
"version": "29.7.0",
|
||||||
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
|
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
|
||||||
@@ -13389,18 +13365,6 @@
|
|||||||
"node": ">=8.6"
|
"node": ">=8.6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/micromatch/node_modules/picomatch": {
|
|
||||||
"version": "2.3.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
|
||||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=8.6"
|
|
||||||
},
|
|
||||||
"funding": {
|
|
||||||
"url": "https://github.com/sponsors/jonschlinkert"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/mime": {
|
"node_modules/mime": {
|
||||||
"version": "1.6.0",
|
"version": "1.6.0",
|
||||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||||
@@ -13594,9 +13558,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/node-forge": {
|
"node_modules/node-forge": {
|
||||||
"version": "1.3.3",
|
"version": "1.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz",
|
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz",
|
||||||
"integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==",
|
"integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==",
|
||||||
"license": "(BSD-3-Clause OR GPL-2.0)",
|
"license": "(BSD-3-Clause OR GPL-2.0)",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 6.13.0"
|
"node": ">= 6.13.0"
|
||||||
@@ -14056,12 +14020,12 @@
|
|||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/picomatch": {
|
"node_modules/picomatch": {
|
||||||
"version": "4.0.3",
|
"version": "2.3.2",
|
||||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
|
||||||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
"integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=12"
|
"node": ">=8.6"
|
||||||
},
|
},
|
||||||
"funding": {
|
"funding": {
|
||||||
"url": "https://github.com/sponsors/jonschlinkert"
|
"url": "https://github.com/sponsors/jonschlinkert"
|
||||||
|
|||||||
@@ -49,5 +49,10 @@
|
|||||||
"react-native-worklets": "^0.7.4",
|
"react-native-worklets": "^0.7.4",
|
||||||
"typescript": "~5.9.2"
|
"typescript": "~5.9.2"
|
||||||
},
|
},
|
||||||
|
"overrides": {
|
||||||
|
"@xmldom/xmldom": "0.8.13",
|
||||||
|
"node-forge": "^1.4.0",
|
||||||
|
"picomatch": "^2.3.2"
|
||||||
|
},
|
||||||
"private": true
|
"private": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ serde_json = { workspace = true }
|
|||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
axum = { workspace = true }
|
axum = { workspace = true }
|
||||||
|
tower-http = { workspace = true }
|
||||||
clap = { version = "4", features = ["derive"] }
|
clap = { version = "4", features = ["derive"] }
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
dirs = "5"
|
dirs = "5"
|
||||||
|
|||||||
@@ -65,6 +65,9 @@ pub struct CsiPipelineState {
|
|||||||
pub current_location: Option<(String, f32)>,
|
pub current_location: Option<(String, f32)>,
|
||||||
/// Night mode — true when camera luminance is below threshold
|
/// Night mode — true when camera luminance is below threshold
|
||||||
pub is_dark: bool,
|
pub is_dark: bool,
|
||||||
|
/// Wall-clock instant the last real ESP32 UDP CSI frame was received.
|
||||||
|
/// `None` if no frame has arrived since startup.
|
||||||
|
pub last_csi_received: Option<std::time::Instant>,
|
||||||
/// Metadata from the on-disk WiFlow JSON, if one is present. NOTE: the
|
/// Metadata from the on-disk WiFlow JSON, if one is present. NOTE: the
|
||||||
/// weights themselves are NOT loaded or executed in this crate — this
|
/// weights themselves are NOT loaded or executed in this crate — this
|
||||||
/// flag merely enables the amplitude-energy heuristic pose code path.
|
/// flag merely enables the amplitude-energy heuristic pose code path.
|
||||||
@@ -91,6 +94,7 @@ impl Default for CsiPipelineState {
|
|||||||
fingerprints: Vec::new(),
|
fingerprints: Vec::new(),
|
||||||
current_location: None,
|
current_location: None,
|
||||||
is_dark: false,
|
is_dark: false,
|
||||||
|
last_csi_received: None,
|
||||||
pose_model_present: detect_pose_model_metadata(),
|
pose_model_present: detect_pose_model_metadata(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -133,6 +137,7 @@ impl CsiPipelineState {
|
|||||||
pub fn process_frame(&mut self, frame: CsiFrame) {
|
pub fn process_frame(&mut self, frame: CsiFrame) {
|
||||||
let node_id = frame.node_id;
|
let node_id = frame.node_id;
|
||||||
self.total_frames += 1;
|
self.total_frames += 1;
|
||||||
|
self.last_csi_received = Some(std::time::Instant::now());
|
||||||
|
|
||||||
// Once every 500 frames log a one-line node stats summary. This keeps
|
// Once every 500 frames log a one-line node stats summary. This keeps
|
||||||
// us honest about the CSI shape we are actually receiving and also
|
// us honest about the CSI shape we are actually receiving and also
|
||||||
@@ -584,6 +589,9 @@ pub fn get_pipeline_output(state: &Arc<Mutex<CsiPipelineState>>) -> PipelineOutp
|
|||||||
num_nodes: st.node_frames.len(),
|
num_nodes: st.node_frames.len(),
|
||||||
current_location: st.current_location.clone(),
|
current_location: st.current_location.clone(),
|
||||||
is_dark: st.is_dark,
|
is_dark: st.is_dark,
|
||||||
|
csi_live: st.last_csi_received
|
||||||
|
.map(|t| t.elapsed() < std::time::Duration::from_secs(5))
|
||||||
|
.unwrap_or(false),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -598,6 +606,10 @@ pub struct PipelineOutput {
|
|||||||
pub num_nodes: usize,
|
pub num_nodes: usize,
|
||||||
pub current_location: Option<(String, f32)>,
|
pub current_location: Option<(String, f32)>,
|
||||||
pub is_dark: bool,
|
pub is_dark: bool,
|
||||||
|
/// True when a real ESP32 CSI frame was received in the last 5 seconds.
|
||||||
|
/// False means the pipeline is running on stale data — show a NO SIGNAL
|
||||||
|
/// indicator in the UI rather than presenting stale skeletons as live.
|
||||||
|
pub csi_live: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Serialize implementations
|
// Serialize implementations
|
||||||
|
|||||||
@@ -8,11 +8,13 @@ use crate::fusion;
|
|||||||
use crate::pointcloud;
|
use crate::pointcloud;
|
||||||
use axum::{
|
use axum::{
|
||||||
extract::State,
|
extract::State,
|
||||||
|
http::{HeaderValue, Method},
|
||||||
response::Html,
|
response::Html,
|
||||||
routing::get,
|
routing::get,
|
||||||
Json, Router,
|
Json, Router,
|
||||||
};
|
};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||||
|
|
||||||
struct AppState {
|
struct AppState {
|
||||||
latest_cloud: Mutex<pointcloud::PointCloud>,
|
latest_cloud: Mutex<pointcloud::PointCloud>,
|
||||||
@@ -108,12 +110,36 @@ pub async fn serve(bind: &str, _brain: Option<&str>) -> anyhow::Result<()> {
|
|||||||
if has_camera { eprintln!(" Camera: LIVE (/dev/video0)"); }
|
if has_camera { eprintln!(" Camera: LIVE (/dev/video0)"); }
|
||||||
else { eprintln!(" Camera: DEMO"); }
|
else { eprintln!(" Camera: DEMO"); }
|
||||||
|
|
||||||
|
// CORS — allow the hosted GitHub Pages viewer to fetch /api/splats from a
|
||||||
|
// locally-running instance of this server. Modern browsers treat
|
||||||
|
// 127.0.0.1/localhost as a "potentially trustworthy" origin so the HTTPS
|
||||||
|
// page can reach a plain-HTTP loopback backend without mixed-content
|
||||||
|
// blocking. Origins permitted:
|
||||||
|
// - https://ruvnet.github.io (the published RuView Pages demo)
|
||||||
|
// - http://localhost:* / http://127.0.0.1:* (developer running the
|
||||||
|
// viewer.html bundled with this binary)
|
||||||
|
// Anything else is denied, so this is not a "wildcard" CORS.
|
||||||
|
let cors = CorsLayer::new()
|
||||||
|
.allow_origin(AllowOrigin::predicate(|origin: &HeaderValue, _req| {
|
||||||
|
let s = match origin.to_str() {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => return false,
|
||||||
|
};
|
||||||
|
s == "https://ruvnet.github.io"
|
||||||
|
|| s.starts_with("http://localhost")
|
||||||
|
|| s.starts_with("http://127.0.0.1")
|
||||||
|
|| s == "null" // file:// origins
|
||||||
|
}))
|
||||||
|
.allow_methods([Method::GET, Method::OPTIONS])
|
||||||
|
.allow_headers([axum::http::header::CONTENT_TYPE]);
|
||||||
|
|
||||||
let app = Router::new()
|
let app = Router::new()
|
||||||
.route("/", get(index))
|
.route("/", get(index))
|
||||||
.route("/api/cloud", get(api_cloud))
|
.route("/api/cloud", get(api_cloud))
|
||||||
.route("/api/splats", get(api_splats))
|
.route("/api/splats", get(api_splats))
|
||||||
.route("/api/status", get(api_status))
|
.route("/api/status", get(api_status))
|
||||||
.route("/health", get(api_health))
|
.route("/health", get(api_health))
|
||||||
|
.layer(cors)
|
||||||
.with_state(state);
|
.with_state(state);
|
||||||
|
|
||||||
println!("╔══════════════════════════════════════════════╗");
|
println!("╔══════════════════════════════════════════════╗");
|
||||||
@@ -193,10 +219,12 @@ async fn api_splats(State(state): State<Arc<AppState>>) -> Json<serde_json::Valu
|
|||||||
let splats = state.latest_splats.lock().unwrap();
|
let splats = state.latest_splats.lock().unwrap();
|
||||||
let frames = *state.frame_count.lock().unwrap();
|
let frames = *state.frame_count.lock().unwrap();
|
||||||
let pipeline = state.latest_pipeline.lock().unwrap();
|
let pipeline = state.latest_pipeline.lock().unwrap();
|
||||||
|
let csi_live = pipeline.as_ref().map(|p| p.csi_live).unwrap_or(false);
|
||||||
Json(serde_json::json!({
|
Json(serde_json::json!({
|
||||||
"splats": &*splats,
|
"splats": &*splats,
|
||||||
"count": splats.len(),
|
"count": splats.len(),
|
||||||
"live": state.use_camera,
|
"live": state.use_camera,
|
||||||
|
"csi_live": csi_live,
|
||||||
"frame": frames,
|
"frame": frames,
|
||||||
"pipeline": &*pipeline,
|
"pipeline": &*pipeline,
|
||||||
"timestamp": chrono::Utc::now().timestamp_millis(),
|
"timestamp": chrono::Utc::now().timestamp_millis(),
|
||||||
|
|||||||
@@ -2,27 +2,60 @@
|
|||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<title>RuView — Camera + WiFi CSI Point Cloud</title>
|
<title>RuView — Camera + WiFi CSI Point Cloud</title>
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate">
|
||||||
|
<meta name="ruview-viewer-version" content="0.2.0-face-mesh">
|
||||||
|
<!-- Inline amber-dot favicon avoids a stray /favicon.ico 404 in the console. -->
|
||||||
|
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||||
<style>
|
<style>
|
||||||
body { margin: 0; background: #0a0a0a; color: #e8a634; font-family: monospace; }
|
body { margin: 0; background: #0a0a0a; color: #e8a634; font-family: monospace; }
|
||||||
canvas { display: block; }
|
canvas { display: block; }
|
||||||
#info { position: absolute; top: 10px; left: 10px; padding: 12px; background: rgba(0,0,0,0.85); border: 1px solid #e8a634; border-radius: 6px; min-width: 240px; font-size: 13px; line-height: 1.5; }
|
#info { position: absolute; top: 10px; left: 10px; padding: 12px; background: rgba(0,0,0,0.85); border: 1px solid #e8a634; border-radius: 6px; min-width: 240px; font-size: 13px; line-height: 1.5; z-index: 10; }
|
||||||
|
#cam-cta { position: absolute; bottom: 16px; left: 50%; transform: translateX(-50%); padding: 10px 18px; background: #e8a634; color: #0a0a0a; border: none; border-radius: 4px; font-family: monospace; font-size: 14px; font-weight: bold; cursor: pointer; z-index: 10; }
|
||||||
|
#cam-cta:hover { background: #ffc04d; }
|
||||||
|
#cam-cta.hidden { display: none; }
|
||||||
|
#esp-cta { position: absolute; bottom: 16px; right: 16px; padding: 8px 14px; background: transparent; color: #e8a634; border: 1px solid #e8a634; border-radius: 4px; font-family: monospace; font-size: 12px; cursor: pointer; z-index: 10; }
|
||||||
|
#esp-cta:hover { background: rgba(232, 166, 52, 0.12); }
|
||||||
|
#esp-cta.connected { background: #4f4; color: #0a0a0a; border-color: #4f4; }
|
||||||
.live { color: #4f4; } .demo { color: #f44; }
|
.live { color: #4f4; } .demo { color: #f44; }
|
||||||
|
.face { color: #4cf; }
|
||||||
.section { margin-top: 6px; padding-top: 6px; border-top: 1px solid #333; }
|
.section { margin-top: 6px; padding-top: 6px; border-top: 1px solid #333; }
|
||||||
.label { color: #888; }
|
.label { color: #888; }
|
||||||
|
#no-signal {
|
||||||
|
display: none;
|
||||||
|
position: absolute; top: 50%; left: 50%;
|
||||||
|
transform: translate(-50%, -50%);
|
||||||
|
background: rgba(160,0,0,0.93); color: #fff;
|
||||||
|
font-family: monospace; font-size: 18px; font-weight: bold;
|
||||||
|
padding: 18px 32px; border-radius: 8px;
|
||||||
|
border: 2px solid #f44; text-align: center;
|
||||||
|
pointer-events: none; z-index: 20;
|
||||||
|
}
|
||||||
|
#no-signal .sub { font-size: 12px; font-weight: normal; margin-top: 6px; color: #fbb; }
|
||||||
</style>
|
</style>
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||||
|
<!-- MediaPipe Face Mesh — runs in demo mode so each visitor sees their own face as a point cloud -->
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4/face_mesh.js"></script>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils@0.3/camera_utils.js"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
<div id="no-signal">
|
||||||
|
● NO CSI SIGNAL
|
||||||
|
<div class="sub">No ESP32 frames received for >5s.<br>Check that your node is powered and provisioned.</div>
|
||||||
|
</div>
|
||||||
<div id="info">
|
<div id="info">
|
||||||
<h3 style="margin:0 0 8px 0">RuView Point Cloud</h3>
|
<h3 style="margin:0 0 4px 0">RuView · Seldon Vault</h3>
|
||||||
|
<div style="font-size: 11px; color: #888; margin-bottom: 8px; max-width: 240px; line-height: 1.4; font-style: italic;">"Psychohistory deals with reactions of human conglomerates to fixed social and economic stimuli." — Hari Seldon</div>
|
||||||
<div id="stats">Loading...</div>
|
<div id="stats">Loading...</div>
|
||||||
</div>
|
</div>
|
||||||
|
<button id="cam-cta">▶ Project Subject — render your face into the Vault</button>
|
||||||
|
<button id="esp-cta" title="Stream live CSI from a local ruview-pointcloud serve instance (e.g. http://127.0.0.1:9880)">📡 Connect ESP32…</button>
|
||||||
<script>
|
<script>
|
||||||
var scene = new THREE.Scene();
|
var scene = new THREE.Scene();
|
||||||
scene.background = new THREE.Color(0x0a0a0a);
|
scene.background = new THREE.Color(0x0a0a0a);
|
||||||
var camera = new THREE.PerspectiveCamera(75, window.innerWidth/window.innerHeight, 0.1, 100);
|
var camera = new THREE.PerspectiveCamera(72, window.innerWidth/window.innerHeight, 0.1, 200);
|
||||||
camera.position.set(0, 2, -4);
|
camera.position.set(0, 0.2, -3.5);
|
||||||
camera.lookAt(0, 0, 2);
|
camera.lookAt(0, 0, 2);
|
||||||
|
|
||||||
var renderer = new THREE.WebGLRenderer({ antialias: true });
|
var renderer = new THREE.WebGLRenderer({ antialias: true });
|
||||||
@@ -38,6 +71,11 @@
|
|||||||
var skeletonGroup = null;
|
var skeletonGroup = null;
|
||||||
var prevTimestamp = 0;
|
var prevTimestamp = 0;
|
||||||
var frameRateVal = 0;
|
var frameRateVal = 0;
|
||||||
|
// No-signal detection: track server-reported csi_live flag
|
||||||
|
var noSignalBanner = document.getElementById("no-signal");
|
||||||
|
function setNoSignal(isNoSignal) {
|
||||||
|
noSignalBanner.style.display = isNoSignal ? "block" : "none";
|
||||||
|
}
|
||||||
|
|
||||||
// COCO skeleton connections: pairs of keypoint indices
|
// COCO skeleton connections: pairs of keypoint indices
|
||||||
// 0=nose 1=leftEye 2=rightEye 3=leftEar 4=rightEar
|
// 0=nose 1=leftEye 2=rightEye 3=leftEar 4=rightEar
|
||||||
@@ -104,10 +142,436 @@
|
|||||||
scene.add(skeletonGroup);
|
scene.add(skeletonGroup);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchCloud() {
|
// ----- Transport configuration -----
|
||||||
|
// ?backend=<url> → fetch splats from <url>/api/splats (CORS-permitting host)
|
||||||
|
// ?backend=auto → try /api/splats, fall back to synthetic demo on failure (default)
|
||||||
|
// ?backend=demo → always render synthetic demo (no network)
|
||||||
|
// ?live=1 → require live; show error instead of demo fallback
|
||||||
|
var urlParams = new URLSearchParams(window.location.search);
|
||||||
|
var backendArg = urlParams.get("backend") || "auto";
|
||||||
|
var requireLive = urlParams.get("live") === "1";
|
||||||
|
var transportMode = "demo"; // resolved at first fetch: "live" | "remote" | "demo"
|
||||||
|
var demoStartMs = Date.now();
|
||||||
|
var demoFrameNum = 0;
|
||||||
|
var latestFaceLandmarks = null; // populated by MediaPipe when camera enabled
|
||||||
|
var faceMeshState = "idle"; // "idle" | "starting" | "running" | "denied" | "unavailable"
|
||||||
|
|
||||||
|
// ----- MediaPipe Face Mesh (browser equivalent of camera-depth backprojection) -----
|
||||||
|
// Locally, ruview-pointcloud serve fuses real camera depth + WiFi CSI. In the
|
||||||
|
// browser we don't have depth from a webcam, but Face Mesh produces 468
|
||||||
|
// 3D landmarks (x,y in [0,1], z roughly in [-0.5,0.5]) at ~30 fps — enough to
|
||||||
|
// reproduce the "I can see the outline of my face in points" experience. The
|
||||||
|
// landmarks feed into the same splat render path as live /api/splats data.
|
||||||
|
async function startFaceMesh() {
|
||||||
|
if (faceMeshState !== "idle") return;
|
||||||
|
if (!window.FaceMesh || !window.Camera) {
|
||||||
|
faceMeshState = "unavailable";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
faceMeshState = "starting";
|
||||||
try {
|
try {
|
||||||
var resp = await fetch("/api/splats");
|
var videoEl = document.createElement("video");
|
||||||
|
videoEl.style.display = "none";
|
||||||
|
videoEl.autoplay = true;
|
||||||
|
videoEl.playsInline = true;
|
||||||
|
videoEl.muted = true;
|
||||||
|
document.body.appendChild(videoEl);
|
||||||
|
|
||||||
|
var fm = new FaceMesh({
|
||||||
|
locateFile: function(file) {
|
||||||
|
return "https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4/" + file;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
fm.setOptions({
|
||||||
|
maxNumFaces: 1,
|
||||||
|
refineLandmarks: true,
|
||||||
|
minDetectionConfidence: 0.5,
|
||||||
|
minTrackingConfidence: 0.5
|
||||||
|
});
|
||||||
|
fm.onResults(function(results) {
|
||||||
|
if (results.multiFaceLandmarks && results.multiFaceLandmarks[0]) {
|
||||||
|
latestFaceLandmarks = results.multiFaceLandmarks[0];
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
var mpCamera = new Camera(videoEl, {
|
||||||
|
onFrame: async function() { await fm.send({ image: videoEl }); },
|
||||||
|
width: 640,
|
||||||
|
height: 480
|
||||||
|
});
|
||||||
|
await mpCamera.start();
|
||||||
|
faceMeshState = "running";
|
||||||
|
var btn = document.getElementById("cam-cta");
|
||||||
|
if (btn) btn.classList.add("hidden");
|
||||||
|
} catch (err) {
|
||||||
|
faceMeshState = "denied";
|
||||||
|
console.warn("Face mesh unavailable:", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Foundation-inspired galactic context (Asimov / Trantor / Seldon) ----
|
||||||
|
// Shared between face-mesh and synthetic-fallback paths. The subject (face
|
||||||
|
// or procedural figure) is the foreground; this function paints the Seldon
|
||||||
|
// time-vault around it: holographic surveyor grid underfoot, slow galactic
|
||||||
|
// spiral receding into the distance, distant starfield, and a halo ring.
|
||||||
|
function pushFoundationContext(splats) {
|
||||||
|
var t = (Date.now() - demoStartMs) / 1000.0;
|
||||||
|
|
||||||
|
// 1. Holographic surveyor grid — amber lattice at y=+1.4 (renders below
|
||||||
|
// the subject because the renderer flips y to Three.js Y-up).
|
||||||
|
var gx, gz;
|
||||||
|
for (gx = -10; gx <= 10; gx++) {
|
||||||
|
for (gz = 0; gz <= 30; gz++) {
|
||||||
|
var alpha = 0.35 + 0.15 * Math.sin(t * 0.5 + gz * 0.2);
|
||||||
|
splats.push({
|
||||||
|
center: [gx * 0.5, 1.4, gz * 0.4],
|
||||||
|
color: [0.40 * alpha, 0.28 * alpha, 0.10 * alpha],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.018, 0.018, 0.018]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (gz = 0; gz <= 30; gz += 2) {
|
||||||
|
for (gx = -20; gx <= 20; gx++) {
|
||||||
|
splats.push({
|
||||||
|
center: [gx * 0.25, 1.4, gz * 0.4 + 0.1],
|
||||||
|
color: [0.30, 0.22, 0.08],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.014, 0.014, 0.014]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Galactic spiral — Trantor recedes behind the subject. ~640 stars
|
||||||
|
// across two logarithmic arms, slowly rotating. Warmer at the core,
|
||||||
|
// cooler at the edges (Hertzsprung-Russell-ish).
|
||||||
|
var arm, k, theta_arm, r_arm, sx, sz, twist, arm_color;
|
||||||
|
for (arm = 0; arm < 2; arm++) {
|
||||||
|
for (k = 0; k < 320; k++) {
|
||||||
|
var prog = k / 320;
|
||||||
|
theta_arm = arm * Math.PI + prog * 6.0 + t * 0.05;
|
||||||
|
r_arm = 4.0 + prog * 14.0;
|
||||||
|
twist = Math.sin(prog * 8.0) * 0.4;
|
||||||
|
sx = Math.cos(theta_arm) * r_arm + twist;
|
||||||
|
sz = Math.sin(theta_arm) * r_arm + 12.0;
|
||||||
|
var coreFade = Math.max(0.15, 1.0 - prog);
|
||||||
|
arm_color = [
|
||||||
|
coreFade * 0.85 + 0.15 * (1 - prog),
|
||||||
|
coreFade * 0.70 + 0.20,
|
||||||
|
coreFade * 0.55 + 0.45 * prog
|
||||||
|
];
|
||||||
|
splats.push({
|
||||||
|
center: [sx, -2.5 + Math.sin(prog * 12) * 0.3, sz],
|
||||||
|
color: arm_color,
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.025, 0.025, 0.025]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Distant starfield — 800 deterministic stars on a spherical shell.
|
||||||
|
// Fixed LCG seed so visitors don't see noise flicker between frames.
|
||||||
|
var seed = 42;
|
||||||
|
function nextRand() {
|
||||||
|
seed = (seed * 1664525 + 1013904223) >>> 0;
|
||||||
|
return seed / 4294967296;
|
||||||
|
}
|
||||||
|
var s, r_s, phi, costheta, sinphi;
|
||||||
|
for (s = 0; s < 800; s++) {
|
||||||
|
phi = nextRand() * Math.PI * 2;
|
||||||
|
costheta = nextRand() * 2 - 1;
|
||||||
|
sinphi = Math.sqrt(1 - costheta * costheta);
|
||||||
|
r_s = 22 + nextRand() * 8;
|
||||||
|
var brightness = 0.4 + nextRand() * 0.6;
|
||||||
|
var hue = nextRand();
|
||||||
|
splats.push({
|
||||||
|
center: [
|
||||||
|
Math.cos(phi) * sinphi * r_s,
|
||||||
|
costheta * r_s * 0.5 - 1.0,
|
||||||
|
Math.sin(phi) * sinphi * r_s + 5.0
|
||||||
|
],
|
||||||
|
color: hue > 0.85
|
||||||
|
? [brightness, brightness * 0.85, brightness * 0.6]
|
||||||
|
: (hue > 0.3
|
||||||
|
? [brightness * 0.9, brightness * 0.95, brightness]
|
||||||
|
: [brightness * 0.5, brightness * 0.7, brightness]),
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.020, 0.020, 0.020]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Holographic projection halo around the subject — Seldon vault
|
||||||
|
// projections always had a faint encircling ring of particles.
|
||||||
|
var ring;
|
||||||
|
for (ring = 0; ring < 60; ring++) {
|
||||||
|
var rt = ring / 60 * Math.PI * 2 + t * 0.3;
|
||||||
|
splats.push({
|
||||||
|
center: [
|
||||||
|
Math.cos(rt) * 1.6,
|
||||||
|
Math.sin(rt) * 1.2 - 0.2,
|
||||||
|
2.0 + Math.sin(rt * 3 + t * 0.5) * 0.3
|
||||||
|
],
|
||||||
|
color: [0.95, 0.55, 0.15],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.014, 0.014, 0.014]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map a single landmark to world coords. Coord conventions:
|
||||||
|
// x: 0.5 - lm.x → mirror so left-of-screen = your left side (selfie)
|
||||||
|
// y: lm.y - 0.5 → keep MediaPipe's y-DOWN convention; the renderer's
|
||||||
|
// existing -y flip in updateSplats does the single flip
|
||||||
|
// to Three.js Y-up. Pre-flipping here would double-flip
|
||||||
|
// and the face would render upside down.
|
||||||
|
// z: 2.0 + lm.z*8 → amplify lm.z (~[-0.1,+0.1]) so the nose/eye-socket
|
||||||
|
// depth is visible from an oblique camera angle.
|
||||||
|
function lmToCenter(lm) {
|
||||||
|
return [
|
||||||
|
(0.5 - lm.x) * 4.0,
|
||||||
|
(lm.y - 0.5) * 3.0,
|
||||||
|
2.0 + lm.z * 8.0
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushFaceSplat(splats, center, alpha) {
|
||||||
|
splats.push({
|
||||||
|
center: center,
|
||||||
|
color: [0.95 * alpha, 0.65 * alpha, 0.20 * alpha],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.006, 0.006, 0.006]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// FACEMESH_TESSELATION is a flat array [a0,b0, a1,b1, ...] of vertex indices
|
||||||
|
// forming edges of the triangulated face mesh. ~1300 edges × 2 = 2600 entries.
|
||||||
|
// We interpolate 6 splats per edge → ~8000 splats per face vs 478 vertices.
|
||||||
|
var FACE_EDGES = (typeof FACEMESH_TESSELATION !== "undefined") ? FACEMESH_TESSELATION : null;
|
||||||
|
|
||||||
|
// Push the user's face mesh point cloud into `splats` (no Foundation
|
||||||
|
// context — that is the demo path's responsibility). Used both as the
|
||||||
|
// demo subject AND as an overlay on top of live/remote backend data
|
||||||
|
// when the camera is enabled. Returns true if any splats were pushed.
|
||||||
|
function pushFaceSplats(splats) {
|
||||||
|
if (faceMeshState !== "running" || !latestFaceLandmarks) return false;
|
||||||
|
var lms = latestFaceLandmarks;
|
||||||
|
var i;
|
||||||
|
// 1. Original 478 vertices — bright anchor points for features.
|
||||||
|
for (i = 0; i < lms.length; i++) {
|
||||||
|
splats.push({
|
||||||
|
center: lmToCenter(lms[i]),
|
||||||
|
color: [1.0, 0.72, 0.25],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.010, 0.010, 0.010]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// 2. Edge interpolation — 6 splats per FACEMESH_TESSELATION edge.
|
||||||
|
if (FACE_EDGES) {
|
||||||
|
var edgeCount = FACE_EDGES.length;
|
||||||
|
var SAMPLES = 6;
|
||||||
|
var e, a, b, t, f;
|
||||||
|
for (e = 0; e < edgeCount; e += 2) {
|
||||||
|
a = lms[FACE_EDGES[e]];
|
||||||
|
b = lms[FACE_EDGES[e + 1]];
|
||||||
|
if (!a || !b) continue;
|
||||||
|
var aPos = lmToCenter(a);
|
||||||
|
var bPos = lmToCenter(b);
|
||||||
|
var ax = aPos[0], ay = aPos[1], az = aPos[2];
|
||||||
|
var bx = bPos[0], by = bPos[1], bz = bPos[2];
|
||||||
|
for (t = 1; t <= SAMPLES; t++) {
|
||||||
|
f = t / (SAMPLES + 1);
|
||||||
|
pushFaceSplat(splats, [
|
||||||
|
ax * (1 - f) + bx * f,
|
||||||
|
ay * (1 - f) + by * f,
|
||||||
|
az * (1 - f) + bz * f
|
||||||
|
], 0.85);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function faceMeshFrame() {
|
||||||
|
if (faceMeshState !== "running" || !latestFaceLandmarks) return null;
|
||||||
|
var splats = [];
|
||||||
|
pushFaceSplats(splats);
|
||||||
|
pushFoundationContext(splats);
|
||||||
|
demoFrameNum += 1;
|
||||||
|
return {
|
||||||
|
splats: splats,
|
||||||
|
count: splats.length,
|
||||||
|
frame: demoFrameNum,
|
||||||
|
live: false,
|
||||||
|
source: "face-mesh",
|
||||||
|
pipeline: {
|
||||||
|
skeleton: null,
|
||||||
|
vitals: { breathing_rate: 14, motion_score: 0.15 }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildSplatsUrl() {
|
||||||
|
if (backendArg === "demo") return null;
|
||||||
|
if (backendArg === "auto") return "/api/splats";
|
||||||
|
// User-supplied URL — strip trailing slash and append /api/splats.
|
||||||
|
var base = backendArg.replace(/\/+$/, "");
|
||||||
|
return base + "/api/splats";
|
||||||
|
}
|
||||||
|
|
||||||
|
function syntheticFrame() {
|
||||||
|
// Used when camera permission is denied / unavailable. Renders a
|
||||||
|
// procedural standing figure inside the Seldon vault context.
|
||||||
|
// y-down convention: head at small/negative y, feet at large/positive y;
|
||||||
|
// the renderer flips y so head ends up at the top of the screen.
|
||||||
|
var t = (Date.now() - demoStartMs) / 1000.0;
|
||||||
|
var sway = Math.sin(t * 0.8) * 0.05;
|
||||||
|
var breath = Math.sin(t * 1.2) * 0.015;
|
||||||
|
var splats = [];
|
||||||
|
|
||||||
|
// Standing figure — 240 points in a vertical cylinder, denser than
|
||||||
|
// before to feel like a holographic projection.
|
||||||
|
var ring, k_ring, theta, r, py;
|
||||||
|
for (ring = 0; ring < 30; ring++) {
|
||||||
|
py = -1.0 + (ring / 30) * 2.2; // head (-1.0) → feet (+1.2) in y-down
|
||||||
|
r = 0.20 + breath * (py < 0 ? 1.5 : 0); // chest expands more on inhale
|
||||||
|
for (k_ring = 0; k_ring < 16; k_ring++) {
|
||||||
|
theta = (k_ring / 16) * Math.PI * 2;
|
||||||
|
splats.push({
|
||||||
|
center: [
|
||||||
|
sway + Math.cos(theta) * r,
|
||||||
|
py,
|
||||||
|
2.3 + Math.sin(theta) * r
|
||||||
|
],
|
||||||
|
color: [0.91, 0.65, 0.20],
|
||||||
|
opacity: 1.0,
|
||||||
|
scale: [0.018, 0.018, 0.018]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 17 COCO keypoints in normalized [0,1] image coords (matches live shape)
|
||||||
|
var headY = 0.18;
|
||||||
|
var keypoints = [
|
||||||
|
[0.50 + sway * 0.05, headY, 0.95], // 0 nose
|
||||||
|
[0.52 + sway * 0.05, headY - 0.01, 0.92], // 1 leftEye
|
||||||
|
[0.48 + sway * 0.05, headY - 0.01, 0.92], // 2 rightEye
|
||||||
|
[0.54 + sway * 0.05, headY, 0.85], // 3 leftEar
|
||||||
|
[0.46 + sway * 0.05, headY, 0.85], // 4 rightEar
|
||||||
|
[0.60 + sway * 0.04, 0.32, 0.93], // 5 leftShoulder
|
||||||
|
[0.40 + sway * 0.04, 0.32, 0.93], // 6 rightShoulder
|
||||||
|
[0.65 + sway * 0.03, 0.46, 0.90], // 7 leftElbow
|
||||||
|
[0.35 + sway * 0.03, 0.46, 0.90], // 8 rightElbow
|
||||||
|
[0.68, 0.60 + Math.sin(t * 1.4) * 0.02, 0.86], // 9 leftWrist
|
||||||
|
[0.32, 0.60 - Math.sin(t * 1.4) * 0.02, 0.86], // 10 rightWrist
|
||||||
|
[0.57, 0.58, 0.94], // 11 leftHip
|
||||||
|
[0.43, 0.58, 0.94], // 12 rightHip
|
||||||
|
[0.58, 0.74, 0.90], // 13 leftKnee
|
||||||
|
[0.42, 0.74, 0.90], // 14 rightKnee
|
||||||
|
[0.59, 0.92, 0.88], // 15 leftAnkle
|
||||||
|
[0.41, 0.92, 0.88] // 16 rightAnkle
|
||||||
|
];
|
||||||
|
|
||||||
|
// Wrap the figure in the Seldon-vault context (grid, spiral, starfield, halo)
|
||||||
|
pushFoundationContext(splats);
|
||||||
|
|
||||||
|
demoFrameNum += 1;
|
||||||
|
return {
|
||||||
|
splats: splats,
|
||||||
|
count: splats.length,
|
||||||
|
frame: demoFrameNum,
|
||||||
|
live: false,
|
||||||
|
pipeline: {
|
||||||
|
skeleton: { keypoints: keypoints, confidence: 0.86 },
|
||||||
|
vitals: {
|
||||||
|
breathing_rate: 14 + Math.round(Math.sin(t * 0.05) * 2),
|
||||||
|
motion_score: 0.18 + Math.abs(sway) * 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickDemoFrame() {
|
||||||
|
// Prefer real face-mesh data when the camera is running; else procedural.
|
||||||
|
return faceMeshFrame() || syntheticFrame();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once auto mode confirms there is no /api/splats backend on this origin,
|
||||||
|
// set this flag so we stop hammering the network with 404 fetches every
|
||||||
|
// tick. Console stays clean; demo renders locally.
|
||||||
|
var networkDisabled = false;
|
||||||
|
|
||||||
|
// Exponential backoff state for explicit ?backend=<url>. The user's
|
||||||
|
// local server may be down (ERR_CONNECTION_REFUSED) and we shouldn't
|
||||||
|
// hammer it 10 Hz indefinitely. After each failure we lengthen the
|
||||||
|
// delay; on success we snap back to the normal cadence.
|
||||||
|
var BASE_INTERVAL_MS = 250;
|
||||||
|
var MAX_INTERVAL_MS = 30000;
|
||||||
|
var currentIntervalMs = BASE_INTERVAL_MS;
|
||||||
|
var consecutiveFailures = 0;
|
||||||
|
var fetchTimer = null;
|
||||||
|
var lastBackendError = null;
|
||||||
|
|
||||||
|
function scheduleNextFetch(delayMs) {
|
||||||
|
if (fetchTimer) clearTimeout(fetchTimer);
|
||||||
|
fetchTimer = setTimeout(fetchCloud, delayMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchCloud() {
|
||||||
|
// Demo-only mode: never hit the network. Use the normal cadence.
|
||||||
|
if (backendArg === "demo" || networkDisabled) {
|
||||||
|
transportMode = "demo";
|
||||||
|
handleData(pickDemoFrame());
|
||||||
|
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
var resp = await fetch(buildSplatsUrl(), { cache: "no-store" });
|
||||||
|
if (!resp.ok) throw new Error("HTTP " + resp.status);
|
||||||
var data = await resp.json();
|
var data = await resp.json();
|
||||||
|
transportMode = (backendArg === "auto") ? "live" : "remote";
|
||||||
|
consecutiveFailures = 0;
|
||||||
|
currentIntervalMs = BASE_INTERVAL_MS;
|
||||||
|
lastBackendError = null;
|
||||||
|
handleData(data);
|
||||||
|
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||||
|
} catch (err) {
|
||||||
|
consecutiveFailures += 1;
|
||||||
|
lastBackendError = err && err.message ? err.message : String(err);
|
||||||
|
if (requireLive) {
|
||||||
|
document.getElementById("stats").innerHTML =
|
||||||
|
'<span class="demo">● OFFLINE</span><br>Live backend required (?live=1) but unreachable.<br><span class="label">' + lastBackendError + '</span>';
|
||||||
|
// Even strict-live: back off so we don't spam.
|
||||||
|
currentIntervalMs = Math.min(currentIntervalMs * 2, MAX_INTERVAL_MS);
|
||||||
|
scheduleNextFetch(currentIntervalMs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Auto mode + first failure → assume static host (Pages), disable
|
||||||
|
// network entirely so the console stays clean.
|
||||||
|
if (backendArg === "auto") {
|
||||||
|
networkDisabled = true;
|
||||||
|
transportMode = "demo";
|
||||||
|
handleData(pickDemoFrame());
|
||||||
|
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Explicit backend (?backend=<url>) — keep trying with
|
||||||
|
// exponential backoff: 250 ms → 500 ms → 1 s → 2 s … up to 30 s.
|
||||||
|
// Render the demo while we wait so the scene stays alive, and
|
||||||
|
// surface the failure so the user knows the server is down.
|
||||||
|
currentIntervalMs = Math.min(Math.max(BASE_INTERVAL_MS * Math.pow(2, consecutiveFailures - 1), 1000), MAX_INTERVAL_MS);
|
||||||
|
transportMode = "demo";
|
||||||
|
var demoFrame = pickDemoFrame();
|
||||||
|
demoFrame._backendUnreachable = true;
|
||||||
|
demoFrame._backendUrl = backendArg;
|
||||||
|
demoFrame._backendError = lastBackendError;
|
||||||
|
demoFrame._retryInMs = currentIntervalMs;
|
||||||
|
handleData(demoFrame);
|
||||||
|
scheduleNextFetch(currentIntervalMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleData(data) {
|
||||||
|
try {
|
||||||
if (data.splats && data.frame !== lastFrame) {
|
if (data.splats && data.frame !== lastFrame) {
|
||||||
// Compute CSI frame rate
|
// Compute CSI frame rate
|
||||||
var now = Date.now();
|
var now = Date.now();
|
||||||
@@ -117,24 +581,75 @@
|
|||||||
}
|
}
|
||||||
prevTimestamp = now;
|
prevTimestamp = now;
|
||||||
lastFrame = data.frame;
|
lastFrame = data.frame;
|
||||||
updateSplats(data.splats);
|
|
||||||
|
|
||||||
// Draw skeleton if available
|
// Overlay browser face mesh on top of backend splats when both
|
||||||
|
// are active — lets visitors see their own face *plus* the
|
||||||
|
// ESP32-driven point cloud in the same scene. Demo mode (where
|
||||||
|
// data.source === "face-mesh") already includes the face, so
|
||||||
|
// we skip this branch there to avoid double-counting.
|
||||||
|
var rendered = data.splats;
|
||||||
|
var faceOverlay = false;
|
||||||
|
if (data.source !== "face-mesh"
|
||||||
|
&& faceMeshState === "running"
|
||||||
|
&& latestFaceLandmarks) {
|
||||||
|
rendered = data.splats.slice();
|
||||||
|
pushFaceSplats(rendered);
|
||||||
|
faceOverlay = true;
|
||||||
|
}
|
||||||
|
data._faceOverlay = faceOverlay;
|
||||||
|
updateSplats(rendered);
|
||||||
|
|
||||||
|
// No-signal detection: hide skeleton and show banner when
|
||||||
|
// the server reports no live CSI frames in the last 5s.
|
||||||
var pipe = data.pipeline;
|
var pipe = data.pipeline;
|
||||||
if (pipe && pipe.skeleton && pipe.skeleton.keypoints) {
|
var csiLive = data.csi_live || (pipe && pipe.csi_live);
|
||||||
|
// Only show no-signal when connected to a real backend
|
||||||
|
// (not demo/face-mesh mode where csi_live is always false).
|
||||||
|
var showNoSignal = (transportMode === "live" || transportMode === "remote")
|
||||||
|
&& csiLive === false;
|
||||||
|
setNoSignal(showNoSignal);
|
||||||
|
if (showNoSignal) {
|
||||||
|
clearSkeleton();
|
||||||
|
} else if (pipe && pipe.skeleton && pipe.skeleton.keypoints) {
|
||||||
drawSkeleton(pipe.skeleton.keypoints);
|
drawSkeleton(pipe.skeleton.keypoints);
|
||||||
} else {
|
} else {
|
||||||
clearSkeleton();
|
clearSkeleton();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build info panel
|
// Build info panel — badge reflects active transport
|
||||||
var mode = data.live
|
var mode;
|
||||||
? '<span class="live">● LIVE</span>'
|
if (transportMode === "live") {
|
||||||
: '<span class="demo">● DEMO</span>';
|
mode = '<span class="live">● LIVE</span> Local Backend';
|
||||||
var html = mode + " Camera + CSI<br>"
|
} else if (transportMode === "remote") {
|
||||||
+ "Splats: " + data.count + "<br>"
|
mode = '<span class="live">● REMOTE</span> ' + backendArg;
|
||||||
|
} else if (data.source === "face-mesh") {
|
||||||
|
mode = '<span class="face">● DEMO</span> Your Face (MediaPipe)';
|
||||||
|
} else {
|
||||||
|
mode = '<span class="demo">● DEMO</span> Synthetic';
|
||||||
|
}
|
||||||
|
if (data._faceOverlay) {
|
||||||
|
mode += ' <span class="face">+ face overlay</span>';
|
||||||
|
}
|
||||||
|
var splatCount = rendered ? rendered.length : data.count;
|
||||||
|
var html = mode + "<br>"
|
||||||
|
+ "Splats: " + splatCount + "<br>"
|
||||||
+ "Frame: " + data.frame;
|
+ "Frame: " + data.frame;
|
||||||
|
|
||||||
|
// Unreachable backend banner — explicit ?backend=<url> failed
|
||||||
|
// to connect. Show actionable guidance instead of leaving the
|
||||||
|
// user staring at a "demo" badge wondering why their ESP32
|
||||||
|
// feed isn't visible.
|
||||||
|
if (data._backendUnreachable) {
|
||||||
|
var nextSec = Math.round((data._retryInMs || 1000) / 1000);
|
||||||
|
html += '<div class="section">'
|
||||||
|
+ '<span class="demo">● ' + data._backendUrl + '</span> unreachable'
|
||||||
|
+ '<br><span class="label">' + (data._backendError || "connection failed") + '</span>'
|
||||||
|
+ '<br><span class="label">retry in ' + nextSec + 's</span>'
|
||||||
|
+ '<br><br><span class="label">start the server:</span>'
|
||||||
|
+ '<br><code style="color:#e8a634">cargo run -p wifi-densepose-pointcloud --release \\<br> -- serve --bind 127.0.0.1:9880</code>'
|
||||||
|
+ '</div>';
|
||||||
|
}
|
||||||
|
|
||||||
// CSI frame rate
|
// CSI frame rate
|
||||||
html += '<div class="section">'
|
html += '<div class="section">'
|
||||||
+ '<span class="label">CSI Rate:</span> '
|
+ '<span class="label">CSI Rate:</span> '
|
||||||
@@ -187,8 +702,69 @@
|
|||||||
}
|
}
|
||||||
} catch(e) {}
|
} catch(e) {}
|
||||||
}
|
}
|
||||||
|
// Wire the camera CTA. The camera is now overlay-able on every
|
||||||
|
// transport mode: in demo it IS the subject; in live/remote it
|
||||||
|
// overlays the backend splats so the visitor sees their face
|
||||||
|
// alongside the ESP32-driven point cloud.
|
||||||
|
(function wireCamCta() {
|
||||||
|
var btn = document.getElementById("cam-cta");
|
||||||
|
if (!btn) return;
|
||||||
|
if (requireLive) {
|
||||||
|
// Strict-live mode shows the offline panel — no camera UI.
|
||||||
|
btn.classList.add("hidden");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// In remote mode, label the button as an overlay action.
|
||||||
|
if (backendArg.startsWith("http")) {
|
||||||
|
btn.textContent = "▶ Add face overlay";
|
||||||
|
}
|
||||||
|
btn.addEventListener("click", function() {
|
||||||
|
btn.textContent = backendArg.startsWith("http")
|
||||||
|
? "Starting overlay…"
|
||||||
|
: "Initializing the Vault…";
|
||||||
|
startFaceMesh();
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// Wire the ESP32 backend CTA: prompts for a ruview-pointcloud serve URL,
|
||||||
|
// persists last-used value in localStorage, and reloads with the
|
||||||
|
// ?backend=<url> query so the existing remote-mode path takes over.
|
||||||
|
// Disconnect by clicking again when already connected.
|
||||||
|
(function wireEspCta() {
|
||||||
|
var btn = document.getElementById("esp-cta");
|
||||||
|
if (!btn) return;
|
||||||
|
var connected = backendArg.startsWith("http");
|
||||||
|
if (connected) {
|
||||||
|
btn.classList.add("connected");
|
||||||
|
btn.textContent = "📡 ESP32 connected · disconnect";
|
||||||
|
}
|
||||||
|
btn.addEventListener("click", function() {
|
||||||
|
if (connected) {
|
||||||
|
// Strip ?backend= from current URL and reload — return to demo.
|
||||||
|
var u = new URL(window.location.href);
|
||||||
|
u.searchParams.delete("backend");
|
||||||
|
window.location.href = u.toString();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var stored;
|
||||||
|
try { stored = localStorage.getItem("ruview.backendUrl"); } catch (_) { stored = null; }
|
||||||
|
var def = stored || "http://127.0.0.1:9880";
|
||||||
|
var url = window.prompt(
|
||||||
|
"Enter the ruview-pointcloud serve URL (run `ruview-pointcloud serve` locally with your ESP32 streaming CSI to UDP port 3333):",
|
||||||
|
def
|
||||||
|
);
|
||||||
|
if (!url) return;
|
||||||
|
url = url.replace(/\/+$/, "");
|
||||||
|
try { localStorage.setItem("ruview.backendUrl", url); } catch (_) {}
|
||||||
|
var u2 = new URL(window.location.href);
|
||||||
|
u2.searchParams.set("backend", url);
|
||||||
|
window.location.href = u2.toString();
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// fetchCloud self-schedules via setTimeout — no setInterval to avoid
|
||||||
|
// overlapping calls on slow networks and to support exponential backoff.
|
||||||
fetchCloud();
|
fetchCloud();
|
||||||
setInterval(fetchCloud, 500);
|
|
||||||
|
|
||||||
function updateSplats(splats) {
|
function updateSplats(splats) {
|
||||||
if (pointsMesh) scene.remove(pointsMesh);
|
if (pointsMesh) scene.remove(pointsMesh);
|
||||||
|
|||||||
Reference in New Issue
Block a user