mirror of
https://github.com/ruvnet/RuView
synced 2026-06-22 12:23:18 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ruv
|
53f52bbe74 | ||
|
ruv
|
f5ec749d5c | ||
|
ruv
|
01f7209f19 | ||
|
ruv
|
dfe1ce8084 | ||
|
ruv
|
779cb83343 | ||
|
ruv
|
b34e5ee8cf | ||
|
ruv
|
f25b82ddb5 | ||
|
ruv
|
cedb28db83 | ||
|
ruv
|
6fe405a5f7 | ||
|
ruv
|
21ad10e8d8 | ||
|
ruv
|
eed5feeab2 | ||
|
ruv
|
18c09d3305 | ||
|
ruv
|
c9fbda12fd | ||
|
ruv
|
1c922ed4ab | ||
|
ruv
|
4483a88b22 | ||
|
ruv
|
79826b9a4d | ||
|
ruv
|
5846c3d6d2 | ||
|
ruv
|
39ec05edcb | ||
|
ruv
|
db1ccbff49 | ||
|
ruv
|
508e2c65d4 | ||
|
ruv
|
b2dd5851c0 | ||
|
ruv
|
d285ba6216 | ||
|
ruv
|
49d18671ba | ||
|
ruv
|
5faeddcf47 | ||
|
ruv
|
436d383c99 | ||
|
ruv
|
2dddd458e7 | ||
|
ruv
|
177624174e | ||
|
ruv
|
8c062fbaa4 | ||
|
ruv
|
a6ac08c662 | ||
|
ruv
|
9c95bfac0c |
@@ -1,58 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Keep all third-party GitHub Actions on verified, pinned commit SHAs.
|
||||
# Pairs with the SHA pinning in security-scan.yml and ci.yml so that
|
||||
# future bumps stay automated and reviewable rather than drifting back
|
||||
# to mutable @master / @main refs. See issue #442.
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- dependencies
|
||||
- github-actions
|
||||
|
||||
# Mobile app npm deps. Includes the @xmldom/xmldom, node-forge, and
|
||||
# picomatch advisories from #442 plus axios and any future surface.
|
||||
- package-ecosystem: npm
|
||||
directory: /ui/mobile
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- dependencies
|
||||
- mobile
|
||||
|
||||
# Desktop UI npm deps. Direct vite devDep currently has a HIGH advisory
|
||||
# (dev-server-only path traversal); track future bumps automatically.
|
||||
- package-ecosystem: npm
|
||||
directory: /v2/crates/wifi-densepose-desktop/ui
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- dependencies
|
||||
- desktop
|
||||
|
||||
# Python deps used by v1/ and the FastAPI service. requirements.txt is
|
||||
# only loosely pinned; let Dependabot surface upstream CVE bumps.
|
||||
- package-ecosystem: pip
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- dependencies
|
||||
- python
|
||||
|
||||
# Rust workspace (15+ crates). cargo audit is not currently wired into
|
||||
# any workflow, so Dependabot is the primary automated bump path.
|
||||
- package-ecosystem: cargo
|
||||
directory: /v2
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- dependencies
|
||||
- rust
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
docker stop test-container
|
||||
|
||||
- name: Run container security scan
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
format: 'sarif'
|
||||
|
||||
@@ -98,32 +98,6 @@ jobs:
|
||||
echo "Flash image integrity verified"
|
||||
fi
|
||||
|
||||
- name: Verify embedded version string matches version.txt (fixes #505)
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
EXPECTED=$(cat version.txt | tr -d '[:space:]')
|
||||
BIN=build/esp32-csi-node.bin
|
||||
# Extract version from ESP-IDF app_desc: magic 0xABCD5432 at offset 0
|
||||
# followed by version string at offset 16, null-terminated, max 32 chars.
|
||||
EMBEDDED=$(python3 -c "
|
||||
import struct, sys
|
||||
data = open('$BIN','rb').read()
|
||||
magic = struct.pack('<I', 0xABCD5432)
|
||||
i = data.find(magic)
|
||||
if i < 0:
|
||||
sys.exit('app_desc magic not found')
|
||||
ver = data[i+16:i+48].split(b'\\x00',1)[0].decode('ascii','replace')
|
||||
print(ver)
|
||||
" 2>&1)
|
||||
echo "Expected version: $EXPECTED"
|
||||
echo "Embedded version: $EMBEDDED"
|
||||
if [ "$EMBEDDED" != "$EXPECTED" ]; then
|
||||
echo "::error::Version string mismatch! version.txt='$EXPECTED' but binary reports '$EMBEDDED'."
|
||||
echo "::error::Ensure version.txt is updated before building and tagging."
|
||||
exit 1
|
||||
fi
|
||||
echo "Version string verified: $EMBEDDED"
|
||||
|
||||
- name: Stage release binaries with variant-specific names
|
||||
working-directory: firmware/esp32-csi-node
|
||||
run: |
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Point Cloud Viewer → GitHub Pages
|
||||
|
||||
# Publishes the live 3D point cloud viewer to gh-pages/pointcloud/.
|
||||
# The viewer defaults to a synthetic in-browser demo; users can append
|
||||
# ?backend=<url> or ?backend=auto to point it at a real ruview-pointcloud
|
||||
# server (CORS-permitting host required). See ADR-094.
|
||||
#
|
||||
# Uses keep_files: true to preserve the existing observatory/, pose-fusion/,
|
||||
# nvsim/, and root index.html demos already on gh-pages.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'v2/crates/wifi-densepose-pointcloud/src/viewer.html'
|
||||
- '.github/workflows/pointcloud-pages.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: pointcloud-pages
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Stage viewer for Pages
|
||||
run: |
|
||||
mkdir -p _site/pointcloud
|
||||
cp v2/crates/wifi-densepose-pointcloud/src/viewer.html _site/pointcloud/index.html
|
||||
# Drop a tiny README so direct browsers of the directory get context.
|
||||
cat > _site/pointcloud/README.md <<'EOF'
|
||||
# RuView — Live 3D Point Cloud Viewer
|
||||
|
||||
Hosted at: https://ruvnet.github.io/RuView/pointcloud/
|
||||
|
||||
## Modes
|
||||
|
||||
- Default — synthetic in-browser demo (no backend, no network calls).
|
||||
- `?backend=auto` — fetch from `/api/splats` on the same origin
|
||||
(only works when the viewer is served by `ruview-pointcloud serve`).
|
||||
- `?backend=<url>` — fetch from `<url>/api/splats`. The intended
|
||||
local-ESP32 use is `?backend=http://127.0.0.1:9880`: run
|
||||
`ruview-pointcloud serve --bind 127.0.0.1:9880` on the same
|
||||
machine with your ESP32 streaming CSI to UDP port 3333, then
|
||||
visit the URL above. The local server's CorsLayer permits
|
||||
requests from `https://ruvnet.github.io`, and modern browsers
|
||||
permit HTTPS→127.0.0.1 mixed-content as a trustworthy origin.
|
||||
The "📡 Connect ESP32" button in the viewer prompts for this
|
||||
URL and persists it in localStorage.
|
||||
- `?live=1` — require a live backend; show an offline message instead
|
||||
of falling back to the synthetic demo.
|
||||
|
||||
See ADR-094 for the deployment design.
|
||||
EOF
|
||||
|
||||
- name: Deploy to gh-pages/pointcloud/
|
||||
uses: peaceiris/actions-gh-pages@v4
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: ./_site/pointcloud
|
||||
destination_dir: pointcloud
|
||||
# CRITICAL: preserves observatory/, pose-fusion/, nvsim/, and root
|
||||
# index.html already on gh-pages.
|
||||
keep_files: true
|
||||
commit_message: 'deploy(pointcloud): ${{ github.sha }}'
|
||||
user_name: 'github-actions[bot]'
|
||||
user_email: 'github-actions[bot]@users.noreply.github.com'
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Snyk vulnerability scan
|
||||
uses: snyk/actions/python@9adf32b1121593767fc3c057af55b55db032dc04 # v1.0.0
|
||||
uses: snyk/actions/python@master
|
||||
env:
|
||||
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
||||
with:
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: 'wifi-densepose:scan'
|
||||
format: 'sarif'
|
||||
@@ -221,7 +221,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Checkov IaC scan
|
||||
uses: bridgecrewio/checkov-action@99bb2caf247dfd9f03cf984373bc6043d4e32ebf # v12.1347.0
|
||||
uses: bridgecrewio/checkov-action@master
|
||||
with:
|
||||
directory: .
|
||||
framework: kubernetes,dockerfile,terraform,ansible
|
||||
@@ -238,7 +238,7 @@ jobs:
|
||||
category: checkov
|
||||
|
||||
- name: Run Terrascan IaC scan
|
||||
uses: tenable/terrascan-action@3a6e87da8e244513bd77b631e624552643f794c6 # v1.4.1
|
||||
uses: tenable/terrascan-action@main
|
||||
with:
|
||||
iac_type: 'k8s'
|
||||
iac_version: 'v1'
|
||||
@@ -247,7 +247,7 @@ jobs:
|
||||
sarif_upload: true
|
||||
|
||||
- name: Run KICS IaC scan
|
||||
uses: checkmarx/kics-github-action@05aa5eb70eede1355220f4ca5238d96b397e30a6 # v2.1.20
|
||||
uses: checkmarx/kics-github-action@master
|
||||
with:
|
||||
path: '.'
|
||||
output_path: kics-results
|
||||
@@ -277,7 +277,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run TruffleHog secret scan
|
||||
uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
path: ./
|
||||
base: main
|
||||
|
||||
+1
-5
@@ -167,11 +167,7 @@ firing cleanly, HEALTH mesh packets sent.
|
||||
Kconfig surface added under "Adaptive Controller (ADR-081)".
|
||||
|
||||
### Fixed
|
||||
- **Firmware: SPI flash cache crash under high CSI callback pressure** (RuView#396, #397) — ESP32-S3 nodes crashed in `cache_ll_l1_resume_icache` / `wDev_ProcessFiq` after ~2400 callbacks when the promiscuous filter admitted DATA frames at 100–500 Hz. Fixed by narrowing the filter mask to `WIFI_PROMIS_FILTER_MASK_MGMT` (~10 Hz beacons), adding a 50 Hz early callback rate gate (`CSI_MIN_PROCESS_INTERVAL_US`) that drops excess callbacks before any processing work, and enabling `CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y` as defense-in-depth. Stability validated with a 4-min-per-node soak.
|
||||
- **Firmware: `filter_mac` / `node_id` clobber by WiFi driver init** (#232, #375, #385, #386, #390, #397) — `g_nvs_config` can be corrupted during `wifi_init_sta()` on some devices (confirmed on `80:b5:4e:c1:be:b8`), reverting `node_id` to the Kconfig default and producing garbage MAC-filter reads in the CSI callback (100–500 Hz). New `csi_collector_set_node_id()` API called from `app_main()` **before** `wifi_init_sta()` captures both fields into module-local statics (`s_node_id`, `s_filter_mac`, `s_filter_mac_set`). `csi_collector_init()` now runs a canary that distinguishes "early≠g_nvs_config" (corruption confirmed) from a no-op match. All CSI runtime paths use the defensive copies exclusively.
|
||||
- **Firmware: `edge_processing` sample rate mismatch** (#397) — `estimate_bpm_zero_crossing()` was called with a hard-coded `sample_rate = 20.0f`, but MGMT-only promiscuous delivers ~10 Hz. Breathing and heart-rate reports were 2× too high. Corrected to `10.0f` with an explicit comment tying it to the callback rate.
|
||||
- **`provision.py` esptool command form** (#391, #397) — ESP-IDF v5.4 bundles `esptool 4.10.0`, which only accepts `write_flash` (underscore). Standalone `pip install esptool` v5.x accepts both forms but prefers `write-flash`. #391 switched to `write-flash` which broke the documented ESP-IDF Python venv flow; #397 reverts to `write_flash` (works with both esptool 4.x and 5.x) with an inline comment warning future maintainers not to "re-fix" it.
|
||||
- **`provision.py` esptool v5 dry-run hint** (#391) — Stale `write_flash` (underscore) syntax in the dry-run manual-flash hint now uses `write-flash` (hyphenated) for esptool >= 5.x. The primary flash command was already correct.
|
||||
- **`provision.py` esptool v5 compat** (#391) — Stale `write_flash` (underscore) syntax in the dry-run manual-flash hint now uses `write-flash` (hyphenated) for esptool >= 5.x. The primary flash command was already correct.
|
||||
- **`provision.py` silent NVS wipe** (#391) — The script replaces the entire `csi_cfg` NVS namespace on every run, so partial invocations were silently erasing WiFi credentials and causing `Retrying WiFi connection (10/10)` in the field. Now refuses to run without `--ssid`, `--password`, and `--target-ip` unless `--force-partial` is passed. `--force-partial` prints a warning listing which keys will be wiped.
|
||||
- **Firmware: defensive `node_id` capture** (#232, #375, #385, #386, #390) — Users on multi-node deployments reported `node_id` reverting to the Kconfig default (`1`) in UDP frames and in the `csi_collector` init log, despite NVS loading the correct value. The root cause (memory corruption of `g_nvs_config`) has not been definitively isolated, but the UDP frame header is now tamper-proof: `csi_collector_init()` captures `g_nvs_config.node_id` into a module-local `s_node_id` once, and `csi_serialize_frame()` plus all other consumers (`edge_processing.c`, `wasm_runtime.c`, `display_ui.c`, `swarm_bridge_init`) read it via the new `csi_collector_get_node_id()` accessor. A canary logs `WARN` if `g_nvs_config.node_id` diverges from `s_node_id` at end-of-init, helping isolate the upstream corruption path. Validated on attached ESP32-S3 (COM8): NVS `node_id=2` propagates through boot log, capture log, init log, and byte[4] of every UDP frame.
|
||||
|
||||
|
||||
@@ -9,35 +9,26 @@ export class NvSettingsDrawer extends LitElement {
|
||||
@state() private open = false;
|
||||
|
||||
static styles = css`
|
||||
/* The host covers the viewport without transforming itself. Only the
|
||||
* inner .panel is transformed; otherwise the host's transform would
|
||||
* create a containing block for the fixed-position scrim, clipping
|
||||
* it to the panel's 420 px width and breaking outside-to-dismiss. */
|
||||
:host {
|
||||
position: fixed; inset: 0;
|
||||
z-index: 51;
|
||||
pointer-events: none;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
:host([open]) { pointer-events: auto; opacity: 1; }
|
||||
.scrim {
|
||||
position: absolute; inset: 0;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
backdrop-filter: blur(2px);
|
||||
}
|
||||
.panel {
|
||||
position: absolute;
|
||||
top: 0; right: 0; bottom: 0;
|
||||
position: fixed; top: 0; right: 0; bottom: 0;
|
||||
width: 420px; max-width: 100vw;
|
||||
background: var(--bg-1);
|
||||
border-left: 1px solid var(--line);
|
||||
z-index: 51;
|
||||
transform: translateX(100%);
|
||||
transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
display: flex; flex-direction: column;
|
||||
box-shadow: -20px 0 60px -20px rgba(0, 0, 0, 0.5);
|
||||
box-shadow: -20px 0 60px -20px rgba(0,0,0,0.5);
|
||||
}
|
||||
:host([open]) .panel { transform: translateX(0); }
|
||||
:host([open]) { transform: translateX(0); }
|
||||
.scrim {
|
||||
position: fixed; inset: 0;
|
||||
background: rgba(0,0,0,0.5);
|
||||
z-index: 50;
|
||||
opacity: 0; pointer-events: none;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
:host([open]) .scrim { opacity: 1; pointer-events: auto; }
|
||||
.h {
|
||||
padding: 14px 16px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
@@ -132,7 +123,6 @@ export class NvSettingsDrawer extends LitElement {
|
||||
override render() {
|
||||
return html`
|
||||
<div class="scrim" @click=${() => this.close()}></div>
|
||||
<div class="panel" role="dialog" aria-modal="true" aria-label="Settings">
|
||||
<div class="h">
|
||||
<div class="ttl">Settings</div>
|
||||
<button class="close" @click=${() => this.close()}>×</button>
|
||||
@@ -266,7 +256,6 @@ export class NvSettingsDrawer extends LitElement {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
# ADR-094: Live 3D Point Cloud Viewer — GitHub Pages Deployment with Optional Real-Data Stream
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| **Status** | Proposed (2026-04-29) |
|
||||
| **Date** | 2026-04-29 |
|
||||
| **Authors** | ruv |
|
||||
| **Related** | ADR-092 (nvsim dashboard Pages deployment), ADR-059 (live ESP32 CSI pipeline), ADR-079 (camera ground-truth training) |
|
||||
| **Branch** | `feat/pointcloud-pages-demo` |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
The `wifi-densepose-pointcloud` crate ships a Three.js-based viewer
|
||||
(`v2/crates/wifi-densepose-pointcloud/src/viewer.html`) that renders the
|
||||
fused camera-depth + WiFi CSI + mmWave point cloud produced by the
|
||||
`ruview-pointcloud serve` binary. Today the viewer is local-only:
|
||||
|
||||
- It is served by the Axum binary on `127.0.0.1:9880`.
|
||||
- It polls `/api/splats` every 500 ms expecting a backend on the same
|
||||
origin.
|
||||
- There is no GitHub Pages deployment, so the README's
|
||||
"▶ Live 3D Point Cloud" link points at the moved-content section in
|
||||
`docs/readme-details.md`, not at a hosted demo. The two sibling demos
|
||||
(Live Observatory, Dual-Modal Pose Fusion) are already hosted at
|
||||
`https://ruvnet.github.io/RuView/` and `…/pose-fusion.html`.
|
||||
|
||||
This is an asymmetry: a first-time visitor can preview the WiFi pose
|
||||
demo and the Observatory in one click, but cannot preview the point
|
||||
cloud without cloning the repo, building Rust, plugging in an ESP32,
|
||||
and pointing a webcam at themselves. That gap suppresses the most
|
||||
visually compelling demonstration of the v0.7+ sensor-fusion work.
|
||||
|
||||
A naive fix — drop the static HTML at `gh-pages/pointcloud/` — does
|
||||
not work because the viewer's `fetch("/api/splats")` will 404 on Pages
|
||||
and the canvas will hang at "Loading…". A second naive fix — bake in a
|
||||
fixed sample dataset — solves the loading state but loses the live-data
|
||||
story entirely, and forks the viewer into a "demo build" and a "real
|
||||
build" that drift apart.
|
||||
|
||||
## 2. Decision
|
||||
|
||||
Ship **one** viewer that auto-selects its transport from URL parameters,
|
||||
and publish it to `gh-pages/pointcloud/` alongside the other demos:
|
||||
|
||||
1. **Default mode** — when the viewer is opened with no query parameters
|
||||
on `https://ruvnet.github.io/RuView/pointcloud/`, present a "▶ Enable
|
||||
camera" CTA. On click the viewer requests webcam access, runs
|
||||
**MediaPipe Face Mesh** in-browser (~30 fps, 478 refined landmarks),
|
||||
and renders the visitor's own face as a point cloud — the closest
|
||||
browser equivalent of the local pipeline's depth-backprojected face
|
||||
geometry that motivated this ADR (`I could see the outline of my face
|
||||
in points`). The viewer mirrors x to match selfie convention and
|
||||
maps Face Mesh's relative-z to the same world-coordinate range the
|
||||
live `/api/splats` payload uses, so a single render path drives both.
|
||||
Badge reads `● DEMO Your Face (MediaPipe)`. If the user denies
|
||||
camera permission, dismisses the prompt, or visits on a device
|
||||
without a webcam, the viewer falls back automatically to a
|
||||
procedural scaffold (floor grid, walls, breathing figure, 17-keypoint
|
||||
skeleton). All processing is client-side; no frames leave the
|
||||
browser. ~480-500 splats from the face plus ~110 floor/wall context
|
||||
splats.
|
||||
2. **Auto mode** (`?backend=auto`) — fetch from `/api/splats` on the same
|
||||
origin. This is the local-development case (`ruview-pointcloud serve`
|
||||
serves the viewer and the API together). On any failure (404, network
|
||||
error, CORS), fall back silently to synthetic-demo rendering so the
|
||||
tab never dies.
|
||||
3. **Remote mode** (`?backend=<url>`) — fetch from `<url>/api/splats`.
|
||||
This is the **integrated-ESP32** path: the user runs
|
||||
`ruview-pointcloud serve --bind 127.0.0.1:9880` locally with an
|
||||
ESP32-S3 streaming CSI to UDP port 3333, then opens
|
||||
`https://ruvnet.github.io/RuView/pointcloud/?backend=http://127.0.0.1:9880`.
|
||||
The hosted Pages viewer becomes a thin client for the local Rust
|
||||
fusion pipeline (camera depth + WiFi CSI + mmWave) without a clone
|
||||
or rebuild. The viewer also exposes a "📡 Connect ESP32" button that
|
||||
prompts for the URL, persists it in `localStorage`, and reloads
|
||||
with the query param.
|
||||
|
||||
For this to work the local server must answer the browser's CORS
|
||||
preflight. `stream.rs` therefore installs a `tower_http` `CorsLayer`
|
||||
that allows three origin classes:
|
||||
|
||||
- `https://ruvnet.github.io` — the published Pages demo.
|
||||
- `http://localhost:*` and `http://127.0.0.1:*` — developer running
|
||||
the bundled `viewer.html` directly.
|
||||
- `null` — `file://` origins.
|
||||
|
||||
Mixed-content (HTTPS Pages → HTTP loopback) is permitted because
|
||||
modern browsers (Chrome 94+, Firefox 116+, Safari 16.4+) classify
|
||||
`127.0.0.1` and `localhost` as "potentially trustworthy" origins.
|
||||
Any other origin (a public hostname, etc.) is denied — this is not
|
||||
a wildcard CORS posture. Badge reads `● REMOTE <url>`. Same silent
|
||||
demo fallback on failure.
|
||||
4. **Strict-live mode** (`?live=1`) — disable the demo fallback. If the
|
||||
chosen transport fails, replace the info panel with an explicit offline
|
||||
message (`● OFFLINE — Live backend required but unreachable`). Useful
|
||||
for embedding the viewer in a status page or kiosk.
|
||||
|
||||
The synthetic frame returned by the in-browser generator matches the
|
||||
JSON shape of the live `/api/splats` payload exactly (`splats`, `count`,
|
||||
`frame`, `live`, `pipeline.{skeleton,vitals,…}`), so a single render path
|
||||
drives both modes. There is no demo build vs real build — only one HTML
|
||||
file, one render path, and one set of bugs.
|
||||
|
||||
A new GitHub Actions workflow (`.github/workflows/pointcloud-pages.yml`)
|
||||
copies the viewer to `gh-pages/pointcloud/index.html` on every push to
|
||||
`main` that touches the viewer, using `peaceiris/actions-gh-pages@v4`
|
||||
with `keep_files: true` to preserve the existing observatory, pose-fusion,
|
||||
and nvsim deployments.
|
||||
|
||||
## 3. Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- **First-click demo.** Visitors clicking the README's
|
||||
"▶ Live 3D Point Cloud" link land on a working Three.js scene in <1 s,
|
||||
no toolchain required. Matches the parity of the other two demos.
|
||||
- **Real-data on demand.** Users with their own `ruview-pointcloud serve`
|
||||
host can use the same hosted viewer URL with
|
||||
`?backend=https://their-host.example.com` — no clone, no rebuild. The
|
||||
hosted demo doubles as a thin client for self-hosted backends.
|
||||
- **Single render path.** Synthetic frames flow through the same
|
||||
`handleData → updateSplats → drawSkeleton` pipeline as live frames, so
|
||||
visual regressions surface in the demo and the live build at the same
|
||||
time. This is the same dual-transport pattern ADR-092 chose for nvsim.
|
||||
- **No backend deploy required.** Pages serves static HTML; the demo
|
||||
works without standing up an Axum host on the public internet, and
|
||||
there is no per-visitor CSI/camera plumbing to provision.
|
||||
- **Preserves existing deployments.** `keep_files: true` plus the
|
||||
`pointcloud/` destination means observatory/, pose-fusion/, nvsim/,
|
||||
and the root index.html on gh-pages are untouched.
|
||||
|
||||
### Negative / tradeoffs
|
||||
|
||||
- **Face mesh ≠ CSI.** Browser webcam + MediaPipe gives real face
|
||||
geometry but does not produce CSI-derived pose. Visitors who want to
|
||||
see the *WiFi-driven* path still need `?backend=<their-host>`. The
|
||||
procedural fallback is not WiFi-driven either; it is purely visual
|
||||
scaffolding. We accept this — the goal of the hosted demo is to
|
||||
convey the *shape* of what the local pipeline produces (a point
|
||||
cloud of the user) rather than reproduce the WiFi physics in the
|
||||
browser. The latter is a future ADR (WASM port of the fusion crate).
|
||||
- **CORS burden on remote mode.** Users who want to share their backend
|
||||
must add `Access-Control-Allow-Origin: https://ruvnet.github.io` (or
|
||||
`*`) to their `ruview-pointcloud serve` config. We document this in the
|
||||
workflow's generated README; we do **not** add a public proxy.
|
||||
- **Synthetic generator lives in the viewer.** ~80 LOC of procedural JS
|
||||
is now part of `viewer.html`. Acceptable: the file is already the
|
||||
client-side render bundle, and the generator is bounded and inert
|
||||
(deterministic, no I/O, no eval).
|
||||
- **No replay-from-recording in this ADR.** A future ADR may add a
|
||||
`?recording=<url>.jsonl` mode that replays captured frames at native
|
||||
rate; that is out of scope here.
|
||||
|
||||
### Neutral
|
||||
|
||||
- The local-dev experience is unchanged. `ruview-pointcloud serve` still
|
||||
serves `viewer.html` from the bundled asset and the viewer still hits
|
||||
`/api/splats` because `?backend` defaults to `auto`. Nothing in the
|
||||
Rust crate changes — this is HTML + workflow only.
|
||||
|
||||
## 4. Implementation
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `v2/crates/wifi-densepose-pointcloud/src/viewer.html` | Add URL-param transport selector (`backend`, `live`), synthetic frame generator, demo-fallback path, transport-aware mode badge. ~120 LOC added, no removed behavior. |
|
||||
| `.github/workflows/pointcloud-pages.yml` | New workflow: stage viewer to `_site/pointcloud/index.html`, deploy to `gh-pages/pointcloud/` with `keep_files: true`. Triggers on viewer changes and on manual dispatch. |
|
||||
| `README.md` | Already updated — `▶ Live 3D Point Cloud` link will be retargeted to `https://ruvnet.github.io/RuView/pointcloud/` once the first deploy succeeds. (Tracked separately, not blocking this ADR.) |
|
||||
| `docs/adr/README.md` | ADR index — add ADR-094 row. |
|
||||
|
||||
## 5. Acceptance Gates
|
||||
|
||||
This ADR is **Implemented** when all of the following hold:
|
||||
|
||||
1. Pushing to `main` with a viewer change triggers
|
||||
`pointcloud-pages.yml`, which deploys to `gh-pages/pointcloud/` in
|
||||
under 60 seconds.
|
||||
2. `https://ruvnet.github.io/RuView/pointcloud/` loads, shows the
|
||||
"Enable camera" CTA, and on accept renders the visitor's face as a
|
||||
point cloud with badge `● DEMO Your Face (MediaPipe)` and non-zero
|
||||
splat + frame counts. On camera denial, falls back to the
|
||||
procedural scene with badge `● DEMO Synthetic`.
|
||||
3. Existing demos at `https://ruvnet.github.io/RuView/` and
|
||||
`…/pose-fusion.html` and `…/nvsim/` are still reachable after the
|
||||
first deploy (smoke-tested manually).
|
||||
4. `https://ruvnet.github.io/RuView/pointcloud/?live=1` shows the
|
||||
`● OFFLINE` panel (because no same-origin backend exists on Pages).
|
||||
5. `https://ruvnet.github.io/RuView/pointcloud/?backend=https://example.invalid`
|
||||
falls back to demo within one poll interval (~500 ms) without
|
||||
throwing in the console.
|
||||
6. Running `./target/release/ruview-pointcloud serve` locally and
|
||||
opening `http://127.0.0.1:9880/` (which serves the same HTML) still
|
||||
shows live-mode rendering with the `● LIVE Local Backend` badge.
|
||||
|
||||
## 6. Out of Scope
|
||||
|
||||
- Replaying recorded JSONL frames in the browser (future ADR).
|
||||
- WASM-side execution of the fusion pipeline in the browser (would
|
||||
require porting the camera + mmWave path; deferred).
|
||||
- Authentication / signed splats payloads — backend-side concern,
|
||||
unaffected by this client-side change.
|
||||
- Hosting a public CORS proxy for users without their own backend.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -25,20 +25,13 @@
|
||||
/* ADR-060: Access the global NVS config for MAC filter and channel override. */
|
||||
extern nvs_config_t g_nvs_config;
|
||||
|
||||
/* Defensive fix (#232, #375, #385, #386, #390): capture NVS config fields into
|
||||
* module-local statics BEFORE wifi_init_sta() runs, because WiFi driver init
|
||||
* can corrupt g_nvs_config (confirmed on device 80:b5:4e:c1:be:b8).
|
||||
* main.c calls csi_collector_set_node_id() immediately after nvs_config_load(),
|
||||
* and all runtime paths use the local copies exclusively. */
|
||||
/* Defensive fix (#232, #375, #385, #386, #390): capture node_id at init-time
|
||||
* into a module-local static. Using the global g_nvs_config.node_id directly
|
||||
* at every callback is vulnerable to any memory corruption that clobbers the
|
||||
* struct (which users have reported reverting node_id to the Kconfig default
|
||||
* of 1). The local copy is set once at csi_collector_init() and then used
|
||||
* exclusively by csi_serialize_frame(). */
|
||||
static uint8_t s_node_id = 1;
|
||||
static bool s_node_id_early_set = false;
|
||||
|
||||
/* Defensive copy of MAC filter config — the CSI callback fires at 100-500 Hz
|
||||
* and reads filter_mac_set + filter_mac on every invocation. If wifi_init_sta()
|
||||
* corrupts g_nvs_config, the callback would read garbage, potentially causing
|
||||
* LoadProhibited panics (observed: Core 0 panic after ~2400 callbacks). */
|
||||
static uint8_t s_filter_mac[6] = {0};
|
||||
static bool s_filter_mac_set = false;
|
||||
|
||||
/* ADR-057: Build-time guard — fail early if CSI is not enabled in sdkconfig.
|
||||
* Without this, the firmware compiles but crashes at runtime with:
|
||||
@@ -67,24 +60,6 @@ static uint32_t s_rate_skip = 0;
|
||||
#define CSI_MIN_SEND_INTERVAL_US (20 * 1000)
|
||||
static int64_t s_last_send_us = 0;
|
||||
|
||||
/**
|
||||
* Minimum interval between processing ANY CSI callback in microseconds.
|
||||
* Promiscuous MGMT+DATA can fire 100-500+ times/sec. At rates above ~50 Hz,
|
||||
* the WiFi FIQ handler (wDev_ProcessFiq) races with SPI flash cache operations,
|
||||
* causing Core 0 LoadProhibited panics in cache_ll_l1_resume_icache.
|
||||
*
|
||||
* This early gate drops excess callbacks BEFORE any processing (serialization,
|
||||
* UDP, edge enqueue), keeping the effective callback rate at ~50 Hz while
|
||||
* preserving the full MGMT+DATA promiscuous filter and HT-LTF/STBC CSI quality.
|
||||
*
|
||||
* The WiFi hardware still captures all frames and the CSI data is generated,
|
||||
* but we simply discard the excess in software. This reduces the time spent
|
||||
* in callback context per second, giving the WiFi ISR more headroom.
|
||||
*/
|
||||
#define CSI_MIN_PROCESS_INTERVAL_US (20 * 1000) /* 50 Hz */
|
||||
static int64_t s_last_process_us = 0;
|
||||
static uint32_t s_early_drop = 0;
|
||||
|
||||
/* ---- ADR-029: Channel-hop state ---- */
|
||||
|
||||
/** Channel hop table (populated from NVS at boot or via set_hop_table). */
|
||||
@@ -190,20 +165,9 @@ static void wifi_csi_callback(void *ctx, wifi_csi_info_t *info)
|
||||
{
|
||||
(void)ctx;
|
||||
|
||||
/* Early rate gate: drop excess callbacks to ~50 Hz to prevent
|
||||
* SPI flash cache crash in WiFi ISR (wDev_ProcessFiq). */
|
||||
int64_t now_us = esp_timer_get_time();
|
||||
if ((now_us - s_last_process_us) < CSI_MIN_PROCESS_INTERVAL_US) {
|
||||
s_early_drop++;
|
||||
return;
|
||||
}
|
||||
s_last_process_us = now_us;
|
||||
|
||||
/* ADR-060: MAC address filtering — drop frames from non-matching sources.
|
||||
* Uses defensively-copied s_filter_mac instead of g_nvs_config (which can
|
||||
* be corrupted by wifi_init_sta — same root cause as the node_id clobber). */
|
||||
if (s_filter_mac_set) {
|
||||
if (memcmp(info->mac, s_filter_mac, 6) != 0) {
|
||||
/* ADR-060: MAC address filtering — drop frames from non-matching sources. */
|
||||
if (g_nvs_config.filter_mac_set) {
|
||||
if (memcmp(info->mac, g_nvs_config.filter_mac, 6) != 0) {
|
||||
return; /* Source MAC doesn't match filter — skip frame. */
|
||||
}
|
||||
}
|
||||
@@ -258,60 +222,14 @@ static void wifi_promiscuous_cb(void *buf, wifi_promiscuous_pkt_type_t type)
|
||||
(void)type;
|
||||
}
|
||||
|
||||
void csi_collector_set_node_id(uint8_t node_id)
|
||||
{
|
||||
s_node_id = node_id;
|
||||
s_node_id_early_set = true;
|
||||
ESP_LOGI(TAG, "Early capture node_id=%u (before WiFi init, #232/#390)",
|
||||
(unsigned)node_id);
|
||||
|
||||
/* Also capture MAC filter config now — same struct, same corruption risk.
|
||||
* The CSI callback reads filter_mac_set on every invocation (100-500 Hz),
|
||||
* so a corrupted value could cause erratic filtering or crash. */
|
||||
s_filter_mac_set = (g_nvs_config.filter_mac_set != 0);
|
||||
if (s_filter_mac_set) {
|
||||
memcpy(s_filter_mac, g_nvs_config.filter_mac, 6);
|
||||
ESP_LOGI(TAG, "Early capture filter_mac=%02x:%02x:%02x:%02x:%02x:%02x",
|
||||
s_filter_mac[0], s_filter_mac[1], s_filter_mac[2],
|
||||
s_filter_mac[3], s_filter_mac[4], s_filter_mac[5]);
|
||||
}
|
||||
}
|
||||
|
||||
void csi_collector_init(void)
|
||||
{
|
||||
if (!s_node_id_early_set) {
|
||||
/* Fallback: no early capture — use current g_nvs_config (may be clobbered). */
|
||||
s_node_id = g_nvs_config.node_id;
|
||||
ESP_LOGW(TAG, "Late capture node_id=%u (no early set_node_id call)",
|
||||
(unsigned)s_node_id);
|
||||
} else if (g_nvs_config.node_id != s_node_id) {
|
||||
/* Canary: early capture disagrees with current g_nvs_config — corruption
|
||||
* happened between nvs_config_load() and here (likely wifi_init_sta). */
|
||||
ESP_LOGW(TAG, "node_id clobber CONFIRMED: early=%u g_nvs_config=%u "
|
||||
"(WiFi init likely corrupted struct, using early value)",
|
||||
(unsigned)s_node_id, (unsigned)g_nvs_config.node_id);
|
||||
} else {
|
||||
ESP_LOGI(TAG, "node_id=%u verified (early capture matches g_nvs_config)",
|
||||
(unsigned)s_node_id);
|
||||
}
|
||||
|
||||
/* Canary for filter_mac: check if WiFi init corrupted the filter fields. */
|
||||
if (s_node_id_early_set) {
|
||||
bool mac_set_now = (g_nvs_config.filter_mac_set != 0);
|
||||
if (mac_set_now != s_filter_mac_set) {
|
||||
ESP_LOGW(TAG, "filter_mac_set clobber CONFIRMED: early=%d g_nvs_config=%d",
|
||||
(int)s_filter_mac_set, (int)mac_set_now);
|
||||
} else if (s_filter_mac_set &&
|
||||
memcmp(s_filter_mac, g_nvs_config.filter_mac, 6) != 0) {
|
||||
ESP_LOGW(TAG, "filter_mac clobber CONFIRMED: bytes differ after WiFi init");
|
||||
}
|
||||
} else {
|
||||
/* No early capture — grab filter config now (may already be corrupted). */
|
||||
s_filter_mac_set = (g_nvs_config.filter_mac_set != 0);
|
||||
if (s_filter_mac_set) {
|
||||
memcpy(s_filter_mac, g_nvs_config.filter_mac, 6);
|
||||
}
|
||||
}
|
||||
/* Capture node_id into module-local static at init time. After this point
|
||||
* csi_serialize_frame() uses s_node_id exclusively, isolating the UDP
|
||||
* frame node_id field from any memory corruption of g_nvs_config. */
|
||||
s_node_id = g_nvs_config.node_id;
|
||||
ESP_LOGI(TAG, "Captured node_id=%u at init (defensive copy for #232/#375/#385/#390)",
|
||||
(unsigned)s_node_id);
|
||||
|
||||
/* ADR-060: Determine the CSI channel.
|
||||
* Priority: 1) NVS override (--channel), 2) connected AP channel, 3) Kconfig default. */
|
||||
@@ -342,19 +260,12 @@ void csi_collector_init(void)
|
||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous(true));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_rx_cb(wifi_promiscuous_cb));
|
||||
|
||||
/* MGMT-only promiscuous filter + active probe injection (RuView#396).
|
||||
*
|
||||
* DATA frames cause 100-500+ WiFi HW interrupts/sec which crashes Core 0
|
||||
* in wDev_ProcessFiq (SPI flash cache race in ESP-IDF WiFi blob).
|
||||
* MGMT-only gives ~10 Hz (beacons). Probe request injection at 10 Hz
|
||||
* adds ~10 Hz probe responses from APs → ~20 Hz total, matching the
|
||||
* edge processing designed sample rate of 20 Hz. */
|
||||
wifi_promiscuous_filter_t filt = {
|
||||
.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT,
|
||||
.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT | WIFI_PROMIS_FILTER_MASK_DATA,
|
||||
};
|
||||
ESP_ERROR_CHECK(esp_wifi_set_promiscuous_filter(&filt));
|
||||
|
||||
ESP_LOGI(TAG, "Promiscuous mode enabled (MGMT-only, RuView#396)");
|
||||
ESP_LOGI(TAG, "Promiscuous mode enabled for CSI capture");
|
||||
|
||||
wifi_csi_config_t csi_config = {
|
||||
.lltf_en = true,
|
||||
@@ -379,6 +290,16 @@ void csi_collector_init(void)
|
||||
|
||||
ESP_LOGI(TAG, "CSI collection initialized (node_id=%u, channel=%u)",
|
||||
(unsigned)s_node_id, (unsigned)csi_channel);
|
||||
|
||||
/* Clobber-detection canary: if g_nvs_config.node_id no longer matches the
|
||||
* value we captured, something corrupted the struct between nvs_config_load
|
||||
* and here. This is the historic #232/#375 symptom. */
|
||||
if (g_nvs_config.node_id != s_node_id) {
|
||||
ESP_LOGW(TAG, "node_id clobber detected: captured=%u but g_nvs_config=%u "
|
||||
"(frames will use captured value %u). Please report to #390.",
|
||||
(unsigned)s_node_id, (unsigned)g_nvs_config.node_id,
|
||||
(unsigned)s_node_id);
|
||||
}
|
||||
}
|
||||
|
||||
/* Accessor for other modules that need the authoritative runtime node_id. */
|
||||
|
||||
@@ -30,24 +30,14 @@
|
||||
void csi_collector_init(void);
|
||||
|
||||
/**
|
||||
* Capture node_id BEFORE wifi_init_sta() or any other heavy init.
|
||||
* Get the runtime node_id captured at csi_collector_init().
|
||||
*
|
||||
* Must be called from app_main() immediately after nvs_config_load().
|
||||
* WiFi driver initialization can corrupt g_nvs_config.node_id (confirmed
|
||||
* on device 80:b5:4e:c1:be:b8, NVS=3 but post-WiFi reads as 1).
|
||||
* This early capture shields s_node_id from that corruption window.
|
||||
* This is a defensive copy of g_nvs_config.node_id taken at init time. Other
|
||||
* modules (edge_processing, wasm_runtime, display_ui) should prefer this
|
||||
* accessor over reading g_nvs_config.node_id directly, because the global
|
||||
* struct can be clobbered by memory corruption (see #232, #375, #385, #390).
|
||||
*
|
||||
* @param node_id Value from g_nvs_config.node_id, read right after NVS load.
|
||||
*/
|
||||
void csi_collector_set_node_id(uint8_t node_id);
|
||||
|
||||
/**
|
||||
* Get the runtime node_id (early capture if available, otherwise init-time).
|
||||
*
|
||||
* Other modules (edge_processing, wasm_runtime, display_ui) should prefer
|
||||
* this accessor over reading g_nvs_config.node_id directly.
|
||||
*
|
||||
* @return Node ID (0-255) as loaded from NVS at boot.
|
||||
* @return Node ID (0-255) as loaded from NVS or Kconfig default at boot.
|
||||
*/
|
||||
uint8_t csi_collector_get_node_id(void);
|
||||
|
||||
|
||||
@@ -714,11 +714,8 @@ static void process_frame(const edge_ring_slot_t *slot)
|
||||
s_frame_count++;
|
||||
s_latest_rssi = slot->rssi;
|
||||
|
||||
/* CSI sample rate. MGMT-only promiscuous filter (RuView#396, csi_collector.c)
|
||||
* yields ~10 Hz from beacons; keep this value aligned with csi_collector's
|
||||
* effective callback rate or estimate_bpm_zero_crossing() reports the wrong
|
||||
* BPM (2× rate mismatch → 2× wrong breathing/HR). */
|
||||
const float sample_rate = 10.0f;
|
||||
/* Assumed CSI sample rate (~20 Hz for typical ESP32 CSI). */
|
||||
const float sample_rate = 20.0f;
|
||||
|
||||
/* --- Step 1-2: Phase extraction + unwrapping per subcarrier --- */
|
||||
float phases[EDGE_MAX_SUBCARRIERS];
|
||||
|
||||
@@ -140,11 +140,6 @@ void app_main(void)
|
||||
/* Load runtime config (NVS overrides Kconfig defaults) */
|
||||
nvs_config_load(&g_nvs_config);
|
||||
|
||||
/* Capture node_id IMMEDIATELY — before wifi_init_sta() can corrupt
|
||||
* g_nvs_config. See #232/#375/#390: WiFi driver init clobbers the struct
|
||||
* on some devices, reverting node_id to the Kconfig default of 1. */
|
||||
csi_collector_set_node_id(g_nvs_config.node_id);
|
||||
|
||||
const esp_app_desc_t *app_desc = esp_app_get_description();
|
||||
ESP_LOGI(TAG, "ESP32-S3 CSI Node (ADR-018) — v%s — Node ID: %d",
|
||||
app_desc->version, g_nvs_config.node_id);
|
||||
|
||||
@@ -155,10 +155,7 @@ def flash_nvs(port, baud, nvs_bin):
|
||||
"--chip", "esp32s3",
|
||||
"--port", port,
|
||||
"--baud", str(baud),
|
||||
# Keep underscore form — ESP-IDF v5.4 bundles esptool 4.10.0 which only
|
||||
# accepts "write_flash". pip's esptool >=5.x accepts both (hyphenated
|
||||
# form preferred) but keeps underscore working. Do not "correct" this.
|
||||
"write_flash",
|
||||
"write-flash",
|
||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||
]
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
||||
|
||||
@@ -32,5 +32,6 @@ CONFIG_LWIP_SO_RCVBUF=y
|
||||
# FreeRTOS: increase task stack for CSI processing
|
||||
CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
|
||||
|
||||
# Extra WiFi IRAM placement (defense-in-depth for RuView#396 SPI cache race)
|
||||
CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y
|
||||
# ADR-081: adaptive_controller runs emit_feature_state + stream_sender
|
||||
# network I/O inside Timer Svc callbacks, exceeding the 2 KiB default.
|
||||
CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=8192
|
||||
|
||||
@@ -1 +1 @@
|
||||
0.6.4
|
||||
0.6.2
|
||||
|
||||
Generated
+46
-10
@@ -5127,9 +5127,9 @@
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/@xmldom/xmldom": {
|
||||
"version": "0.8.13",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
|
||||
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
|
||||
"version": "0.8.11",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
|
||||
"integrity": "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
@@ -5310,6 +5310,18 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/anymatch/node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
||||
@@ -11923,6 +11935,18 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/jest-util/node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/jest-validate": {
|
||||
"version": "29.7.0",
|
||||
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
|
||||
@@ -13365,6 +13389,18 @@
|
||||
"node": ">=8.6"
|
||||
}
|
||||
},
|
||||
"node_modules/micromatch/node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/mime": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||
@@ -13558,9 +13594,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/node-forge": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz",
|
||||
"integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==",
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz",
|
||||
"integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==",
|
||||
"license": "(BSD-3-Clause OR GPL-2.0)",
|
||||
"engines": {
|
||||
"node": ">= 6.13.0"
|
||||
@@ -14020,12 +14056,12 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/picomatch": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
|
||||
"integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
|
||||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
|
||||
@@ -49,10 +49,5 @@
|
||||
"react-native-worklets": "^0.7.4",
|
||||
"typescript": "~5.9.2"
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
"node-forge": "^1.4.0",
|
||||
"picomatch": "^2.3.2"
|
||||
},
|
||||
"private": true
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ serde_json = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
chrono = "0.4"
|
||||
dirs = "5"
|
||||
|
||||
@@ -65,9 +65,6 @@ pub struct CsiPipelineState {
|
||||
pub current_location: Option<(String, f32)>,
|
||||
/// Night mode — true when camera luminance is below threshold
|
||||
pub is_dark: bool,
|
||||
/// Wall-clock instant the last real ESP32 UDP CSI frame was received.
|
||||
/// `None` if no frame has arrived since startup.
|
||||
pub last_csi_received: Option<std::time::Instant>,
|
||||
/// Metadata from the on-disk WiFlow JSON, if one is present. NOTE: the
|
||||
/// weights themselves are NOT loaded or executed in this crate — this
|
||||
/// flag merely enables the amplitude-energy heuristic pose code path.
|
||||
@@ -94,7 +91,6 @@ impl Default for CsiPipelineState {
|
||||
fingerprints: Vec::new(),
|
||||
current_location: None,
|
||||
is_dark: false,
|
||||
last_csi_received: None,
|
||||
pose_model_present: detect_pose_model_metadata(),
|
||||
}
|
||||
}
|
||||
@@ -137,7 +133,6 @@ impl CsiPipelineState {
|
||||
pub fn process_frame(&mut self, frame: CsiFrame) {
|
||||
let node_id = frame.node_id;
|
||||
self.total_frames += 1;
|
||||
self.last_csi_received = Some(std::time::Instant::now());
|
||||
|
||||
// Once every 500 frames log a one-line node stats summary. This keeps
|
||||
// us honest about the CSI shape we are actually receiving and also
|
||||
@@ -589,9 +584,6 @@ pub fn get_pipeline_output(state: &Arc<Mutex<CsiPipelineState>>) -> PipelineOutp
|
||||
num_nodes: st.node_frames.len(),
|
||||
current_location: st.current_location.clone(),
|
||||
is_dark: st.is_dark,
|
||||
csi_live: st.last_csi_received
|
||||
.map(|t| t.elapsed() < std::time::Duration::from_secs(5))
|
||||
.unwrap_or(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -606,10 +598,6 @@ pub struct PipelineOutput {
|
||||
pub num_nodes: usize,
|
||||
pub current_location: Option<(String, f32)>,
|
||||
pub is_dark: bool,
|
||||
/// True when a real ESP32 CSI frame was received in the last 5 seconds.
|
||||
/// False means the pipeline is running on stale data — show a NO SIGNAL
|
||||
/// indicator in the UI rather than presenting stale skeletons as live.
|
||||
pub csi_live: bool,
|
||||
}
|
||||
|
||||
// Serialize implementations
|
||||
|
||||
@@ -8,13 +8,11 @@ use crate::fusion;
|
||||
use crate::pointcloud;
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::{HeaderValue, Method},
|
||||
response::Html,
|
||||
routing::get,
|
||||
Json, Router,
|
||||
};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||
|
||||
struct AppState {
|
||||
latest_cloud: Mutex<pointcloud::PointCloud>,
|
||||
@@ -110,36 +108,12 @@ pub async fn serve(bind: &str, _brain: Option<&str>) -> anyhow::Result<()> {
|
||||
if has_camera { eprintln!(" Camera: LIVE (/dev/video0)"); }
|
||||
else { eprintln!(" Camera: DEMO"); }
|
||||
|
||||
// CORS — allow the hosted GitHub Pages viewer to fetch /api/splats from a
|
||||
// locally-running instance of this server. Modern browsers treat
|
||||
// 127.0.0.1/localhost as a "potentially trustworthy" origin so the HTTPS
|
||||
// page can reach a plain-HTTP loopback backend without mixed-content
|
||||
// blocking. Origins permitted:
|
||||
// - https://ruvnet.github.io (the published RuView Pages demo)
|
||||
// - http://localhost:* / http://127.0.0.1:* (developer running the
|
||||
// viewer.html bundled with this binary)
|
||||
// Anything else is denied, so this is not a "wildcard" CORS.
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(AllowOrigin::predicate(|origin: &HeaderValue, _req| {
|
||||
let s = match origin.to_str() {
|
||||
Ok(v) => v,
|
||||
Err(_) => return false,
|
||||
};
|
||||
s == "https://ruvnet.github.io"
|
||||
|| s.starts_with("http://localhost")
|
||||
|| s.starts_with("http://127.0.0.1")
|
||||
|| s == "null" // file:// origins
|
||||
}))
|
||||
.allow_methods([Method::GET, Method::OPTIONS])
|
||||
.allow_headers([axum::http::header::CONTENT_TYPE]);
|
||||
|
||||
let app = Router::new()
|
||||
.route("/", get(index))
|
||||
.route("/api/cloud", get(api_cloud))
|
||||
.route("/api/splats", get(api_splats))
|
||||
.route("/api/status", get(api_status))
|
||||
.route("/health", get(api_health))
|
||||
.layer(cors)
|
||||
.with_state(state);
|
||||
|
||||
println!("╔══════════════════════════════════════════════╗");
|
||||
@@ -219,12 +193,10 @@ async fn api_splats(State(state): State<Arc<AppState>>) -> Json<serde_json::Valu
|
||||
let splats = state.latest_splats.lock().unwrap();
|
||||
let frames = *state.frame_count.lock().unwrap();
|
||||
let pipeline = state.latest_pipeline.lock().unwrap();
|
||||
let csi_live = pipeline.as_ref().map(|p| p.csi_live).unwrap_or(false);
|
||||
Json(serde_json::json!({
|
||||
"splats": &*splats,
|
||||
"count": splats.len(),
|
||||
"live": state.use_camera,
|
||||
"csi_live": csi_live,
|
||||
"frame": frames,
|
||||
"pipeline": &*pipeline,
|
||||
"timestamp": chrono::Utc::now().timestamp_millis(),
|
||||
|
||||
@@ -2,60 +2,27 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>RuView — Camera + WiFi CSI Point Cloud</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate">
|
||||
<meta name="ruview-viewer-version" content="0.2.0-face-mesh">
|
||||
<!-- Inline amber-dot favicon avoids a stray /favicon.ico 404 in the console. -->
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
body { margin: 0; background: #0a0a0a; color: #e8a634; font-family: monospace; }
|
||||
canvas { display: block; }
|
||||
#info { position: absolute; top: 10px; left: 10px; padding: 12px; background: rgba(0,0,0,0.85); border: 1px solid #e8a634; border-radius: 6px; min-width: 240px; font-size: 13px; line-height: 1.5; z-index: 10; }
|
||||
#cam-cta { position: absolute; bottom: 16px; left: 50%; transform: translateX(-50%); padding: 10px 18px; background: #e8a634; color: #0a0a0a; border: none; border-radius: 4px; font-family: monospace; font-size: 14px; font-weight: bold; cursor: pointer; z-index: 10; }
|
||||
#cam-cta:hover { background: #ffc04d; }
|
||||
#cam-cta.hidden { display: none; }
|
||||
#esp-cta { position: absolute; bottom: 16px; right: 16px; padding: 8px 14px; background: transparent; color: #e8a634; border: 1px solid #e8a634; border-radius: 4px; font-family: monospace; font-size: 12px; cursor: pointer; z-index: 10; }
|
||||
#esp-cta:hover { background: rgba(232, 166, 52, 0.12); }
|
||||
#esp-cta.connected { background: #4f4; color: #0a0a0a; border-color: #4f4; }
|
||||
#info { position: absolute; top: 10px; left: 10px; padding: 12px; background: rgba(0,0,0,0.85); border: 1px solid #e8a634; border-radius: 6px; min-width: 240px; font-size: 13px; line-height: 1.5; }
|
||||
.live { color: #4f4; } .demo { color: #f44; }
|
||||
.face { color: #4cf; }
|
||||
.section { margin-top: 6px; padding-top: 6px; border-top: 1px solid #333; }
|
||||
.label { color: #888; }
|
||||
#no-signal {
|
||||
display: none;
|
||||
position: absolute; top: 50%; left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
background: rgba(160,0,0,0.93); color: #fff;
|
||||
font-family: monospace; font-size: 18px; font-weight: bold;
|
||||
padding: 18px 32px; border-radius: 8px;
|
||||
border: 2px solid #f44; text-align: center;
|
||||
pointer-events: none; z-index: 20;
|
||||
}
|
||||
#no-signal .sub { font-size: 12px; font-weight: normal; margin-top: 6px; color: #fbb; }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
<!-- MediaPipe Face Mesh — runs in demo mode so each visitor sees their own face as a point cloud -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4/face_mesh.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils@0.3/camera_utils.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="no-signal">
|
||||
● NO CSI SIGNAL
|
||||
<div class="sub">No ESP32 frames received for >5s.<br>Check that your node is powered and provisioned.</div>
|
||||
</div>
|
||||
<div id="info">
|
||||
<h3 style="margin:0 0 4px 0">RuView · Seldon Vault</h3>
|
||||
<div style="font-size: 11px; color: #888; margin-bottom: 8px; max-width: 240px; line-height: 1.4; font-style: italic;">"Psychohistory deals with reactions of human conglomerates to fixed social and economic stimuli." — Hari Seldon</div>
|
||||
<h3 style="margin:0 0 8px 0">RuView Point Cloud</h3>
|
||||
<div id="stats">Loading...</div>
|
||||
</div>
|
||||
<button id="cam-cta">▶ Project Subject — render your face into the Vault</button>
|
||||
<button id="esp-cta" title="Stream live CSI from a local ruview-pointcloud serve instance (e.g. http://127.0.0.1:9880)">📡 Connect ESP32…</button>
|
||||
<script>
|
||||
var scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x0a0a0a);
|
||||
var camera = new THREE.PerspectiveCamera(72, window.innerWidth/window.innerHeight, 0.1, 200);
|
||||
camera.position.set(0, 0.2, -3.5);
|
||||
var camera = new THREE.PerspectiveCamera(75, window.innerWidth/window.innerHeight, 0.1, 100);
|
||||
camera.position.set(0, 2, -4);
|
||||
camera.lookAt(0, 0, 2);
|
||||
|
||||
var renderer = new THREE.WebGLRenderer({ antialias: true });
|
||||
@@ -71,11 +38,6 @@
|
||||
var skeletonGroup = null;
|
||||
var prevTimestamp = 0;
|
||||
var frameRateVal = 0;
|
||||
// No-signal detection: track server-reported csi_live flag
|
||||
var noSignalBanner = document.getElementById("no-signal");
|
||||
function setNoSignal(isNoSignal) {
|
||||
noSignalBanner.style.display = isNoSignal ? "block" : "none";
|
||||
}
|
||||
|
||||
// COCO skeleton connections: pairs of keypoint indices
|
||||
// 0=nose 1=leftEye 2=rightEye 3=leftEar 4=rightEar
|
||||
@@ -142,436 +104,10 @@
|
||||
scene.add(skeletonGroup);
|
||||
}
|
||||
|
||||
// ----- Transport configuration -----
|
||||
// ?backend=<url> → fetch splats from <url>/api/splats (CORS-permitting host)
|
||||
// ?backend=auto → try /api/splats, fall back to synthetic demo on failure (default)
|
||||
// ?backend=demo → always render synthetic demo (no network)
|
||||
// ?live=1 → require live; show error instead of demo fallback
|
||||
var urlParams = new URLSearchParams(window.location.search);
|
||||
var backendArg = urlParams.get("backend") || "auto";
|
||||
var requireLive = urlParams.get("live") === "1";
|
||||
var transportMode = "demo"; // resolved at first fetch: "live" | "remote" | "demo"
|
||||
var demoStartMs = Date.now();
|
||||
var demoFrameNum = 0;
|
||||
var latestFaceLandmarks = null; // populated by MediaPipe when camera enabled
|
||||
var faceMeshState = "idle"; // "idle" | "starting" | "running" | "denied" | "unavailable"
|
||||
|
||||
// ----- MediaPipe Face Mesh (browser equivalent of camera-depth backprojection) -----
|
||||
// Locally, ruview-pointcloud serve fuses real camera depth + WiFi CSI. In the
|
||||
// browser we don't have depth from a webcam, but Face Mesh produces 468
|
||||
// 3D landmarks (x,y in [0,1], z roughly in [-0.5,0.5]) at ~30 fps — enough to
|
||||
// reproduce the "I can see the outline of my face in points" experience. The
|
||||
// landmarks feed into the same splat render path as live /api/splats data.
|
||||
async function startFaceMesh() {
|
||||
if (faceMeshState !== "idle") return;
|
||||
if (!window.FaceMesh || !window.Camera) {
|
||||
faceMeshState = "unavailable";
|
||||
return;
|
||||
}
|
||||
faceMeshState = "starting";
|
||||
try {
|
||||
var videoEl = document.createElement("video");
|
||||
videoEl.style.display = "none";
|
||||
videoEl.autoplay = true;
|
||||
videoEl.playsInline = true;
|
||||
videoEl.muted = true;
|
||||
document.body.appendChild(videoEl);
|
||||
|
||||
var fm = new FaceMesh({
|
||||
locateFile: function(file) {
|
||||
return "https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4/" + file;
|
||||
}
|
||||
});
|
||||
fm.setOptions({
|
||||
maxNumFaces: 1,
|
||||
refineLandmarks: true,
|
||||
minDetectionConfidence: 0.5,
|
||||
minTrackingConfidence: 0.5
|
||||
});
|
||||
fm.onResults(function(results) {
|
||||
if (results.multiFaceLandmarks && results.multiFaceLandmarks[0]) {
|
||||
latestFaceLandmarks = results.multiFaceLandmarks[0];
|
||||
}
|
||||
});
|
||||
|
||||
var mpCamera = new Camera(videoEl, {
|
||||
onFrame: async function() { await fm.send({ image: videoEl }); },
|
||||
width: 640,
|
||||
height: 480
|
||||
});
|
||||
await mpCamera.start();
|
||||
faceMeshState = "running";
|
||||
var btn = document.getElementById("cam-cta");
|
||||
if (btn) btn.classList.add("hidden");
|
||||
} catch (err) {
|
||||
faceMeshState = "denied";
|
||||
console.warn("Face mesh unavailable:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Foundation-inspired galactic context (Asimov / Trantor / Seldon) ----
|
||||
// Shared between face-mesh and synthetic-fallback paths. The subject (face
|
||||
// or procedural figure) is the foreground; this function paints the Seldon
|
||||
// time-vault around it: holographic surveyor grid underfoot, slow galactic
|
||||
// spiral receding into the distance, distant starfield, and a halo ring.
|
||||
function pushFoundationContext(splats) {
|
||||
var t = (Date.now() - demoStartMs) / 1000.0;
|
||||
|
||||
// 1. Holographic surveyor grid — amber lattice at y=+1.4 (renders below
|
||||
// the subject because the renderer flips y to Three.js Y-up).
|
||||
var gx, gz;
|
||||
for (gx = -10; gx <= 10; gx++) {
|
||||
for (gz = 0; gz <= 30; gz++) {
|
||||
var alpha = 0.35 + 0.15 * Math.sin(t * 0.5 + gz * 0.2);
|
||||
splats.push({
|
||||
center: [gx * 0.5, 1.4, gz * 0.4],
|
||||
color: [0.40 * alpha, 0.28 * alpha, 0.10 * alpha],
|
||||
opacity: 1.0,
|
||||
scale: [0.018, 0.018, 0.018]
|
||||
});
|
||||
}
|
||||
}
|
||||
for (gz = 0; gz <= 30; gz += 2) {
|
||||
for (gx = -20; gx <= 20; gx++) {
|
||||
splats.push({
|
||||
center: [gx * 0.25, 1.4, gz * 0.4 + 0.1],
|
||||
color: [0.30, 0.22, 0.08],
|
||||
opacity: 1.0,
|
||||
scale: [0.014, 0.014, 0.014]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Galactic spiral — Trantor recedes behind the subject. ~640 stars
|
||||
// across two logarithmic arms, slowly rotating. Warmer at the core,
|
||||
// cooler at the edges (Hertzsprung-Russell-ish).
|
||||
var arm, k, theta_arm, r_arm, sx, sz, twist, arm_color;
|
||||
for (arm = 0; arm < 2; arm++) {
|
||||
for (k = 0; k < 320; k++) {
|
||||
var prog = k / 320;
|
||||
theta_arm = arm * Math.PI + prog * 6.0 + t * 0.05;
|
||||
r_arm = 4.0 + prog * 14.0;
|
||||
twist = Math.sin(prog * 8.0) * 0.4;
|
||||
sx = Math.cos(theta_arm) * r_arm + twist;
|
||||
sz = Math.sin(theta_arm) * r_arm + 12.0;
|
||||
var coreFade = Math.max(0.15, 1.0 - prog);
|
||||
arm_color = [
|
||||
coreFade * 0.85 + 0.15 * (1 - prog),
|
||||
coreFade * 0.70 + 0.20,
|
||||
coreFade * 0.55 + 0.45 * prog
|
||||
];
|
||||
splats.push({
|
||||
center: [sx, -2.5 + Math.sin(prog * 12) * 0.3, sz],
|
||||
color: arm_color,
|
||||
opacity: 1.0,
|
||||
scale: [0.025, 0.025, 0.025]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Distant starfield — 800 deterministic stars on a spherical shell.
|
||||
// Fixed LCG seed so visitors don't see noise flicker between frames.
|
||||
var seed = 42;
|
||||
function nextRand() {
|
||||
seed = (seed * 1664525 + 1013904223) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
var s, r_s, phi, costheta, sinphi;
|
||||
for (s = 0; s < 800; s++) {
|
||||
phi = nextRand() * Math.PI * 2;
|
||||
costheta = nextRand() * 2 - 1;
|
||||
sinphi = Math.sqrt(1 - costheta * costheta);
|
||||
r_s = 22 + nextRand() * 8;
|
||||
var brightness = 0.4 + nextRand() * 0.6;
|
||||
var hue = nextRand();
|
||||
splats.push({
|
||||
center: [
|
||||
Math.cos(phi) * sinphi * r_s,
|
||||
costheta * r_s * 0.5 - 1.0,
|
||||
Math.sin(phi) * sinphi * r_s + 5.0
|
||||
],
|
||||
color: hue > 0.85
|
||||
? [brightness, brightness * 0.85, brightness * 0.6]
|
||||
: (hue > 0.3
|
||||
? [brightness * 0.9, brightness * 0.95, brightness]
|
||||
: [brightness * 0.5, brightness * 0.7, brightness]),
|
||||
opacity: 1.0,
|
||||
scale: [0.020, 0.020, 0.020]
|
||||
});
|
||||
}
|
||||
|
||||
// 4. Holographic projection halo around the subject — Seldon vault
|
||||
// projections always had a faint encircling ring of particles.
|
||||
var ring;
|
||||
for (ring = 0; ring < 60; ring++) {
|
||||
var rt = ring / 60 * Math.PI * 2 + t * 0.3;
|
||||
splats.push({
|
||||
center: [
|
||||
Math.cos(rt) * 1.6,
|
||||
Math.sin(rt) * 1.2 - 0.2,
|
||||
2.0 + Math.sin(rt * 3 + t * 0.5) * 0.3
|
||||
],
|
||||
color: [0.95, 0.55, 0.15],
|
||||
opacity: 1.0,
|
||||
scale: [0.014, 0.014, 0.014]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Map a single landmark to world coords. Coord conventions:
|
||||
// x: 0.5 - lm.x → mirror so left-of-screen = your left side (selfie)
|
||||
// y: lm.y - 0.5 → keep MediaPipe's y-DOWN convention; the renderer's
|
||||
// existing -y flip in updateSplats does the single flip
|
||||
// to Three.js Y-up. Pre-flipping here would double-flip
|
||||
// and the face would render upside down.
|
||||
// z: 2.0 + lm.z*8 → amplify lm.z (~[-0.1,+0.1]) so the nose/eye-socket
|
||||
// depth is visible from an oblique camera angle.
|
||||
function lmToCenter(lm) {
|
||||
return [
|
||||
(0.5 - lm.x) * 4.0,
|
||||
(lm.y - 0.5) * 3.0,
|
||||
2.0 + lm.z * 8.0
|
||||
];
|
||||
}
|
||||
|
||||
function pushFaceSplat(splats, center, alpha) {
|
||||
splats.push({
|
||||
center: center,
|
||||
color: [0.95 * alpha, 0.65 * alpha, 0.20 * alpha],
|
||||
opacity: 1.0,
|
||||
scale: [0.006, 0.006, 0.006]
|
||||
});
|
||||
}
|
||||
|
||||
// FACEMESH_TESSELATION is a flat array [a0,b0, a1,b1, ...] of vertex indices
|
||||
// forming edges of the triangulated face mesh. ~1300 edges × 2 = 2600 entries.
|
||||
// We interpolate 6 splats per edge → ~8000 splats per face vs 478 vertices.
|
||||
var FACE_EDGES = (typeof FACEMESH_TESSELATION !== "undefined") ? FACEMESH_TESSELATION : null;
|
||||
|
||||
// Push the user's face mesh point cloud into `splats` (no Foundation
|
||||
// context — that is the demo path's responsibility). Used both as the
|
||||
// demo subject AND as an overlay on top of live/remote backend data
|
||||
// when the camera is enabled. Returns true if any splats were pushed.
|
||||
function pushFaceSplats(splats) {
|
||||
if (faceMeshState !== "running" || !latestFaceLandmarks) return false;
|
||||
var lms = latestFaceLandmarks;
|
||||
var i;
|
||||
// 1. Original 478 vertices — bright anchor points for features.
|
||||
for (i = 0; i < lms.length; i++) {
|
||||
splats.push({
|
||||
center: lmToCenter(lms[i]),
|
||||
color: [1.0, 0.72, 0.25],
|
||||
opacity: 1.0,
|
||||
scale: [0.010, 0.010, 0.010]
|
||||
});
|
||||
}
|
||||
// 2. Edge interpolation — 6 splats per FACEMESH_TESSELATION edge.
|
||||
if (FACE_EDGES) {
|
||||
var edgeCount = FACE_EDGES.length;
|
||||
var SAMPLES = 6;
|
||||
var e, a, b, t, f;
|
||||
for (e = 0; e < edgeCount; e += 2) {
|
||||
a = lms[FACE_EDGES[e]];
|
||||
b = lms[FACE_EDGES[e + 1]];
|
||||
if (!a || !b) continue;
|
||||
var aPos = lmToCenter(a);
|
||||
var bPos = lmToCenter(b);
|
||||
var ax = aPos[0], ay = aPos[1], az = aPos[2];
|
||||
var bx = bPos[0], by = bPos[1], bz = bPos[2];
|
||||
for (t = 1; t <= SAMPLES; t++) {
|
||||
f = t / (SAMPLES + 1);
|
||||
pushFaceSplat(splats, [
|
||||
ax * (1 - f) + bx * f,
|
||||
ay * (1 - f) + by * f,
|
||||
az * (1 - f) + bz * f
|
||||
], 0.85);
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function faceMeshFrame() {
|
||||
if (faceMeshState !== "running" || !latestFaceLandmarks) return null;
|
||||
var splats = [];
|
||||
pushFaceSplats(splats);
|
||||
pushFoundationContext(splats);
|
||||
demoFrameNum += 1;
|
||||
return {
|
||||
splats: splats,
|
||||
count: splats.length,
|
||||
frame: demoFrameNum,
|
||||
live: false,
|
||||
source: "face-mesh",
|
||||
pipeline: {
|
||||
skeleton: null,
|
||||
vitals: { breathing_rate: 14, motion_score: 0.15 }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildSplatsUrl() {
|
||||
if (backendArg === "demo") return null;
|
||||
if (backendArg === "auto") return "/api/splats";
|
||||
// User-supplied URL — strip trailing slash and append /api/splats.
|
||||
var base = backendArg.replace(/\/+$/, "");
|
||||
return base + "/api/splats";
|
||||
}
|
||||
|
||||
function syntheticFrame() {
|
||||
// Used when camera permission is denied / unavailable. Renders a
|
||||
// procedural standing figure inside the Seldon vault context.
|
||||
// y-down convention: head at small/negative y, feet at large/positive y;
|
||||
// the renderer flips y so head ends up at the top of the screen.
|
||||
var t = (Date.now() - demoStartMs) / 1000.0;
|
||||
var sway = Math.sin(t * 0.8) * 0.05;
|
||||
var breath = Math.sin(t * 1.2) * 0.015;
|
||||
var splats = [];
|
||||
|
||||
// Standing figure — 240 points in a vertical cylinder, denser than
|
||||
// before to feel like a holographic projection.
|
||||
var ring, k_ring, theta, r, py;
|
||||
for (ring = 0; ring < 30; ring++) {
|
||||
py = -1.0 + (ring / 30) * 2.2; // head (-1.0) → feet (+1.2) in y-down
|
||||
r = 0.20 + breath * (py < 0 ? 1.5 : 0); // chest expands more on inhale
|
||||
for (k_ring = 0; k_ring < 16; k_ring++) {
|
||||
theta = (k_ring / 16) * Math.PI * 2;
|
||||
splats.push({
|
||||
center: [
|
||||
sway + Math.cos(theta) * r,
|
||||
py,
|
||||
2.3 + Math.sin(theta) * r
|
||||
],
|
||||
color: [0.91, 0.65, 0.20],
|
||||
opacity: 1.0,
|
||||
scale: [0.018, 0.018, 0.018]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 17 COCO keypoints in normalized [0,1] image coords (matches live shape)
|
||||
var headY = 0.18;
|
||||
var keypoints = [
|
||||
[0.50 + sway * 0.05, headY, 0.95], // 0 nose
|
||||
[0.52 + sway * 0.05, headY - 0.01, 0.92], // 1 leftEye
|
||||
[0.48 + sway * 0.05, headY - 0.01, 0.92], // 2 rightEye
|
||||
[0.54 + sway * 0.05, headY, 0.85], // 3 leftEar
|
||||
[0.46 + sway * 0.05, headY, 0.85], // 4 rightEar
|
||||
[0.60 + sway * 0.04, 0.32, 0.93], // 5 leftShoulder
|
||||
[0.40 + sway * 0.04, 0.32, 0.93], // 6 rightShoulder
|
||||
[0.65 + sway * 0.03, 0.46, 0.90], // 7 leftElbow
|
||||
[0.35 + sway * 0.03, 0.46, 0.90], // 8 rightElbow
|
||||
[0.68, 0.60 + Math.sin(t * 1.4) * 0.02, 0.86], // 9 leftWrist
|
||||
[0.32, 0.60 - Math.sin(t * 1.4) * 0.02, 0.86], // 10 rightWrist
|
||||
[0.57, 0.58, 0.94], // 11 leftHip
|
||||
[0.43, 0.58, 0.94], // 12 rightHip
|
||||
[0.58, 0.74, 0.90], // 13 leftKnee
|
||||
[0.42, 0.74, 0.90], // 14 rightKnee
|
||||
[0.59, 0.92, 0.88], // 15 leftAnkle
|
||||
[0.41, 0.92, 0.88] // 16 rightAnkle
|
||||
];
|
||||
|
||||
// Wrap the figure in the Seldon-vault context (grid, spiral, starfield, halo)
|
||||
pushFoundationContext(splats);
|
||||
|
||||
demoFrameNum += 1;
|
||||
return {
|
||||
splats: splats,
|
||||
count: splats.length,
|
||||
frame: demoFrameNum,
|
||||
live: false,
|
||||
pipeline: {
|
||||
skeleton: { keypoints: keypoints, confidence: 0.86 },
|
||||
vitals: {
|
||||
breathing_rate: 14 + Math.round(Math.sin(t * 0.05) * 2),
|
||||
motion_score: 0.18 + Math.abs(sway) * 2
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function pickDemoFrame() {
|
||||
// Prefer real face-mesh data when the camera is running; else procedural.
|
||||
return faceMeshFrame() || syntheticFrame();
|
||||
}
|
||||
|
||||
// Once auto mode confirms there is no /api/splats backend on this origin,
|
||||
// set this flag so we stop hammering the network with 404 fetches every
|
||||
// tick. Console stays clean; demo renders locally.
|
||||
var networkDisabled = false;
|
||||
|
||||
// Exponential backoff state for explicit ?backend=<url>. The user's
|
||||
// local server may be down (ERR_CONNECTION_REFUSED) and we shouldn't
|
||||
// hammer it 10 Hz indefinitely. After each failure we lengthen the
|
||||
// delay; on success we snap back to the normal cadence.
|
||||
var BASE_INTERVAL_MS = 250;
|
||||
var MAX_INTERVAL_MS = 30000;
|
||||
var currentIntervalMs = BASE_INTERVAL_MS;
|
||||
var consecutiveFailures = 0;
|
||||
var fetchTimer = null;
|
||||
var lastBackendError = null;
|
||||
|
||||
function scheduleNextFetch(delayMs) {
|
||||
if (fetchTimer) clearTimeout(fetchTimer);
|
||||
fetchTimer = setTimeout(fetchCloud, delayMs);
|
||||
}
|
||||
|
||||
async function fetchCloud() {
|
||||
// Demo-only mode: never hit the network. Use the normal cadence.
|
||||
if (backendArg === "demo" || networkDisabled) {
|
||||
transportMode = "demo";
|
||||
handleData(pickDemoFrame());
|
||||
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
var resp = await fetch(buildSplatsUrl(), { cache: "no-store" });
|
||||
if (!resp.ok) throw new Error("HTTP " + resp.status);
|
||||
var resp = await fetch("/api/splats");
|
||||
var data = await resp.json();
|
||||
transportMode = (backendArg === "auto") ? "live" : "remote";
|
||||
consecutiveFailures = 0;
|
||||
currentIntervalMs = BASE_INTERVAL_MS;
|
||||
lastBackendError = null;
|
||||
handleData(data);
|
||||
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||
} catch (err) {
|
||||
consecutiveFailures += 1;
|
||||
lastBackendError = err && err.message ? err.message : String(err);
|
||||
if (requireLive) {
|
||||
document.getElementById("stats").innerHTML =
|
||||
'<span class="demo">● OFFLINE</span><br>Live backend required (?live=1) but unreachable.<br><span class="label">' + lastBackendError + '</span>';
|
||||
// Even strict-live: back off so we don't spam.
|
||||
currentIntervalMs = Math.min(currentIntervalMs * 2, MAX_INTERVAL_MS);
|
||||
scheduleNextFetch(currentIntervalMs);
|
||||
return;
|
||||
}
|
||||
// Auto mode + first failure → assume static host (Pages), disable
|
||||
// network entirely so the console stays clean.
|
||||
if (backendArg === "auto") {
|
||||
networkDisabled = true;
|
||||
transportMode = "demo";
|
||||
handleData(pickDemoFrame());
|
||||
scheduleNextFetch(BASE_INTERVAL_MS);
|
||||
return;
|
||||
}
|
||||
// Explicit backend (?backend=<url>) — keep trying with
|
||||
// exponential backoff: 250 ms → 500 ms → 1 s → 2 s … up to 30 s.
|
||||
// Render the demo while we wait so the scene stays alive, and
|
||||
// surface the failure so the user knows the server is down.
|
||||
currentIntervalMs = Math.min(Math.max(BASE_INTERVAL_MS * Math.pow(2, consecutiveFailures - 1), 1000), MAX_INTERVAL_MS);
|
||||
transportMode = "demo";
|
||||
var demoFrame = pickDemoFrame();
|
||||
demoFrame._backendUnreachable = true;
|
||||
demoFrame._backendUrl = backendArg;
|
||||
demoFrame._backendError = lastBackendError;
|
||||
demoFrame._retryInMs = currentIntervalMs;
|
||||
handleData(demoFrame);
|
||||
scheduleNextFetch(currentIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
function handleData(data) {
|
||||
try {
|
||||
if (data.splats && data.frame !== lastFrame) {
|
||||
// Compute CSI frame rate
|
||||
var now = Date.now();
|
||||
@@ -581,75 +117,24 @@
|
||||
}
|
||||
prevTimestamp = now;
|
||||
lastFrame = data.frame;
|
||||
updateSplats(data.splats);
|
||||
|
||||
// Overlay browser face mesh on top of backend splats when both
|
||||
// are active — lets visitors see their own face *plus* the
|
||||
// ESP32-driven point cloud in the same scene. Demo mode (where
|
||||
// data.source === "face-mesh") already includes the face, so
|
||||
// we skip this branch there to avoid double-counting.
|
||||
var rendered = data.splats;
|
||||
var faceOverlay = false;
|
||||
if (data.source !== "face-mesh"
|
||||
&& faceMeshState === "running"
|
||||
&& latestFaceLandmarks) {
|
||||
rendered = data.splats.slice();
|
||||
pushFaceSplats(rendered);
|
||||
faceOverlay = true;
|
||||
}
|
||||
data._faceOverlay = faceOverlay;
|
||||
updateSplats(rendered);
|
||||
|
||||
// No-signal detection: hide skeleton and show banner when
|
||||
// the server reports no live CSI frames in the last 5s.
|
||||
// Draw skeleton if available
|
||||
var pipe = data.pipeline;
|
||||
var csiLive = data.csi_live || (pipe && pipe.csi_live);
|
||||
// Only show no-signal when connected to a real backend
|
||||
// (not demo/face-mesh mode where csi_live is always false).
|
||||
var showNoSignal = (transportMode === "live" || transportMode === "remote")
|
||||
&& csiLive === false;
|
||||
setNoSignal(showNoSignal);
|
||||
if (showNoSignal) {
|
||||
clearSkeleton();
|
||||
} else if (pipe && pipe.skeleton && pipe.skeleton.keypoints) {
|
||||
if (pipe && pipe.skeleton && pipe.skeleton.keypoints) {
|
||||
drawSkeleton(pipe.skeleton.keypoints);
|
||||
} else {
|
||||
clearSkeleton();
|
||||
}
|
||||
|
||||
// Build info panel — badge reflects active transport
|
||||
var mode;
|
||||
if (transportMode === "live") {
|
||||
mode = '<span class="live">● LIVE</span> Local Backend';
|
||||
} else if (transportMode === "remote") {
|
||||
mode = '<span class="live">● REMOTE</span> ' + backendArg;
|
||||
} else if (data.source === "face-mesh") {
|
||||
mode = '<span class="face">● DEMO</span> Your Face (MediaPipe)';
|
||||
} else {
|
||||
mode = '<span class="demo">● DEMO</span> Synthetic';
|
||||
}
|
||||
if (data._faceOverlay) {
|
||||
mode += ' <span class="face">+ face overlay</span>';
|
||||
}
|
||||
var splatCount = rendered ? rendered.length : data.count;
|
||||
var html = mode + "<br>"
|
||||
+ "Splats: " + splatCount + "<br>"
|
||||
// Build info panel
|
||||
var mode = data.live
|
||||
? '<span class="live">● LIVE</span>'
|
||||
: '<span class="demo">● DEMO</span>';
|
||||
var html = mode + " Camera + CSI<br>"
|
||||
+ "Splats: " + data.count + "<br>"
|
||||
+ "Frame: " + data.frame;
|
||||
|
||||
// Unreachable backend banner — explicit ?backend=<url> failed
|
||||
// to connect. Show actionable guidance instead of leaving the
|
||||
// user staring at a "demo" badge wondering why their ESP32
|
||||
// feed isn't visible.
|
||||
if (data._backendUnreachable) {
|
||||
var nextSec = Math.round((data._retryInMs || 1000) / 1000);
|
||||
html += '<div class="section">'
|
||||
+ '<span class="demo">● ' + data._backendUrl + '</span> unreachable'
|
||||
+ '<br><span class="label">' + (data._backendError || "connection failed") + '</span>'
|
||||
+ '<br><span class="label">retry in ' + nextSec + 's</span>'
|
||||
+ '<br><br><span class="label">start the server:</span>'
|
||||
+ '<br><code style="color:#e8a634">cargo run -p wifi-densepose-pointcloud --release \\<br> -- serve --bind 127.0.0.1:9880</code>'
|
||||
+ '</div>';
|
||||
}
|
||||
|
||||
// CSI frame rate
|
||||
html += '<div class="section">'
|
||||
+ '<span class="label">CSI Rate:</span> '
|
||||
@@ -702,69 +187,8 @@
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
// Wire the camera CTA. The camera is now overlay-able on every
|
||||
// transport mode: in demo it IS the subject; in live/remote it
|
||||
// overlays the backend splats so the visitor sees their face
|
||||
// alongside the ESP32-driven point cloud.
|
||||
(function wireCamCta() {
|
||||
var btn = document.getElementById("cam-cta");
|
||||
if (!btn) return;
|
||||
if (requireLive) {
|
||||
// Strict-live mode shows the offline panel — no camera UI.
|
||||
btn.classList.add("hidden");
|
||||
return;
|
||||
}
|
||||
// In remote mode, label the button as an overlay action.
|
||||
if (backendArg.startsWith("http")) {
|
||||
btn.textContent = "▶ Add face overlay";
|
||||
}
|
||||
btn.addEventListener("click", function() {
|
||||
btn.textContent = backendArg.startsWith("http")
|
||||
? "Starting overlay…"
|
||||
: "Initializing the Vault…";
|
||||
startFaceMesh();
|
||||
});
|
||||
})();
|
||||
|
||||
// Wire the ESP32 backend CTA: prompts for a ruview-pointcloud serve URL,
|
||||
// persists last-used value in localStorage, and reloads with the
|
||||
// ?backend=<url> query so the existing remote-mode path takes over.
|
||||
// Disconnect by clicking again when already connected.
|
||||
(function wireEspCta() {
|
||||
var btn = document.getElementById("esp-cta");
|
||||
if (!btn) return;
|
||||
var connected = backendArg.startsWith("http");
|
||||
if (connected) {
|
||||
btn.classList.add("connected");
|
||||
btn.textContent = "📡 ESP32 connected · disconnect";
|
||||
}
|
||||
btn.addEventListener("click", function() {
|
||||
if (connected) {
|
||||
// Strip ?backend= from current URL and reload — return to demo.
|
||||
var u = new URL(window.location.href);
|
||||
u.searchParams.delete("backend");
|
||||
window.location.href = u.toString();
|
||||
return;
|
||||
}
|
||||
var stored;
|
||||
try { stored = localStorage.getItem("ruview.backendUrl"); } catch (_) { stored = null; }
|
||||
var def = stored || "http://127.0.0.1:9880";
|
||||
var url = window.prompt(
|
||||
"Enter the ruview-pointcloud serve URL (run `ruview-pointcloud serve` locally with your ESP32 streaming CSI to UDP port 3333):",
|
||||
def
|
||||
);
|
||||
if (!url) return;
|
||||
url = url.replace(/\/+$/, "");
|
||||
try { localStorage.setItem("ruview.backendUrl", url); } catch (_) {}
|
||||
var u2 = new URL(window.location.href);
|
||||
u2.searchParams.set("backend", url);
|
||||
window.location.href = u2.toString();
|
||||
});
|
||||
})();
|
||||
|
||||
// fetchCloud self-schedules via setTimeout — no setInterval to avoid
|
||||
// overlapping calls on slow networks and to support exponential backoff.
|
||||
fetchCloud();
|
||||
setInterval(fetchCloud, 500);
|
||||
|
||||
function updateSplats(splats) {
|
||||
if (pointsMesh) scene.remove(pointsMesh);
|
||||
|
||||
Reference in New Issue
Block a user