mirror of
https://github.com/ruvnet/RuView
synced 2026-06-25 12:53:19 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 58b8003cd9 | |||
| 41bee64593 | |||
| 5bc3b634b7 | |||
| e1f4897269 | |||
| 9f80b66ae3 | |||
| 02cb84e0bb | |||
| ebfaee4437 | |||
| db3d94a313 | |||
| a369fbe66e | |||
| d2089c342a | |||
| 306d009e72 | |||
| df617145d6 | |||
| f250149e94 | |||
| faca0530de | |||
| 6f6c867629 | |||
| 95a5ecc746 | |||
| 1f05456588 | |||
| f756a8af49 | |||
| 261ce80a72 |
@@ -33,6 +33,8 @@ jobs:
|
||||
working-directory: v2
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: rustup show && rustc --version
|
||||
|
||||
@@ -53,6 +53,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -42,6 +42,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Determine deployment environment
|
||||
id: determine-env
|
||||
@@ -86,6 +88,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
@@ -132,6 +136,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -82,6 +83,13 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# ADR-262 P1: `wifi-densepose-rufield` path-deps the `vendor/rufield`
|
||||
# submodule. Without a recursive checkout the workspace build fails to
|
||||
# resolve those path deps in CI even though it passes locally.
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# `wifi-densepose-desktop` is a Tauri v2 app — `glib-sys`, `gtk-sys`,
|
||||
# `webkit2gtk-sys`, etc. need the Linux dev libraries via pkg-config or the
|
||||
@@ -202,6 +210,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
continue-on-error: true
|
||||
@@ -267,6 +277,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
@@ -335,6 +347,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
@@ -407,6 +421,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -35,6 +35,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Fetch /traffic/clones + /traffic/views from GitHub
|
||||
env:
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -78,6 +80,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -145,6 +149,8 @@ jobs:
|
||||
vars.HAS_GCP_CREDENTIALS == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download x86_64 artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -20,6 +20,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with: { targets: wasm32-unknown-unknown }
|
||||
|
||||
@@ -26,6 +26,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust + wasm32 target
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -83,6 +85,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -131,6 +135,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
if: github.ref_type == 'tag'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Check firmware version.txt == tag
|
||||
run: |
|
||||
# Tag form: vX.Y.Z-esp32 → expect version.txt to contain X.Y.Z
|
||||
@@ -71,6 +73,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Build firmware (${{ matrix.variant }})
|
||||
working-directory: firmware/esp32-csi-node
|
||||
|
||||
@@ -100,6 +100,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -214,6 +216,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install clang
|
||||
run: |
|
||||
@@ -263,6 +267,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install NVS generator
|
||||
run: pip install esp-idf-nvs-partition-gen
|
||||
@@ -317,6 +323,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
|
||||
@@ -41,6 +41,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install mosquitto + clients and start with allow_anonymous
|
||||
run: |
|
||||
|
||||
@@ -26,6 +26,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
|
||||
@@ -76,6 +76,8 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# Linux aarch64 needs QEMU for cross-build on x86_64 runners.
|
||||
- name: Set up QEMU
|
||||
@@ -121,6 +123,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install maturin
|
||||
run: pip install maturin>=1.7
|
||||
- name: Build sdist
|
||||
@@ -144,6 +148,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -29,6 +29,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage viewer for Pages
|
||||
run: |
|
||||
|
||||
@@ -40,6 +40,8 @@ jobs:
|
||||
- { label: 'full+train', flags: '--features full,train' }
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -60,6 +62,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# v2/rust-toolchain.toml pins channel "1.89" with profile "minimal" (no
|
||||
# clippy). dtolnay@stable installs clippy on the floating "stable"
|
||||
# toolchain, but the override makes cargo use the separate "1.89"
|
||||
@@ -93,6 +97,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -127,6 +133,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: publish = false is present (no accidental crates.io publish)
|
||||
run: |
|
||||
CARGO=v2/crates/ruview-swarm/Cargo.toml
|
||||
|
||||
@@ -28,6 +28,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -97,6 +98,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -164,6 +167,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
@@ -245,6 +250,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Run Checkov IaC scan
|
||||
continue-on-error: true
|
||||
@@ -307,6 +314,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run TruffleHog secret scan
|
||||
@@ -341,6 +349,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -378,6 +388,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Check security policy files
|
||||
continue-on-error: true
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage demos for Pages
|
||||
run: |
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -18,3 +18,6 @@
|
||||
path = v2/crates/ruv-neural
|
||||
url = https://github.com/ruvnet/ruv-neural.git
|
||||
branch = main
|
||||
[submodule "vendor/rufield"]
|
||||
path = vendor/rufield
|
||||
url = https://github.com/ruvnet/rufield
|
||||
|
||||
@@ -8,6 +8,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## [Unreleased]
|
||||
|
||||
### Security
|
||||
- **`homecore-recorder` security review (ADR-132 surfaces) — two real bounding fixes; SQL-injection & NaN-index dimensions confirmed clean with evidence.** Beyond-SOTA review of the HA-compat state recorder (DB persistence + history + ruvector semantic search), the crux being its DB-backed SQL-injection surface. **Findings + fixes:** (1) **Memory-DoS — unbounded `get_state_history`.** The history query carried no `LIMIT`, so a wide `[since, until]` window over a high-frequency entity (a per-second sensor ≈ 86k rows/day) would load an unbounded row set into a single in-memory `Vec`. Added a hard `LIMIT MAX_HISTORY_ROWS` (1,000,000 — generous enough never to truncate a realistic history graph, bounded enough to cap the worst case); the sibling search paths were already `k`-bounded. (2) **Disk-DoS / documented-but-missing `purge`.** The README + HA-compat table advertised `Recorder::purge(older_than)` as a capability, but **no such method existed** — i.e. no retention path at all → unbounded disk growth. Implemented a **transactional** `purge` that deletes `states` + `events` strictly **older than** the cutoff (**exclusive** boundary — idempotent, no off-by-one; a row at the cutoff instant is kept) and **garbage-collects** orphaned `state_attributes` blobs (a dedup-shared blob is dropped only once its last referencing state is gone); all three deletes run in one transaction so a mid-purge failure rolls back cleanly (no states-deleted-but-events-kept corruption). **Confirmed clean with evidence:** SQL injection — **every** query in `db.rs` uses bound `?` parameters (no `format!`/string-concat of user data into SQL); the lone `format!` builds the LIKE *pattern*, which is itself bound as a parameter with `ESCAPE '\\'` and metacharacter escaping. Pinned: a state value `'; DROP TABLE states; --` is stored/queried **literally** (table survives), and a `%`/`_` in a search query matches **literally**, not as a wildcard. NaN-index poisoning (the calibration/vitals/geo class) — **structurally impossible** here: embeddings are SHA-256 → `i32` → `f32` (an `i32` cast to `f32` is always finite, never NaN/Inf), with an all-zero-digest norm guard; probed empty-index search, empty-string query, and `k=0` — all return `Ok(0)`, **no panic**. Fail-closed write path — a removal event yields `Ok(None)`, semantic-index failure is logged not propagated (best-effort, never blocks the durable SQLite write), and `EntityId` parsing failures fall back rather than panic. **6 new pinning tests** (SQL-injection literal-storage, LIKE-metacharacter literalness, history `LIMIT`, purge exclusive-boundary, purge attribute-GC-keeps-shared, purge old-events): `homecore-recorder` **19 → 25** (`--no-default-features`) / **25 → 31** (`--features ruvector`), 0 failed; the purge-boundary test is a true pin (fails deleting 2 rows under an inclusive cutoff, passes deleting 1 under the exclusive cutoff). Behaviour otherwise unchanged; Python deterministic proof unchanged (recorder is off the signal proof path).
|
||||
|
||||
### Added
|
||||
- **RuField `rufield-viewer` live-ingest mode — closes the RuView↔RuField visual loop (ADR-262 surfaces).** The dashboard gains `--source live --upstream <RuView-URL>`: it consumes RuView's `/ws/field` SSE (falling back to polling `/api/field`), **verifies every event's ed25519 provenance receipt on ingest** (`is_fusable`) — forged/tampered events are flagged ✗ and **never fused** into trusted inferences — and renders real RuView `FieldEvent`s through the same room-state/privacy-badge/fusion-graph/receipt path the synthetic mode uses (wire-compatible by construction: both sides use `rufield_core::FieldEvent` serde). **Strict banner honesty:** a single `BannerState` shows `SYNTHETIC` / `LIVE — <upstream>` / `DISCONNECTED — <upstream> unreachable`, mutually exclusive — never SYNTHETIC while showing live data or vice versa; live mode returns **409** on `/api/run` rather than fabricate a synthetic run, and starts DISCONNECTED until first verified contact. Default stays synthetic. 26 tests / 0 failed. `ruvnet/rufield` `crates/rufield-viewer`; `vendor/rufield` submodule bumped.
|
||||
- **ADR-262 P3 — live RuField surface: RuView's running sensing-server now speaks RuField on `/api/field` + `/ws/field`.** Wires the P1 `wifi-densepose-rufield` bridge into the live `wifi-densepose-sensing-server` (the bridge is the only added coupling, ADR-262 §5.4). A new `src/rufield_surface.rs` module (kept out of the 8k-line `main.rs`) holds a `FieldSurface` with a **dedicated ed25519 `Signer`**, a bounded ring buffer of recent signed events (`FIELD_RING_CAPACITY = 64`), and the `/ws/field` broadcast topic; it exposes `GET /api/field` (latest signed `FieldEvent`s + signer pubkey + a `dev_signing_key` flag) and `GET /ws/field` (per-cycle stream, mirroring `/ws/sensing`), plus a standalone `router()` for isolated testing. **Tap:** at the ESP32 governed-trust cycle (`main.rs` `observe_cycle` ~`:5886` / `SensingUpdate` build ~`:5938`), `emit_rufield_event` joins the cycle's real `SensingUpdate` (features/classification/signal_field) with the engine's recorded `effective_class`/`demoted` trust state into a `SensingSnapshot` and surfaces a signed `FieldEvent` — **existing endpoints (`/ws/sensing` etc.) are unchanged; this is purely additive.** **Signer (defers the P2 key decision, §8 Q1):** a **standalone dev/sensing key** from `WDP_RUFIELD_SIGNING_SEED` (64-hex or ≥32-byte value), else a deterministic dev default with a logged `WARN` — reusing the `cog-ha-matter` Ed25519 key is the deferred P2 call, so P3 does not pre-empt it. **Egress privacy (fail-closed):** `network_egress_allowed` is *stricter* than `DefaultPrivacyGuard` for an unattended live surface — only **P1/P2** leave the box; P0 (raw) and P3/P4/P5 are held edge-local, so a `Derived → P4/P5` cycle **never** surfaces; no-presence cycles emit **no phantom event**. **P3 acceptance gates (`tests/rufield_surface_test.rs`, 4 integration via `tower::oneshot` + 4 module unit, 0 failed):** a well-formed **signed** event (`Modality::WifiCsi`, P2 not P1, `is_fusable` ed25519-verified, real timestamp); empty cycle → no phantom; **privacy-safety** — an injected `Derived` trust never surfaces; a mixed stream surfaces only egress-safe events. **Honest scope (ADR-262 §0/§6):** real plumbing on a **live endpoint**, **NOT accuracy** — single-link CSI with its existing caveats (no validated room-coordinate accuracy — `field_localize`), a dedicated dev signing key pending the P2 ownership decision, no accuracy claim. The win is narrowly: "RuView's live sensing now speaks RuField on `/ws/field`."
|
||||
- **ADR-262 P1 — `wifi-densepose-rufield` anti-corruption bridge: RuView WiFi-CSI sensing → signed RuField `FieldEvent`s.** A new v2 workspace member (the *single coupling point* between RuView and the standalone RuField MFS spec, ADR-262 §5.4) that **path-deps** the `vendor/rufield` submodule crates (`rufield-core`/`-provenance`/`-privacy`/`-fusion` — pure-Rust, `--no-default-features`-buildable: serde/sha2/ed25519/toml only, no tch/openblas/ndarray/candle) and **no** RuView internal crate. The bridge takes owned primitives — `SensingSnapshot` mirrors the `/ws/sensing` `SensingUpdate` (features + classification + signal_field) joined with the `TrustedOutput` trust state (`trust_class`/`demoted`/`identity_bound`) — and `snapshot_to_field_event()` emits one **signed** `FieldEvent` (`Modality::WifiCsi`, axis `[Frequency]`): a real `FieldTensor` from the feature scalars with the real `timestamp_ns`; an `Observation` whose `range_m`/`motion_vector`/`space_cell` are derived from the strongest **signal-field peak** when present (else `None` — coordinates are **never fabricated**, per the `field_localize` caveat) and `confidence` from the classification; a real `ProvenanceRef` (sha256 over the tensor bytes, `synthetic=false`) **ed25519-signed** so `rufield_provenance::is_fusable` passes. **The §3.3 privacy mapping is the critical correctness item**, implemented as `map_privacy()` mapping RuView's class onto RuField P0–P5 **by information content, NEVER by byte value** and **fail-closed**: RuView `Derived` (byte `1`, which sorts *below* `Anonymous` byte `2`) carries an identity embedding → maps to **P4** (or **P5** if identity-bound), **never P1** (the single most dangerous mapping mistake); `Raw → P0`, `Anonymous → P2`, `Restricted → P2`; a governed-engine `demoted` cycle floors the egress class to ≥ P2 with raw suppressed. **P1 acceptance gates (15 tests / 0 failed — 5 unit + 9 integration + 1 doc):** round-trip (`SensingSnapshot → FieldEvent →` serde `→` equal), `is_fusable` (verified ed25519 receipt), `RuFieldFusion::ingest` accept + `infer()` runs, **privacy-safety** (`gate_privacy_safety_derived_never_maps_to_low_privacy` — `Derived → P4/P5`, never P1; a table test over every RuView class; fail-closed demotion), and determinism (same snapshot + same signer seed → byte-identical event). **Honest scope:** this is **P1 plumbing** — a tested conversion + a safe privacy mapping. It is **not** wired into the live server (that is P3) and makes **no accuracy claim** (RuField v0.1 is synthetic; RuView's single-link CSI carries its own caveats). CI: the `rust-tests` workflow checkout gains `submodules: recursive` so the path-deps resolve. Python deterministic proof unchanged (off the signal proof path).
|
||||
- **ADR-262 (Proposed): RuField MFS ↔ RuView integration — a live `SensingServerAdapter`, a privacy/provenance bridge, MAPPED not papered-over.** Researched integration design for wiring RuField into RuView. Recommends: a thin **`wifi-densepose-rufield` bridge crate** (anti-corruption layer, path-deps on the `vendor/rufield` submodule — the `vendor/rvcsi` pattern, since rufield crates are unpublished); a **live `SensingServerAdapter`** that taps the real `SensingUpdate` emit site joined with `TrustedOutput` trust state and emits one signed `FieldEvent`/cycle (the file-based `CsiReplayAdapter` stays for offline replay); **vertical fusion composition** (ruvsense fuses *within* WiFi → one `wifi_csi` event → rufield-fusion graph fuses *across* modalities above it); and **one canonical privacy/provenance model** (RuView `effective_class` is source-of-truth, mapped to RuField P0–P5 at egress; reuse the existing `cog-ha-matter` SHA-256+Ed25519 chain for the `ProvenanceReceipt`). **Key honest finding:** RuView has **two privacy enums + three witness mechanisms across two hash algorithms** that do not map 1:1 onto P0–P5, and a real trap — RuView's `Derived` privacy byte (`1`) sorts *below* `Anonymous` (`2`) yet carries identity embeddings, so the bridge must map by **information content** (`Derived → P4/P5`), never by byte value, or it would leak identity as low-privacy P1. 4 independently-shippable phases, each with a test gate (round-trip / `is_fusable` / privacy-monotonicity / ed25519-verify). Honest scope: this is **plumbing architecture, not accuracy** — RuField v0.1 is synthetic and RuView's only real-CSI path is unlabeled replay; the ADR claims only architecture, gated by round-trip/monotonicity/signature tests.
|
||||
- **RuField `CsiReplayAdapter` — first real (non-synthetic) WiFi-CSI adapter (ADR-260 §17).** RuField now ingests **real captured WiFi CSI** instead of only the synthetic simulator. New `rufield-adapters::csi_replay` parses RuView's `.csi.jsonl` recording format (`{timestamp, subcarriers[]}`), normalizes each frame to a `FieldTensor` (`WifiCsi`, real amplitudes + real `timestamp_ns`), establishes a per-subcarrier Welford **empty-room baseline** via `calibrate()`, derives a **physically-grounded CSI-variance motion/presence proxy** (normalized MAD vs baseline → P2 motion/presence, else P1), and emits `FieldEvent`s with a **real sha256 + ed25519 provenance receipt** (`synthetic=false`). **Measured on 199 real captured frames:** 184 presence-proxy / 69 motion-proxy → fed through `RuFieldFusion` → **182 fused inferences (115 breathing, 67 person_present) from real signal.** 12 tests (9 unit + 3 integration over real-CSI fixtures), deterministic (byte-identical stream per file). **Honest caveats (stated everywhere):** it's **replay from file, not live hardware**; recordings are **unlabeled**, so the motion/presence output is a **proxy, NOT validated accuracy** (no pose, no accuracy numbers); live streaming + labeled validation remain roadmap; mmWave/thermal stay synthetic. The win is "RuField ingests real WiFi CSI and produces fused events from it." [`ruvnet/rufield`](https://github.com/ruvnet/rufield) `crates/rufield-adapters`; `vendor/rufield` submodule bumped.
|
||||
- **RuField `rufield-viewer` web dashboard — completes ADR-260 §27.9 (all §27 criteria 1–10 now PASS).** A read-only Axum + vanilla-JS dashboard (no build step — `cargo run -p rufield-viewer`) that streams the deterministic SyntheticSim→fusion camera-free room-intelligence demo: live room-state inferences with confidence, a scrolling event log where every event carries its modality + a colour-coded **P0–P5 privacy badge**, the fusion graph (supporting=green / contradicting=red per inference), and a click-to-open **provenance-receipt modal** (sha256 + ed25519 signer + verified ✓ / fusable ✓) — behind a permanent, undismissable `SYNTHETIC — simulated sensors, no hardware` banner. Endpoints `/` · `/app.js` · `/health` · `/api/run` (full deterministic JSON) · `/events` (SSE). 12 new tests. Honest scope: a read-only SYNTHETIC demo viewer, **not** a device-management console — fleet/real-adapter management is a separate later milestone. Lives in [`ruvnet/rufield`](https://github.com/ruvnet/rufield) (`crates/rufield-viewer`, repo now 7 crates / 72 tests); `vendor/rufield` submodule bumped to include it.
|
||||
- **ADR-261: RuVector graph-ANN index — a real HNSW baseline + a SymphonyQG-style quantized variant, MEASURED (honest negative).** Closes the [ADR-156 §5 #1](docs/adr/ADR-156-ruvector-fusion-beyond-sota.md) gap: the SymphonyQG (SIGMOD 2025) **3.5–17× QPS-over-HNSW** claim was CLAIMED-only because **no HNSW baseline existed to compare against**. This adds one. New pure-Rust, `--no-default-features`-buildable modules in `wifi-densepose-ruvector`: `hnsw.rs` (a correct float HNSW — Malkov & Yashunin: multi-layer NSW graph, `ef_construction`/`ef_search`, Algorithm-4 neighbour selection, **seeded-deterministic** level assignment via SplitMix64, L2 + cosine, full degenerate-case guards), `hnsw_quantized.rs` (the SymphonyQG-style variant — the **same** graph traversed by a cheap **1-bit Hamming** score over the RaBitQ Pass-2 rotated sign code, then **exact-float rerank**), `ann_measure.rs` + `benches/ann_bench.rs` (one shared deterministic planted-cluster fixture; the `ann_bench_report` test is the source of truth). **MEASURED (dim=128, N=10k, K=10, `--release`):** float HNSW = **~25× QPS over linear scan at recall ≥0.99** (the baseline this gap needed; recall@10 correctness gate ≥0.95 holds, L2 + cosine). **Honest negative:** the 1-bit quantized traversal is **too coarse to beat float HNSW at equal recall at this scale** — its best recall is **0.738**, never reaching the ≥0.90 equal-recall point, so there is **no QPS win** over float HNSW; the 3.5–17× is **not reproduced** by our 1-bit construction here. The recall gate also **caught a real index-out-of-bounds bug** in the insert path (disclosed in ADR-261 §4). Caveat: this is **our** HNSW + **our** 1-bit quant, not SymphonyQG's exact system — it tests the *direction* of the claim, with the expected crossover at large N + a multi-bit traversal code. **We did not tune to manufacture a speedup.** +20 tests (ruvector lib 131→151, 0 failed). ADR-156 §5 #1 / §8 backlog: CLAIMED → **MEASURED-direction-tested**. Python deterministic proof unchanged (off the signal proof path).
|
||||
- **ADR-261 Milestone-2: multi-bit quantized HNSW traversal + large-N scaling study — MEASURED (honest negative).** Extends ADR-261's quantized index from 1-bit to **`b`-bit-per-dimension** (`b ∈ {1,2,4}`, 16/32/64 B/node) over the Pass-2 rotated coordinates, and runs a deterministic scaling study (N ∈ {10k, 100k, 250k}) to test M1's *prediction* of a large-N crossover. **Result: no crossover at any measured (N, b), and the trend refutes the prediction.** At N=10k more bits lift the equal-recall QPS ratio (0.19×→0.46×→0.48×) and let b≥2 reach the 0.90 recall bar 1-bit missed — but quant stays slower than float HNSW at equal recall; at N=100k/250k quant recall *collapses* (b=4: 1.000→0.788→0.624, never ≥0.90) while float holds ≥0.92 (denser graph → low-bit codes can't separate near-neighbours, beam goes off-path faster than the float-distance saving repays). Caveat: our HNSW + our per-node multi-bit code, not SymphonyQG's RaBitQ-fused graph — refutes the *direction* at ≤250k, not their million-scale numbers. ruvector lib **151→156** (+5 tests; `scaling_report` `#[ignore]` produced the table). A published negative with the mechanism explained. ADR-261 §11.
|
||||
- **ADR-260: RuField MFS — the open specification for camera-free multimodal field sensing.** A common event / tensor / calibration / privacy / provenance model that sits *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and future quantum sensors (each modality emits a normalized `FieldEvent` → `FieldTensor` → `FusionGraph` → `PrivacyClass` → `ProvenanceReceipt`). Published as a **standalone repo** [`ruvnet/rufield`](https://github.com/ruvnet/rufield) and vendored here as the `vendor/rufield` submodule (the `vendor/rvcsi` pattern — not a `v2/` workspace member). The v0.1 reference stack is a self-contained 6-crate Rust workspace (`rufield-core`, `-provenance` [sha256 + ed25519], `-privacy` [P0–P5 guard], `-adapters` [deterministic `SyntheticSim` across wifi_csi/mmwave_radar/infrared_thermal], `-fusion` [graph + TOML weighted-Bayes rules → 7 room-state inferences], `-bench` [deterministic runner + the §31 acceptance test]). **60 tests / 0 failed, clippy-clean.** §27 acceptance criteria 1–8 and 10 PASS; the live dashboard (9) is deferred. **All benchmark metrics are SYNTHETIC** (scored against the simulator's own ground truth — presence/breathing/bed_exit/room_transition F1 = 1.000, nocturnal_scratch 0.923 reported honestly, p95 latency ~0.01 ms, provenance coverage 100%, 0 privacy violations) — they prove the pipeline recovers known truth, **not** field accuracy; real hardware adapters (ESP32 CSI, mmWave, thermal IR) are a documented roadmap item, none validated in v0.1. The Python deterministic proof is unchanged (rufield is off the signal-processing proof path).
|
||||
|
||||
### Security
|
||||
- **`homecore-automation` security review — two real DoS findings fixed (template unbounded-expansion + delay panic-on-config), each pinned by a fails-on-old test; condition-bypass / fail-closed / action-authz dimensions confirmed clean (ADR-129 §8a).** Beyond-SOTA review of the HA-compat automation engine (the execution/eval surface: triggers → conditions → actions, with user-config Jinja2 templates), un-covered by the ADR-154–159 sweep. **HC-SEC-01 (template DoS, HIGH):** a `template:` condition / `value_template` is user config and was rendered with MiniJinja's defaults — **no instruction budget, no output cap**. A single nested-loop condition rendered a **100 MB string in ~11 s on one render call** (measured) — the bfld-class unbounded expansion (MiniJinja's per-call `range()` 10k cap does **not** stop nesting). **Fixed** by enabling MiniJinja's `fuel` feature + `set_fuel(Some(1_000_000))` (the attack now fails fast ~90 ms with "engine ran out of fuel") and a 64 KiB source-length cap; legitimate templates unaffected. **HC-SEC-02 (panic-on-config DoS, MEDIUM):** `Action::Delay`/`WaitForTrigger` fed the user float straight into `Duration::from_secs_f64`, which **panics** on negative/NaN/inf/overflow — all reachable from a crafted or typo'd YAML (`delay: {seconds: -1}`, `.nan`, `.inf`, `1e308`), aborting the spawned run task (measured panic). **Fixed** by a `safe_duration_from_secs` guard that saturates (NaN/±inf/negative → `0`, matching HA's lenient "non-positive delay = no delay"; huge → clamped to ~100 yr). **Dimensions probed clean (evidence in ADR-129 §8a):** condition eval is **fail-closed** (template-render error → `false`; un-parseable `choose` branch condition → branch skipped, never silently passing); run-modes are **bounded** (Single/Restart/Queued/`max:N` — a self-triggering automation does not livelock, ADR-162 tests); templates are **read-only sandboxed** (no service-call/state-set global exposed to template scope, so a template cannot escalate to an action); no `unwrap`/`expect`/index panic reachable from a crafted config in the eval/exec path beyond the fixed `from_secs_f64`. Fails-on-old verified by reverting each fix in isolation (delay tests panic; template nested-loop test runs unbounded >60 s; oversized-source test fails). `cargo test -p homecore-automation --no-default-features`: **40 → 54 passed, 0 failed** (+14: 4 template-DoS, 1 no-regression render, 5 delay/wait + safe-duration unit). Workspace green; Python deterministic proof unchanged (homecore-automation is off the signal proof path).
|
||||
- **`cog-ha-matter` witness/manifest crypto review — engine-class signed-digest collision confirmed ABSENT (length-prefixing already correct); domain-separation tag ADDED + `verify_strict` HARDENED; key-handling & verify-before-trust confirmed clean (ADR-116 §2.2).** Beyond-SOTA crypto+security review of the Cognitum/HA-Matter bridge's SHA-256 + Ed25519 witness chain — the exact signing chain ADR-262 P2 proposes to reuse — un-covered by the ADR-154–159 sweep. **Top-priority check: the sibling `wifi-densepose-engine` bug class (unframed boundary-to-boundary concatenation of operator-influenceable strings into a signed/hashed digest).** Result reported honestly: **that bug class is ABSENT here** — `witness::canonical_bytes` already length-prefixes the two variable-length operator-influenceable fields (`kind_len:u32-be ‖ kind`, `payload_len:u32-be ‖ payload`) over fixed-width `prev_hash[32] ‖ seq:u64-be ‖ ts:u64-be`, an injective encoding (proven pre-existing by `canonical_bytes_length_prefixing_prevents_ambiguity`), and `witness_signing::sign_event`/`verify_signature` sign/verify the **identical** bytes the hash chain commits to (no separate unframed concatenation). The manifest `binary_signature` (Ed25519 over the fixed 64-hex-char `binary_sha256`) is signed **at build time by the Makefile**, not in-crate, and over a single fixed-length value — no in-crate manifest-signing concatenation surface. **Two real hardening gaps fixed, the first pinned by fails-on-old tests:**
|
||||
- **CHM-WIT-01 (missing domain-separation tag, LOW) — ADDED.** The engine review's prescribed fix is "domain-tag **+** length-prefix"; the length-prefix half was present, the **domain tag was absent**. The witness SHA-256 preimage / Ed25519 message carried no tag distinguishing it from any other signing context that shares key infrastructure — notably the manifest `binary_signature`, the very chain ADR-262 P2 reuses. **Fix:** prepend a versioned, NUL-terminated `WITNESS_DOMAIN_TAG = b"cog-ha-matter/witness-event/v1\x00"` to `canonical_bytes` (the doc-comment already anticipated a leading version migration). Cross-protocol separation now holds: a witness signature can never be replayed as a message for another Ed25519 context. **Witness-bytes change by design** (prior on-disk witness hashes/signatures invalidated, like the engine fix) — verified safe: **no in-repo crate consumes cog-ha-matter's witness bytes/signatures programmatically** (all references are doc-comment mentions; the crate is self-contained, no `use cog_ha_matter::` anywhere). Pinned by `canonical_bytes_is_domain_separated`, `canonical_bytes_starts_with_domain_tag_then_prev_hash`, `witness_preimage_cannot_collide_with_a_bare_manifest_digest` (witness.rs) and `signature_commits_to_domain_tag_not_bare_fields` (witness_signing.rs — a signature over the **un-tagged** field concatenation must NOT verify); the domain-separation guard **FAILED on the reverted un-tagged encoding** ("canonical message is not domain-separated").
|
||||
- **CHM-WIT-02 (permissive Ed25519 verification, LOW) — HARDENED to `verify_strict`.** For a tamper-evident **audit** chain the signature is the attestation, so `verify_signature` now uses `VerifyingKey::verify_strict` (rejects non-canonical encodings + small-order public keys per RFC 8032) instead of the permissive `Verifier::verify` — giving auditors the "one canonical signature per event" property they rely on when comparing/deduplicating signed records. Not a forgery fix (the public key is caller-pinned, never parsed from the event), reported at true LOW severity. Guarded by `verify_uses_strict_path_and_pins_caller_key`.
|
||||
- **Dimensions confirmed clean (with evidence, no invented issues):** (1) **verify-before-trust + key-pinning** — `verify_signature` takes the verifying key as a **caller-supplied parameter** (the Seed's known key), never reads a key from the event/manifest, so a forged event carrying its own key cannot self-attest; `WitnessChain::read_jsonl` re-derives and re-checks every `this_hash` on load (tampered bundle → `HashMismatch`) and runs a chain-level `verify()` catching reordered/spliced events (existing `verify_rejects_*`, `jsonl_parser_rejects_tampered_payload`, `read_jsonl_chain_verify_catches_reordered_events`). (2) **key handling** — the crate **never generates, stores, logs, or serializes** a signing key: `sign_event` takes `&SigningKey` by reference, the manifest struct has no key field, and the only key material in-crate is the **test-only** fixed seed (clearly documented "DO NOT use in production"); production keys come from the Seed's secure key store (out of scope, ADR-116 §key-management). No hardcoded/default/predictable production key, no key in the manifest, no world-readable key path (the crate does no key file I/O). (3) **determinism/canonicalization** — `canonical_bytes` is pure positional bytes (no HashMap iteration, no float formatting); Ed25519 is deterministic (pinned by `signature_is_deterministic_for_same_event_and_key`); the JSONL wire form is hand-rolled with **alphabetically-locked** field order (`jsonl_field_order_is_alphabetical_for_byte_stability`) and the mdns TXT records are `sort()`-ed for byte-stable advertisement — no iteration-order or float-format nondeterminism feeds any hash/signature. (4) **fail-closed parsing / DoS** — `from_jsonl_line`/`from_hex`/`hex_decode` return structured errors (never panic) on wrong length, non-hex, missing field, odd-length payload, or hash mismatch (`jsonl_parser_rejects_non_hex_hash`, `hex_decode_rejects_odd_length`, …); `main.rs` reads no untrusted files/paths (clap args only; `--print-manifest` emits a static template) — no path/injection surface. (5) **de-magic** — the witness/signing byte layout is already expressed as named widths; no bare security-relevant literals worth extracting beyond the new named `WITNESS_DOMAIN_TAG`. `cog-ha-matter --no-default-features`: **64→68 tests**, 0 failed (+3 domain-tag witness, +1 signing-layer domain-commit, +1 strict-verify key-pin; one pre-existing test renamed to assert the tag). Workspace green; Python deterministic proof unchanged (`f8e76f21…46f7a`, bit-exact — cog-ha-matter is off the signal proof path). Review notes appended to ADR-116 §2.2.
|
||||
- **`homecore-api` (HA-wire-compat REST + WebSocket) beyond-SOTA security review — `GET /api/` auth-gate gap FIXED + WS event-stream lag-DoS robustness FIXED; auth/traversal/injection/info-leak dimensions confirmed clean (ADR-161 / ADR-130).** Network-facing review of the HA-wire-compat API layer (remote attack surface), not covered by the ADR-154–159 sweep — same scrutiny the sibling `wifi-densepose-engine` and `-bfld` reviews got. **Two real bugs fixed, each pinned by a fails-on-old test.**
|
||||
- **HC-API-AUTH-01 (auth-gate gap, LOW) — `GET /api/` was unauthenticated; FIXED.** Every sibling REST route (`/api/config`, `/api/states`, `/api/services`, …) calls `BearerAuth::from_headers` first, but `rest::api_root` took no headers and unconditionally returned `200 {"message":"API running."}`. HA's `APIStatusView` inherits `requires_auth = True`, so an unauthenticated/wrong-token request to `/api/` must be **401** — HA clients use this status route as a token-validation probe, and a 200 both told a bad-token client its token was good and let an unauthenticated party confirm a live endpoint. Severity is LOW (the body is a static string — no entity/state data leaks), reported at true severity, not inflated. **Fix:** `api_root` now validates the bearer like its siblings. Pinned by `api_root_rejects_missing_bearer` + `api_root_rejects_wrong_bearer` (both 200→assert-401 on old code) and guarded by `api_root_accepts_correct_bearer`.
|
||||
- **HC-WS-LAG-01 (DoS-adjacent silent failure, LOW) — `subscribe_events` killed the event stream on a broadcast lag; FIXED.** The per-subscription task matched `Err(_) => break` on both `broadcast::Receiver::recv()` arms, but `Lagged(n)` (a slow consumer falling >4,096 events — `EVENT_CHANNEL_CAPACITY` — behind) is **recoverable**: the bus doc itself says "Lagged receivers must re-sync", and HA's WS contract keeps the subscription alive across a lag. The old code treated the first lag as fatal, so after an event burst the client's stream went **permanently silent** with no error frame — a self-inflicted event-delivery DoS under load. **Fix:** `Lagged(_) => continue` (skip the dropped window, re-sync), `Closed => break`, on both the system and domain arms. Pinned by `subscription_survives_broadcast_lag` (subscribes, floods 6,000 filtered events past the 4,096 capacity to force a `Lagged`, then asserts a subsequent subscribed event is still delivered — 5s-timeout panic on old code).
|
||||
- **Dimensions confirmed clean (with evidence, no invented issues):** (1) **AuthN/AuthZ** — all 7 other REST handlers (`get_config`/`get_states`/`get_state`/`set_state`/`delete_state`/`get_services`/`call_service`) gate on `BearerAuth::from_headers` → `LongLivedTokenStore::is_valid` before any work; the WS handshake validates the `auth` token against the **same** store before entering the command loop and the privileged commands are unreachable pre-`auth_ok` (HC-WS-01, already fixed). Token compare is a `HashSet::contains` (content-independent timing, not the byte-`==` oracle ADR-157 §B4 fixed in hardware) — no timing-oracle finding. No route skips the gate, no result-ignored check, no default/empty token accepted (`is_valid` rejects empty internally; `from_env` is non-dev). (2) **Path traversal** — **no route maps user input to a filesystem path** (state lives in an in-memory `DashMap`); `:entity_id` is funneled through `EntityId::parse`, a strict `[a-z0-9_]+\.[a-z0-9_]+` ASCII allowlist that rejects `..`, `/`, `\`, and absolute paths. No traversal surface exists. (3) **Injection** — no SQL, no shell/subprocess, no `format!`-into-response; `call_service`/`set_state` bodies are typed `serde_json::Value` passed to the in-process service registry (matches HA). (4) **Info-leak** — `ApiError` maps to fixed status + a `{message}` derived only from typed variants; `call_service`'s `ServiceError::HandlerFailed(String)` is integration-controlled (mirrors HA surfacing the handler error), not framework internals/paths/stack-traces (no ADR-080-class leak). (5) **CORS** is an explicit allowlist (`allow_credentials(false)`, HC-05 already fixed), not `permissive()`. (6) **De-magic** — no bare security-relevant literals in this crate worth extracting (`EVENT_CHANNEL_CAPACITY` already named in `homecore`; CORS dev-default ports are documented). `homecore-api --no-default-features`: **25→29 tests**, 0 failed (+2 api-root auth, +1 api-root accept-guard, +1 WS lag-survival); workspace green; Python deterministic proof unchanged (homecore-api is off the signal proof path). Review notes appended to ADR-161.
|
||||
- **`wifi-densepose-calibration` per-room calibration review — NaN-poisoning fail-closed gap FIXED + file/path & receipt surfaces confirmed clean (ADR-151).** Beyond-SOTA correctness+security review of the ADR-151 `baseline → enroll → extract → train → bank` pipeline (the appliance-deployed per-room specialist core), un-covered by the ADR-154–159 sweep. **One real numerical-robustness bug fixed.** `Features::from_series` — the live-inference *and* training feature path — computed `mean`/`variance`/`motion` over the raw scalar series with **no non-finite guard**, so a single `NaN`/`±inf` sample (a corrupt CSI frame) produced `mean=NaN, variance=NaN` and an all-`NaN` prototype embedding. Baked into a persisted `PresenceSpecialist::threshold`/`empty_mean` at train time, that `NaN` **silently disabled presence detection** for the life of the bank (every `f.variance > NaN` and `|mean − NaN|` comparison is false → presence always reads *absent*, confidence 0), with **no error raised** — the exact "produce NaN that poisons a specialist / silently accept garbage" failure, and an asymmetry vs the meticulously NaN-guarded `geometry_embedding.rs`. **Fix at the production boundary:** filter non-finite samples before any statistic (a corrupt frame counts as no frame); a wholly-non-finite series degrades to the new `Features::ZERO`, exactly like the empty series. **Value-identical for all-finite input** — `full_loop.rs` and every existing `extract` test pass unchanged. Pinned by two fails-on-old tests (`non_finite_samples_do_not_poison_features`, `all_non_finite_series_is_zero`, both FAILED pre-fix). **Dimensions confirmed clean (with evidence, no invented issues):** (1) **file/path handling** — the crate does **zero** file/path I/O (no `std::fs`/`Path`/`File`/`read`/`write` anywhere in `src/`; only in-memory `serde_json`), so path-traversal / unbounded-read / artifact-path concerns do not exist at the crate boundary — they live in the `wifi-densepose-cli` consumer (`room.rs`), out of this crate's scope; (2) **untrusted-load** — `SpecialistBank::from_json` parse-validates shape via serde (malformed → `CalibrationError::Serde`), and per ADR-151 invariant (B) banks are local-first, never network-received; (3) **receipt/hash integrity** — the crate emits **no** hash/receipt/witness/signature (no `CalibrationReceipt` analogue), so the engine's unframed-concatenation bug class is structurally absent — nothing to mis-frame; (4) **other numerical paths already robust** — `geometry_embedding.rs` sanitizes every input + sweeps to finite (verified by its `adversarial_inputs_never_produce_nan` test); presence/restlessness/anomaly divisions are all `.max(1e-3)`-guarded; `autocorr_dominant` guards `r0 ≤ 1e-6`, `n < 16`, empty bands; `SpecialistBank::train` rejects empty anchors; anomaly requires ≥2 anchors. De-magicked the bare specialist threshold literals (breathing 0.25 / heartbeat 0.3 default min-scores, anomaly 2.0× spread / >0.5 label cutoff) into named documented consts, value-identical, pinned by `default_min_score_constants_match_prior_literals` + `anomaly_constants_match_prior_literals`. `wifi-densepose-calibration --no-default-features`: **58→62 unit tests** (+2 NaN fail-closed, +2 de-magic pins) + 1 full-loop integration, 0 failed. Python deterministic proof unchanged (`f8e76f21…46f7a`, bit-exact — calibration is off the signal proof path). Review notes appended to ADR-151 §6.
|
||||
- **`wifi-densepose-engine` governed-trust review — witness domain-separation gap FIXED + privacy monotonicity confirmed clean (ADR-137 / ADR-141 / ADR-032).** Beyond-SOTA correctness+security review of the security-critical composition root (the cycle enforcing RuView's privacy guarantees), not covered by the ADR-154–159 sweep. **One real witness-integrity bug fixed.** `witness_of` concatenated `model_version`, `calibration_version`, and `privacy_decision` boundary-to-boundary and left the variable-length evidence list without a count, so a string straddling a field boundary collided with a *different* trust decision — e.g. a per-room adapter id (ADR-150 §3.4, operator-influenceable) absorbing the leading bytes of the calibration epoch (`model="…cal:00a"`,`cal="b"`) yields the same witness as `model="…"`,`cal="cal:00ab"`. Two distinct privacy-relevant input tuples → one witness defeats the ADR-137 §2.7 "any privacy-relevant delta → different witness" tamper/drift audit. **Fix:** domain-tag the BLAKE3 hash (`ruview.engine.witness.v1`), write an explicit evidence count, and **length-prefix every field** (8-byte LE length ‖ bytes) — unambiguous framing regardless of contents. Witness-layout change by design (prior witness bytes invalidated); downstream consumers (`engine_bridge`, rufield) assert only witness *relationships* (`assert_ne`/`assert_eq` across runs), never absolute bytes, so nothing breaks. Pinned by two fails-on-old tests: `witness_distinguishes_model_calibration_boundary`, `witness_distinguishes_evidence_model_boundary`. **Dimensions confirmed clean (with evidence, no invented issues):** (1) **privacy monotonicity** — `effective_class` is recomputed each cycle from the active mode's floor with at most a single-step `demote_one` (clamped at `Restricted`), no cross-cycle state, proven over **all 5 modes** by `forced_contradiction_never_relaxes_class` (forced contradiction only ever raises the class byte; clean cycle == base); (2) **fail-closed** — empty cycle errors with no degenerate output (`empty_cycle_fails_closed`), single-node boundary characterized (`single_node_cycle_is_well_formed`), NaN coupling → `max(0.0)`→absent edge→at-risk (more restrictive); (3) **witness determinism** — no HashMap iteration / float formatting feeds the hash; (4) **mesh_guard** (ADR-032) — partition-risk → demotion path verified, thresholds already named documented fields. De-magicked the engine-construction literals (coherence accept gate, ADR-143 SLAM discovery + static-anchor thresholds) into named documented consts, value-identical, pinned by `engine_constants_match_prior_values`. `wifi-densepose-engine --no-default-features`: **27→33 tests**, 0 failed (+2 witness, +1 monotonicity property, +2 fail-closed boundary, +1 de-magic pin). Python deterministic proof unchanged (`f8e76f21…46f7a`, bit-exact — the engine is off the signal proof path). Review notes appended to ADR-137 (witness) and ADR-141 (monotonicity).
|
||||
- **ADR-141 BFLD privacy-bypass closed — `process_to_frame` now routes the payload through `PrivacyGate` (`wifi-densepose-bfld`).** `BfldPipeline::process_to_frame` stamped the emitted `BfldFrame` header with the active `PrivacyClass` but serialized the caller-supplied `BfldPayload` **unchanged** via `BfldFrame::from_payload`. A frame labeled `Anonymous`(2) or `Restricted`(3) therefore carried the full identity-leaky `compressed_angle_matrix` (the beamforming-angle identity surface) + amplitude/phase proxies + `csi_delta` — exactly the sections `PrivacyGate::demote` is documented and tested (`privacy_gate_demote.rs`) to strip at those classes. Because a `NetworkSink` accepts class ≥ `Derived`(1), such a frame would publish the identity surface across the node boundary despite its restrictive class byte; the class byte lied about payload content. **Fix:** after building the frame at the active class, apply `PrivacyGate::demote` to the same class — a no-op class transition that strips the sections that class forbids (research classes `Raw`/`Derived` keep the full payload). Pinned by three fails-on-old tests in `pipeline_to_frame.rs` (`…_at_anonymous_strips_identity_leaky_sections`, `…_in_privacy_mode_strips_amplitude_and_phase` — both FAILED pre-fix; `…_at_derived_preserves_full_payload` guards against over-stripping). Grade: privacy-bypass FIXED + regression-pinned.
|
||||
- **ADR-157 Milestone-1 B4 - constant-time HMAC sync-beacon tag compare (`wifi-densepose-hardware`).** `AuthenticatedBeacon::verify` compared the 8-byte HMAC-SHA256 tag with `self.hmac_tag == expected`, which short-circuits on the first differing byte and leaks, through verification latency, how many leading bytes an attacker's forged tag matched - a byte-by-byte tag-recovery oracle (~256*N trials instead of 256^N). Replaced with a hand-rolled branch-free `constant_time_tag_eq` (XOR-accumulate every byte difference into a single `u8`, no early exit, `#[inline(never)]` + `core::hint::black_box` to stop the optimizer reintroducing a short-circuit or a non-constant-time `memcmp`). **No new dependency** - ADR-157 had deferred this only to avoid adding the `subtle` crate; a fixed 8-byte compare needs none. Grade MEASURED (constant-time *construction*; micro-timing on a noisy host is a smoke check only, gated `#[ignore]`). Pinned by `tag_compare_is_constant_time_shape` (equal/first-differ/last-differ/all-differ/length-mismatch + an end-to-end `verify()` last-byte tamper), proven to fail on a last-byte-skipping constant-time bug. ADR-157 §8 B4 -> RESOLVED.
|
||||
- **ADR-080 open HIGH findings closed on the Rust `wifi-densepose-sensing-server` boundary (ADR-164 G11).** The QE sweep's three HIGH findings — XFF-spoofing bypass, leaked stack traces, JWT-in-URL (CWE-598) — were logged against the Python v1 API and never re-verified against the shipped Rust sensing-server; the HOMECORE/M7 sweep (ADR-161) covered `homecore-server`, not this crate.
|
||||
- **#2 leaked internal errors (the one live exposure) — FIXED.** Six handlers in `main.rs` serialized the internal error `Display` straight into the JSON response body: `edge_registry_endpoint` returned a panicked `spawn_blocking` `JoinError` (`"task … panicked"`) in a `500`, plus the raw upstream error in a `503`; `delete_model`/`delete_recording`/`start_recording` returned `std::io::Error` strings (OS detail / path); `calibration_start`/`calibration_stop` returned the `FieldModel` error chain. New `error_response` module logs the full detail **server-side only** (with a correlation id) and returns a generic body (`{"error":"internal_error","correlation_id":…}`) — no `panicked`, no file paths, no Debug chain. 5 module tests (a leak-substring guard proven to fail on the reverted old body) + the existing handler suite.
|
||||
@@ -15,6 +41,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **#3 JWT-in-URL (CWE-598) — VERIFIED ABSENT, regression-pinned.** `require_bearer` reads the token only from the `Authorization` header; the WebSocket handlers take no token query param and the sole `Query` extractor (`EdgeRegistryParams`) is a non-secret `refresh` flag. Added a regression proving `?token=`/`?access_token=` in the URL never authenticates while the header path still does.
|
||||
|
||||
### Fixed
|
||||
- **`wifi-densepose-geo` numerical-robustness audit — `parse_hgt` degenerate-input panic FIXED + `haversine` antipodal NaN FIXED; pole-singularity & pointcloud NaN-state-poisoning confirmed clean (ADR-154-class sweep).** Targeted numerical-robustness audit of `wifi-densepose-geo` + `wifi-densepose-pointcloud`, hunting the proven non-finite-input-poisons-persistent-state class. **Two real bugs in `geo`, each pinned by a fails-on-old test.** (1) **`terrain.rs::parse_hgt` usize-underflow panic** — `side = sqrt(n_samples)`; for an empty / sub-2x2 buffer `side ≤ 1`, so `1.0 / (side - 1)` underflows `usize` (panic "attempt to subtract with overflow" in debug; wraps to a huge value in release → garbage/inf `cell_size_deg` that then poisons every `ElevationGrid::get` lookup). A truncated SRTM download, a 404 HTML body, or an empty response all reach `parse_hgt` — now `bail!`s with a clear error when `side < 2`. Pinned by `parse_hgt_empty_data_errors_not_panics` (panicked pre-fix) + `parse_hgt_single_sample_errors` (returned inf pre-fix) + a `parse_hgt_minimal_2x2_is_finite` guard. (2) **`coord.rs::haversine` asin-domain → NaN** — for (near-)antipodal points floating rounding can push `h.sqrt()` to `1.0 + ~4e-16`, and `asin(>1)` is NaN, silently breaking every downstream `<`/`>` distance comparison (verified: pair `(-44.4994,-178.95722)→(44.49939999,1.04278001)` yields `h=1.0000000000000004`). Fixed by clamping into `[0,1]` before `asin`. Pinned by `haversine_near_antipodal_is_finite_not_nan` (NaN pre-fix). The ±90° pole-singularity (`cos(lat)=0` division in the ENU transforms) is pinned as no-panic without changing the transform (value-identical for valid inputs). **`wifi-densepose-pointcloud` is confirmed-robust — no bug, no manufactured finding:** the only persistent auto-accumulating state (`occupancy` EMA, vitals) is fed exclusively from the integer-rssi/`sqrt`/`atan2` parser, which can only emit finite values, and the persistent state is provably self-healing even under an adversarial hand-built `CsiFrame` carrying NaN/inf amplitudes+phases (`motion_score=(NaN/100).min(1.0)→1.0`; breathing path `→0→clamp(5,40)→5.0`; tomography EMA uses only integer rssi). Pinned by `nonfinite_frame_does_not_poison_persistent_state` (injects 40 poisoned frames, asserts occupancy/vitals stay finite + the pipeline recovers) and three degenerate-voxel-fusion no-panic tests (empty/single/all-coincident). `wifi-densepose-geo --no-default-features`: 9→15 lib (+6), 8 integration unchanged; `wifi-densepose-pointcloud`: 18→22 (+4); 0 failed; workspace green; Python proof unchanged (`f8e76f21…46f7a`, bit-exact — both crates off the signal proof path).
|
||||
- **Vitals IIR filters self-heal after a non-finite CSI frame — a single NaN/inf no longer permanently kills breathing & heart-rate extraction (`wifi-densepose-vitals`, safety; ADR-021 / ADR-158 §A1).** The 2nd-order resonator in `breathing::BreathingExtractor::bandpass_filter` and `heartrate::HeartRateExtractor::bandpass_filter` latches each output `y[n]` into the filter state (`y1`/`y2`). A non-finite input — one NaN/inf amplitude residual from a corrupt CSI frame — produced a NaN `output` that was written into the state. The existing `extract()` `is_finite()` guard correctly dropped that single sample from history, **but never sanitized the poisoned filter state**, so every subsequent output stayed NaN, was rejected too, and the sliding-window history *never refilled*: the extractor went silently dead (returning `None` forever) until `reset()`. On the vitals alert path this is a safety-relevant denial of service — one bad frame and breathing **and** heart-rate monitoring stop, with no error surfaced. Fix: when `bandpass_filter` computes a non-finite `output` it now resets the IIR state to default and returns `0.0`, so the resonator recovers on the next clean frame (the `0.0` is still dropped by the caller's finite-check — no spurious sample enters history). Same class as the calibration NaN bug (ADR-154 §3) and the firmware vitals fixes (#998/#996/#987): the prior hardening guarded the *history boundary* but not the *filter-state boundary*. Pinned by `breathing::tests::nan_frame_does_not_permanently_poison_filter`, `breathing::tests::inf_mid_stream_does_not_freeze_history`, and `heartrate::tests::nan_frame_does_not_permanently_poison_filter` (all three FAIL on the pre-fix code, verified by reverting). Also de-magicked the safety-critical HR physiological plausibility band into named `HR_PLAUSIBLE_MIN_BPM`/`HR_PLAUSIBLE_MAX_BPM` consts (value-identical 40/180 BPM, pinned by `plausibility_band_constants_pinned`) and added a fabricated-vital negative (`pure_noise_is_never_reported_valid` — broadband noise never yields a clinically `Valid` HR). `wifi-densepose-vitals --no-default-features`: 55→60 lib tests, 0 failed; workspace green; Python proof unchanged (vitals is off the deterministic proof's signal path).
|
||||
- **BFLD MQTT `zone_activity` payload now JSON-escapes the zone name (`wifi-densepose-bfld`).** `mqtt_topics::render_events` emitted the zone payload as `format!("\"{zone}\"")` with no escaping, while `ha_discovery.rs` already escapes operator-controlled strings. A zone name containing a `"` or `\` produced malformed/injectable JSON on the Home-Assistant state topic (e.g. zone `a"b` → payload `"a"b"`). Added a `json_string_literal` escaper mirroring `ha_discovery::push_str_field` and applied it to the zone payload — value-identical for normal zone names (`living_room`, …). Pinned by `zone_payload_escapes_json_metacharacters` (FAILED pre-fix; round-trips through `serde_json`); the existing `zone_payload_is_json_string_with_quotes` still passes unchanged.
|
||||
- **ESP32 vitals: `n_persons` over-counted (reported 4 for one person) + presence flag flickered at close range (#998, #996).** Two firmware logic bugs in `firmware/esp32-csi-node/main/edge_processing.c`, both robustness/logic fixes — **not** validated-accuracy claims (true count/PCK vs labelled ground truth stays hardware/data-gated on the COM9 ESP32-S3).
|
||||
- **#998 over-count — root cause + fix.** `update_multi_person_vitals()` split the top-K subcarriers into `top_k_count/2` groups and marked **every** group `active` unconditionally, so one body's multipath always reported the full `EDGE_MAX_PERSONS` (=4). New pure, host-testable `count_distinct_persons()` gates each candidate group: (1) **energy gate** — a group's phase variance must be ≥ `EDGE_PERSON_MIN_ENERGY_RATIO` (0.35) × the strongest group's, so weak multipath echoes don't count; (2) **spatial dedup** — groups whose representative subcarriers sit within `EDGE_PERSON_MIN_SC_SEP` (4) of each other are the same body. A `person_count_debounce()` then requires the gated count to hold `EDGE_PERSON_PERSIST_FRAMES` (3) consecutive frames before it's emitted, so a single noisy frame can't promote a phantom. The strongest group always counts (a present body yields ≥1). All thresholds are named, documented constants in `edge_processing.h`.
|
||||
- **#996 presence flicker — root cause + fix.** Presence was a bare `score > threshold` compare on a noisy `presence_score` (field-observed 2.6–26.7 frame-to-frame for one stationary person), so the boolean chattered at the boundary while the score clearly indicated a person. New pure `presence_flag_update()` is a Schmitt trigger + clear-debounce: assert above `threshold`, **hold** in the dead band down to `threshold × EDGE_PRESENCE_HYST_RATIO` (0.5), and only clear after the score stays below the low threshold for `EDGE_PRESENCE_CLEAR_FRAMES` (5) consecutive frames. The score itself is unchanged (and still emitted at packet offset 20 for consumer-side thresholding). Constants named/documented in `edge_processing.h`.
|
||||
|
||||
@@ -22,6 +22,8 @@ Dual codebase: Python v1 (`v1/`) and Rust port (`v2/`).
|
||||
| `wifi-densepose-vitals` | ESP32 CSI-grade vital sign extraction (ADR-021) |
|
||||
| `nvsim` | Deterministic NV-diamond magnetometer pipeline simulator (ADR-089) — standalone leaf, WASM-ready |
|
||||
| `vendor/rvcsi` (submodule) | **rvCSI** — edge RF sensing runtime (ADR-095/096): 9 crates (`rvcsi-core`/`-dsp`/`-events`/`-adapter-file`/`-adapter-nexmon`/`-ruvector`/`-runtime`/`-node`/`-cli`). Lives in its own repo ([github.com/ruvnet/rvcsi](https://github.com/ruvnet/rvcsi)), vendored here under `vendor/rvcsi`, published to crates.io as `rvcsi-* 0.3.x` and to npm as `@ruv/rvcsi`. Not a `v2/` workspace member — depend on the published crates (or the submodule's `crates/rvcsi-*` paths). Normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, validate-before-FFI, reusable DSP, typed confidence-scored events, the napi-c Nexmon shim (real nexmon_csi `.pcap` from a Raspberry Pi 5 / 4 / 3B+ — BCM43455c0), the napi-rs SDK, the `rvcsi` CLI, a Claude Code plugin. |
|
||||
| `vendor/rufield` (submodule) | **RuField MFS** — the open spec for camera-free multimodal field sensing (ADR-260). A common `FieldEvent`/`FieldTensor`/`FusionGraph`/`PrivacyClass`/`ProvenanceReceipt` model *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and quantum sensors. Lives in its own repo ([github.com/ruvnet/rufield](https://github.com/ruvnet/rufield)), vendored here under `vendor/rufield`. Not a `v2/` workspace member. v0.1 reference stack = 7 crates (`rufield-core`/`-provenance`/`-privacy`/`-adapters`/`-fusion`/`-bench`/`-viewer`), 72 tests/0 failed; `rufield-viewer` is an Axum + vanilla-JS read-only dashboard (`cargo run -p rufield-viewer`) completing ADR-260 §27.9. The WiFi-CSI modality is now **real-replay-backed** via `CsiReplayAdapter` (ingests real captured `.csi.jsonl` → fused presence/breathing inferences; replay-from-file, unlabeled CSI-variance proxy, not validated accuracy); mmWave/thermal + all synthetic-bench F1 numbers remain **SYNTHETIC** (no live hardware — live streaming + labeled accuracy are roadmap). |
|
||||
| `wifi-densepose-rufield` | ADR-262 P1 **anti-corruption bridge** — converts RuView WiFi-CSI sensing output (`SensingSnapshot` mirroring `SensingUpdate` + `TrustedOutput`, owned primitives, no dep on `wifi-densepose-sensing-server`) into **signed RuField `FieldEvent`s** (`Modality::WifiCsi`, real `timestamp_ns`, sha256 + ed25519 provenance, `synthetic=false`). The single coupling point between RuView and the standalone RuField MFS spec (§5.4); path-deps the `vendor/rufield` submodule crates (`rufield-core`/`-provenance`/`-privacy`/`-fusion`). **Critical §3.3 privacy mapping** (`map_privacy`): maps RuView class → RuField P0–P5 by **information content, never byte value**, fail-closed (`Derived → P4/P5`, never P1; `demoted` floors to ≥ P2). 15 tests / 0 failed (round-trip / `is_fusable` / fusion-ingest / privacy-safety / determinism). P1 plumbing — not wired into the live server (P3), no accuracy claim. |
|
||||
| `ruview-swarm` | Drone swarm control system (ADR-148) — hierarchical-mesh topology, Raft consensus, MARL, CSI sensing payload, MAVLink/PX4 compat, Ruflo AI-agent integration |
|
||||
|
||||
### RuvSense Modules (`signal/src/ruvsense/`)
|
||||
|
||||
@@ -14,6 +14,13 @@ COPY v2/crates/ ./crates/
|
||||
# Copy vendored RuVector crates
|
||||
COPY vendor/ruvector/ /build/vendor/ruvector/
|
||||
|
||||
# Copy vendored RuField submodule — the `wifi-densepose-rufield` bridge crate
|
||||
# (ADR-262) path-deps `../../../vendor/rufield/crates/*`, which from the Docker
|
||||
# build layout (v2/ collapsed into /build) resolves to /vendor/rufield. Copy the
|
||||
# whole tree so the rufield workspace Cargo.toml (workspace-dep inheritance) and
|
||||
# the four bridged crates (rufield-core/-provenance/-privacy/-fusion) all resolve.
|
||||
COPY vendor/rufield/ /vendor/rufield/
|
||||
|
||||
# Build release binaries:
|
||||
# - sensing-server with `mqtt` feature so the HA-DISCO MQTT publisher
|
||||
# (ADR-115) is wired in (auto-discovery topics flow to Home Assistant)
|
||||
|
||||
@@ -1092,6 +1092,12 @@ Two robustness bugs were fixed in the on-device edge path (`firmware/esp32-csi-n
|
||||
|
||||
Both are pinned by host-buildable C99 tests in `firmware/esp32-csi-node/test/test_vitals_count_presence.c` (`make run_vitals`). The exact thresholds are documented constants pending on-device calibration against ground truth.
|
||||
|
||||
### 2026-06 — Rust `wifi-densepose-vitals`: IIR filter NaN/inf self-heal (ADR-158 §A1)
|
||||
|
||||
A correctness/safety review of the Rust extraction crate found a real bug parallel to the firmware robustness class above. The 2nd-order resonator `bandpass_filter` in both `breathing.rs` and `heartrate.rs` latches each output `y[n]` into its filter state (`y1`/`y2`). A single non-finite amplitude residual from a corrupt CSI frame produced a NaN `output` that was written into the state; the existing `extract()` `is_finite()` guard dropped that one sample from the history buffer **but never sanitized the poisoned filter state**, so every later output stayed NaN, was rejected too, and the sliding-window history never refilled — breathing **and** heart-rate extraction went silently dead (returning `None` forever) until `reset()`. On the alert path this is a safety-relevant denial of service (one bad frame stops vitals monitoring with no error surfaced).
|
||||
|
||||
Fix: when `bandpass_filter` computes a non-finite `output`, it resets the IIR state to default and returns `0.0`, so the resonator self-heals on the next clean frame (the `0.0` is still dropped by the caller's finite-check, so no spurious sample enters history). Same shape as the calibration NaN bug (ADR-154 §3) — the prior hardening guarded the *history boundary* but not the *filter-state boundary*. Pinned by `breathing::tests::nan_frame_does_not_permanently_poison_filter`, `breathing::tests::inf_mid_stream_does_not_freeze_history`, and `heartrate::tests::nan_frame_does_not_permanently_poison_filter` (all FAIL pre-fix, verified by reverting). The review also de-magicked the HR physiological plausibility band into named `HR_PLAUSIBLE_MIN_BPM`/`HR_PLAUSIBLE_MAX_BPM` consts (value-identical 40/180 BPM) and added a fabricated-vital negative (`pure_noise_is_never_reported_valid` — broadband noise never yields a clinically `Valid` HR; the extractor honestly returns low-confidence `Unreliable`). Clean dimensions confirmed with evidence: flat/silent input → `None`; pure noise → low-confidence `Unreliable`, never `Valid`; harmonic-rich breathing with no cardiac component → low-confidence, not a confident false HR; out-of-band BPM rejected by the plausibility clamp.
|
||||
|
||||
## References
|
||||
|
||||
- Ramsauer et al. (2020). "Hopfield Networks is All You Need." ICLR 2021. (ModernHopfield formulation)
|
||||
|
||||
@@ -104,6 +104,57 @@ Ranked by build cost × user impact:
|
||||
| **P9** | HACS integration repo (`hass-wifi-densepose`) for HA-side install path | pending |
|
||||
| **P10** | Witness bundle + CSA-style spec compliance check | pending |
|
||||
|
||||
## 4.1 Crypto/security review notes (§2.2 witness chain — ADR-262 P2 prerequisite)
|
||||
|
||||
Beyond-SOTA crypto+security review of the SHA-256 + Ed25519 witness chain
|
||||
(`witness.rs` / `witness_signing.rs`) and the manifest signature surface
|
||||
(`manifest.rs`), because ADR-262 P2 proposes to **reuse this exact signing
|
||||
chain**. Top priority was the sibling `wifi-densepose-engine` bug class —
|
||||
unframed boundary-to-boundary concatenation of operator-influenceable strings
|
||||
into a signed/hashed digest.
|
||||
|
||||
- **Engine bug class ABSENT (good result, reported with byte evidence).**
|
||||
`canonical_bytes` is `DOMAIN_TAG ‖ prev_hash[32] ‖ seq:u64-be ‖ ts:u64-be ‖
|
||||
kind_len:u32-be ‖ kind ‖ payload_len:u32-be ‖ payload`. The two
|
||||
variable-length operator-influenceable fields (`kind`, `payload`) are
|
||||
**length-prefixed**; the fixed-width fields are self-delimiting → the
|
||||
encoding is injective (no two distinct event tuples share a preimage). The
|
||||
Ed25519 signature signs the **identical** bytes the SHA-256 chain commits to.
|
||||
No separate unframed concatenation exists; the manifest `binary_signature`
|
||||
is signed at build time (Makefile) over a single fixed-length `binary_sha256`
|
||||
hex value, not in-crate.
|
||||
|
||||
- **CHM-WIT-01 (FIXED) — domain-separation tag added.** The engine fix
|
||||
prescribed *domain-tag + length-prefix*; length-prefix was present, the
|
||||
domain tag was not. Added a versioned, NUL-terminated
|
||||
`WITNESS_DOMAIN_TAG = b"cog-ha-matter/witness-event/v1\x00"` prefix so the
|
||||
witness message can never be replayed as a message for another Ed25519
|
||||
context that shares key infrastructure (notably the manifest signature).
|
||||
**Witness bytes change by design** (prior on-disk hashes/signatures
|
||||
invalidated, as with the engine fix); verified safe because no in-repo crate
|
||||
consumes cog-ha-matter witness bytes programmatically (doc-mentions only).
|
||||
|
||||
- **CHM-WIT-02 (HARDENED) — `verify_signature` now uses `verify_strict`.** For
|
||||
an audit chain the signature is the attestation, so non-canonical encodings
|
||||
and small-order keys are rejected (RFC 8032 strict), giving the "one
|
||||
canonical signature per event" property. Not a forgery fix — the verifying
|
||||
key is caller-pinned, never read from the event.
|
||||
|
||||
- **Confirmed clean (with evidence):** verify-before-trust + key-pinning
|
||||
(`verify_signature` takes the verifying key as a parameter; `read_jsonl`
|
||||
re-derives every hash and chain-verifies); key handling (the crate never
|
||||
generates/stores/logs/serializes a signing key — only a documented test-only
|
||||
fixed seed; production keys come from the Seed secure store, out of scope);
|
||||
determinism (positional bytes, deterministic Ed25519, alphabetically-locked
|
||||
JSONL field order, sorted TXT records — no HashMap/float nondeterminism feeds
|
||||
any digest); fail-closed parsing (structured errors, no panics; `main.rs`
|
||||
reads no untrusted files/paths).
|
||||
|
||||
Tests: `cog-ha-matter --no-default-features` 64 → **68**, 0 failed (CHM-WIT-01
|
||||
pinned by 4 fails-on-old tests across `witness.rs`/`witness_signing.rs`;
|
||||
CHM-WIT-02 guarded by a key-pinning test). Python deterministic proof
|
||||
unchanged (cog-ha-matter is off the signal proof path).
|
||||
|
||||
## 5. References
|
||||
|
||||
- ADR-101 — `cog-pose-estimation` packaging precedent (signed binaries on GCS, .cog manifest)
|
||||
|
||||
@@ -190,6 +190,23 @@ This is the same Wasmtime host already used for integration plugins (ADR-128)
|
||||
|
||||
---
|
||||
|
||||
## 8a. Security review (beyond-SOTA sweep, post ADR-154–159)
|
||||
|
||||
A focused security review of `homecore-automation` (the execution/eval surface — triggers → conditions → actions, with templates) was run after the ADR-154–159 sweep, applying the same rigor that the sibling engine/bfld/calibration/vitals/geo reviews used. **Two real DoS findings, each pinned by a fails-on-old test; the condition-bypass, fail-closed-parsing, and action-authorization dimensions were probed and found clean.**
|
||||
|
||||
- **HC-SEC-01 (template-injection / unbounded-expansion DoS, HIGH) — FIXED.** A `template:` condition / `value_template` is user automation config, and was rendered with MiniJinja's defaults: **no instruction budget, no output cap**. A single condition such as `{% for i in range(5000) %}{% for j in range(5000) %}xxxx{% endfor %}{% endfor %}` rendered a **100 MB string over ~11 s on one render call** (measured) — a CPU/memory denial of service (the bfld-class "unbounded expansion"; MiniJinja's per-call `range()` 10k cap does **not** stop nested loops). **Fix:** enable MiniJinja's `fuel` feature and set a per-render budget (`set_fuel(Some(1_000_000))`) so a nested loop burns one unit per iteration — the attack now fails fast (~90 ms) with "engine ran out of fuel"; plus a 64 KiB source-length cap rejecting pathological sources before compilation. Legitimate HA templates (a few dozen instructions) are unaffected. Pinned by `nested_loop_template_is_bounded_not_unbounded_dos`, `single_huge_repeat_template_is_bounded`, `oversized_template_source_is_rejected` (all fail-on-old: unbounded render / no rejection), and `legitimate_template_still_renders_within_fuel` (no regression).
|
||||
- **HC-SEC-02 (panic-on-config DoS, MEDIUM) — FIXED.** `Action::Delay { seconds }` and `Action::WaitForTrigger { timeout_seconds }` fed the user-supplied float straight into `Duration::from_secs_f64`, which **panics** on negative, NaN, infinite, or overflowing inputs — all reachable from a crafted (or typo'd) YAML (`delay: {seconds: -1}`, `.nan`, `.inf`, `1e308`). One hostile config aborts the spawned automation run task with a panic (measured: "cannot convert float seconds to Duration: value is negative"). **Fix:** a `safe_duration_from_secs` guard that saturates instead of panicking (NaN/±inf/negative → `Duration::ZERO`, matching HA's lenient "non-positive delay = no delay"; absurdly large → clamped to ~100 years). Pinned by `delay_negative_seconds_does_not_panic`, `delay_nan_seconds_does_not_panic`, `delay_infinite_seconds_does_not_panic`, `wait_for_trigger_negative_timeout_does_not_panic`, `safe_duration_saturates_hostile_values` (incl. overflow clamp).
|
||||
|
||||
**Dimensions confirmed clean (with evidence):**
|
||||
- **Condition bypass / fail-closed eval** — a `Condition::Template` whose render errors evaluates to `false` (`condition.rs` `Err(_) => false`), and a `Choose` branch condition that fails to deserialize is treated as **non-matching** (the branch is skipped), not silently passing (`action.rs` `ChoiceBranch::matches` `Err(_) => return false`). Both fail **closed** (do-not-run), confirmed by the existing `choose_*` tests and template-false-blocks-action behavioral test. No true-by-default-on-parse-error path found.
|
||||
- **Re-entrancy / livelock (DoS)** — run-mode machinery is bounded and tested: `Single`/`IgnoreFirst` re-entrancy guard, `Restart` cancel-and-replace, `Queued` FIFO serialization, and `max: N` semaphore cap (ADR-162; `restart_mode_cancels_prior_run`, `queued_mode_runs_sequentially_not_concurrently`, `max_two_caps_concurrency_at_two`, `single_mode_does_not_double_fire_on_rapid_triggers`). A self-triggering automation does not livelock the engine — each fire is bounded by its run-mode.
|
||||
- **Action authorization** — templates are read-only sandboxed (`states`/`state_attr`/`is_state`/`now` globals; no service-call or state-set global is exposed to template scope), so a template cannot escalate into an action. Service authorization itself is enforced at the `homecore` service-registry boundary (out of this crate's scope); no gap found in what the automation crate enforces.
|
||||
- **Panic-on-config (parse)** — `serde_yaml`/`serde_json` deserialization returns structured `AutomationError` (no `unwrap`/`expect`/index reachable from a crafted config in the eval/exec path); the only remaining panic surface was the `from_secs_f64` path fixed as HC-SEC-02.
|
||||
|
||||
Validation: `cargo test -p homecore-automation --no-default-features` → 54 passed / 0 failed (+14 over baseline). Python deterministic proof unchanged (homecore-automation is off the signal-processing proof path).
|
||||
|
||||
---
|
||||
|
||||
## 9. References
|
||||
|
||||
### HA upstream
|
||||
|
||||
@@ -120,6 +120,42 @@ tested; P3 is planned.
|
||||
HOMECORE-API (ADR-130, P3); automation conditions on historical state are
|
||||
HOMECORE-automation (ADR-129, P3).
|
||||
|
||||
## 3a. Security review (2026-06, post-ADR-154–159 sweep)
|
||||
|
||||
A beyond-SOTA security review of `homecore-recorder` covered SQL injection, retention/purge
|
||||
correctness, fail-closed write integrity, semantic-store NaN poisoning, and PII exposure.
|
||||
|
||||
**Confirmed clean (with evidence):**
|
||||
|
||||
- **SQL injection — clean.** Every query in `db.rs` uses bound `?` parameters; no user- or
|
||||
entity-influenceable value is interpolated into SQL via `format!`/concatenation. The only
|
||||
`format!` builds the `LIKE` *pattern* string, which is itself **bound** as a parameter with
|
||||
`ESCAPE '\\'` and `% _ \` escaping — so a metacharacter payload is matched literally. Pinned
|
||||
by `malicious_entity_id_is_stored_literally_not_executed` (a `'; DROP TABLE states; --` state
|
||||
value leaves the table intact and round-trips verbatim) and
|
||||
`like_metacharacters_in_query_are_literal_not_wildcards`.
|
||||
- **NaN-index poisoning — structurally impossible.** Embeddings are SHA-256 → `i32` →
|
||||
`f32`; an `i32`→`f32` cast is always finite (never NaN/Inf), and an all-zero-digest is
|
||||
guarded by the `norm > 1e-10` check. Empty-index search, empty-string query, and `k=0` were
|
||||
probed and all return `Ok(0)` with no panic. (Unlike the calibration/vitals/geo paths, no raw
|
||||
sensor float ever reaches the index.)
|
||||
- **Fail-closed writes.** A removal event returns `Ok(None)`; semantic-index failure is logged,
|
||||
not propagated, so it never blocks the durable SQLite write; `EntityId` parse failure falls
|
||||
back to a sentinel rather than panicking.
|
||||
|
||||
**Fixed (real bounding bugs):**
|
||||
|
||||
- **Memory-DoS — `get_state_history` was unbounded.** No `LIMIT`, so a wide time window over a
|
||||
high-frequency entity loaded an unbounded row set into memory. Now capped at
|
||||
`MAX_HISTORY_ROWS` (1,000,000); sibling search paths were already `k`-bounded.
|
||||
- **Disk-DoS / documented-but-missing `purge`.** The README advertised `Recorder::purge`, but
|
||||
no retention path existed → unbounded disk growth. Added a **transactional** `purge(older_than)`
|
||||
with an **exclusive** cutoff (idempotent, no off-by-one) that deletes old `states`/`events` and
|
||||
GCs orphaned `state_attributes` blobs (dedup-shared blobs kept until their last referrer is gone).
|
||||
|
||||
`homecore-recorder` tests: 19 → 25 (`--no-default-features`) / 25 → 31 (`--features ruvector`),
|
||||
0 failed. Python deterministic proof unchanged (recorder is off the signal proof path).
|
||||
|
||||
## 4. Links
|
||||
|
||||
- Crate: `v2/crates/homecore-recorder/` — `Cargo.toml`, `README.md`, `src/lib.rs`,
|
||||
|
||||
@@ -495,3 +495,34 @@ Rejected. `ViewpointFusionEvent` (viewpoint/fusion.rs lines 183–219) is an int
|
||||
**Integration glue -- not yet on the live path:** emission of `CalibrationIdMismatch` / `DriftProfileConflict` / `PhaseAlignmentFailed` once `calibration_id` propagation and the phase-align convergence signal are threaded onto frames; the BFLD witness record emitted on privacy demotion.
|
||||
|
||||
**Trust contribution:** sensor *agreement made explicit* -- fusion records the evidence it relied on, and any disagreement automatically tightens the downstream privacy class.
|
||||
|
||||
---
|
||||
|
||||
## Witness Integrity Review (2026-06-14) — domain-separation fix
|
||||
|
||||
A beyond-SOTA security review of `wifi-densepose-engine` (the composition root
|
||||
that builds the §2.7 trust witness in `witness_of`) found a real **witness
|
||||
domain-separation gap**, now fixed.
|
||||
|
||||
**Finding (witness-gap, HIGH).** `witness_of` concatenated `model_version`,
|
||||
`calibration_version`, and `privacy_decision` boundary-to-boundary, and the
|
||||
variable-length `evidence` list carried no explicit count. A string straddling a
|
||||
field boundary therefore collided with a *different* trust decision —
|
||||
e.g. a per-room adapter id (ADR-150 §3.4, operator-influenceable) that absorbs
|
||||
the leading bytes of the calibration epoch (`model="…cal:00a"`, `cal="b"`)
|
||||
produces the **same** witness as `model="…"`, `cal="cal:00ab"`. Two distinct
|
||||
privacy-relevant input tuples → one witness defeats the "any privacy-relevant
|
||||
delta → different witness" guarantee this ADR's §2.7 witness exists to provide.
|
||||
|
||||
**Fix.** The witness now (a) prepends a domain tag `ruview.engine.witness.v1`,
|
||||
(b) writes an explicit 8-byte evidence count, and (c) **length-prefixes every
|
||||
field** (8-byte LE length ‖ bytes), so field framing is unambiguous regardless
|
||||
of contents. This is a witness-layout change (all prior witness bytes are
|
||||
invalidated by design); downstream consumers only assert witness *relationships*
|
||||
(`assert_ne`/`assert_eq` across runs), not absolute bytes, so nothing breaks.
|
||||
|
||||
Pinned by `witness_distinguishes_model_calibration_boundary` and
|
||||
`witness_distinguishes_evidence_model_boundary` (both fail on the old
|
||||
concatenation). Witness **determinism** was reviewed and confirmed clean: no
|
||||
HashMap iteration and no float formatting feed the hash (floats appear only in
|
||||
the `SemanticState` statement, which is outside the witness).
|
||||
|
||||
@@ -599,3 +599,53 @@ Per ADR-028/ADR-010, three rows are added to the witness log:
|
||||
**Integration glue -- not yet on the live path:** wiring the registry into `PrivacyGate` class transitions, the MQTT discovery payload, and a read-only Home Assistant diagnostic entity exposing the active mode + proof hash.
|
||||
|
||||
**Trust contribution:** the *policy spine* -- privacy posture is a tamper-evident, auditable chain rather than a checkbox; an operator's mode choice actively governs whether identity data may even exist.
|
||||
|
||||
---
|
||||
|
||||
## Privacy Monotonicity Review (2026-06-14) — confirmed clean
|
||||
|
||||
A beyond-SOTA security review of the governed-trust cycle
|
||||
(`wifi-densepose-engine::StreamingEngine::process_cycle_calibrated`) examined
|
||||
the privacy-demotion path this ADR governs. **The monotonicity invariant holds:
|
||||
demotion only ever makes the emitted class more restrictive, never less.**
|
||||
|
||||
Verification (no behaviour change, the result is a clean bill with evidence):
|
||||
|
||||
- Each cycle computes `effective_class` fresh from the active mode's
|
||||
`target_class()` (the floor) and applies at most a **single-step** demotion
|
||||
(`demote_one`, clamped at `Restricted`). There is no cross-cycle state that
|
||||
could let a permissive class overwrite a restrictive one.
|
||||
- A forced contradiction (calibration mismatch / array-geometry insufficiency /
|
||||
mesh partition risk, ADR-032) raises the class byte; a clean cycle emits
|
||||
exactly the base class.
|
||||
- Pinned by `forced_contradiction_never_relaxes_class`, a property test over
|
||||
**all five** `PrivacyMode`s asserting `effective_class.as_u8() >=
|
||||
base_class.as_u8()` (strictly greater unless already clamped at `Restricted`)
|
||||
under a forced contradiction, and `== base` on a clean cycle.
|
||||
|
||||
Fail-closed boundaries were also pinned: an empty cycle errors (no degenerate
|
||||
over-permissive output, `empty_cycle_fails_closed`) and the single-node boundary
|
||||
is characterized as a valid non-demoting mode (`single_node_cycle_is_well_formed`).
|
||||
|
||||
The related witness domain-separation fix from the same review is recorded in
|
||||
ADR-137 (the witness folds `effective_class`, so the demotion is auditable).
|
||||
## Security & Privacy Review (2026-06-14)
|
||||
|
||||
Beyond-SOTA privacy+security review of `wifi-densepose-bfld` (the crate was not in the ADR-154–159 sweep). Two real bugs fixed (each pinned by a fails-on-old test), several dimensions confirmed clean.
|
||||
|
||||
### Findings
|
||||
|
||||
| # | Severity | Site | Issue | Fix | Pinned by |
|
||||
|---|----------|------|-------|-----|-----------|
|
||||
| 1 | **privacy-bypass (HIGH)** | `pipeline.rs::process_to_frame` | The documented wire-bytes production path stamped the frame header with the active `PrivacyClass` but serialized the caller's `BfldPayload` **unchanged** via `BfldFrame::from_payload` — never routing through `PrivacyGate::demote`. A frame labeled `Anonymous`(2)/`Restricted`(3) carried the full `compressed_angle_matrix` (identity surface) + amplitude/phase + `csi_delta`. A `NetworkSink` accepts class ≥ `Derived`(1), so the identity surface could cross the node boundary despite the restrictive class byte — the byte lied about content. | Apply `PrivacyGate::demote(frame, active_class)` after construction: a same-class transition that strips the sections the class forbids; `Raw`/`Derived` keep the full payload. | `tests/pipeline_to_frame.rs::process_to_frame_at_anonymous_strips_identity_leaky_sections`, `…_in_privacy_mode_strips_amplitude_and_phase` (both FAILED pre-fix); `…_at_derived_preserves_full_payload` (over-strip guard) |
|
||||
| 2 | **PII/injection (MEDIUM)** | `mqtt_topics.rs::render_events` | `zone_activity` payload built as `format!("\"{zone}\"")` with no JSON escaping (while `ha_discovery.rs` already escapes). A zone name with `"`/`\` produced malformed/injectable JSON on the HA state topic. | `json_string_literal()` escaper mirroring `ha_discovery::push_str_field`. Value-identical for normal zone names. | `tests/mqtt_topic_routing.rs::zone_payload_escapes_json_metacharacters` (FAILED pre-fix) |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **Event-field privacy gating** — `BfldEvent::apply_privacy_gating` nulls `identity_risk_score` + `rf_signature_hash` at `Restricted`, and `serde(skip_serializing_if = "Option::is_none")` omits them entirely. `render_events`/`render_discovery_payloads` refuse class < `Anonymous` (stricter than the `sink.rs` `NetworkKind` `MIN_CLASS = Derived` — defense in depth toward less leakage). Covered by `event_privacy_gating.rs`, `mqtt_topic_routing.rs`, `ha_discovery.rs`.
|
||||
- **Witness/hash framing (the engine `witness_of` bug class)** — CLEAN. `SignatureHasher::compute` prefixes a **fixed 4-byte** `day_epoch` then a **fixed-width canonical-f32** feature block (`IdentityFeatures`: Embedding = `EMBEDDING_DIM*4`, RiskFactors = 16 B). `PrivacyAttestationProof::compute` hashes a fixed 32-byte `prev_hash` + three fixed 1-byte values. No variable-length operator-influenceable string is concatenated into any digest — no length-prefix-framing collision is possible.
|
||||
- **Fail-closed** — `payload.rs::from_bytes` rejects truncated/overflowing/trailing-byte sections (`checked_add`, bounds checks); `frame.rs::from_bytes` validates magic/version/length/CRC; `PrivacyClass::try_from` rejects unknown bytes; `identity_risk::score` maps NaN/degenerate factors → 0.0 (privacy-conservative). The `from_score(NaN) → Accept` choice is a documented, deliberate publish-aggregate-only fallback (NaN never reaches it from `score()`); risk-driven NaN cannot leak identity because identity gating is class-byte-driven, not risk-driven.
|
||||
|
||||
### Observation (not a bug)
|
||||
|
||||
The ADR-141 control plane (`PrivacyMode`/`PrivacyModeRegistry`) is **not yet wired into the emit path** — the emitter/pipeline enforce the raw `PrivacyClass` directly; the registry is exported + unit-tested but advisory. This matches the "Integration glue — not yet on the live path" status above. The class-byte enforcement (emitter + event + renderers + the now-fixed `process_to_frame`) is the live guarantee. Wiring the registry is the documented next step.
|
||||
|
||||
@@ -253,6 +253,54 @@ Validation per CLAUDE.md: `cargo test --workspace --no-default-features` green;
|
||||
|
||||
---
|
||||
|
||||
## 6. Review notes
|
||||
|
||||
### 6.1 Correctness + security review (2026-06-14)
|
||||
|
||||
Beyond-SOTA correctness+security review of `wifi-densepose-calibration` (this
|
||||
ADR's pipeline), un-covered by the ADR-154–159 sweep.
|
||||
|
||||
**Finding (FIXED) — NaN-poisoning of the feature path (numerical / fail-closed).**
|
||||
`Features::from_series` — the carrier for both live inference and training-anchor
|
||||
extraction — computed `mean`/`variance`/`motion` over the raw scalar series with
|
||||
no non-finite guard. A single `NaN`/`±inf` sample (corrupt CSI frame) yielded
|
||||
`mean=NaN, variance=NaN` and an all-`NaN` prototype embedding. Persisted into a
|
||||
`PresenceSpecialist::threshold`/`empty_mean` at train time, the `NaN` **silently
|
||||
disabled presence detection** for the bank's lifetime (every `>` / `|·|`
|
||||
comparison against `NaN` is false → always reads *absent*, confidence 0), with no
|
||||
error — and an asymmetry against the rigorously NaN-guarded `geometry_embedding`.
|
||||
Fixed at the production boundary: non-finite samples are dropped (a corrupt frame
|
||||
counts as no frame), an all-non-finite series degrades to `Features::ZERO` like
|
||||
the empty series. Value-identical for all-finite input (full-loop + extract tests
|
||||
unchanged); pinned by `non_finite_samples_do_not_poison_features` and
|
||||
`all_non_finite_series_is_zero` (both fail on the old code).
|
||||
|
||||
**Clean dimensions (evidence, no invented issues).**
|
||||
- *File/path handling:* the crate performs **zero** file/path I/O (no
|
||||
`std::fs`/`Path`/`File`/`read`/`write` in `src/`; only in-memory `serde_json`).
|
||||
Path-traversal / unbounded-read / artifact-path handling live entirely in the
|
||||
`wifi-densepose-cli` consumer (`room.rs`), outside this crate's boundary.
|
||||
- *Untrusted-load:* `SpecialistBank::from_json` shape-validates via serde
|
||||
(malformed → `CalibrationError::Serde`); banks are local-first (invariant B),
|
||||
never network-received. A well-formed bank with adversarial numerics is trusted
|
||||
as-is — acceptable under the local-first threat model; a validate-on-load
|
||||
defense-in-depth pass is a possible future hardening, not a present bug.
|
||||
- *Receipt/hash integrity:* the crate emits no hash/receipt/witness/signature, so
|
||||
the unframed-concatenation bug class (cf. the engine `witness_of` fix) is
|
||||
structurally absent.
|
||||
- *Other numerical paths:* `geometry_embedding` sanitizes every input and sweeps
|
||||
to finite; presence/restlessness/anomaly divisions are `.max(1e-3)`-guarded;
|
||||
`autocorr_dominant` guards `r0`, short signals, and empty bands; `train` rejects
|
||||
empty anchors; anomaly requires ≥2 anchors.
|
||||
|
||||
De-magicked the bare specialist threshold literals (breathing/heartbeat default
|
||||
min-scores, anomaly outlier-spread multiple + label cutoff) into named documented
|
||||
consts, value-identical, pinned by const-equality tests. Tests
|
||||
**58→62 unit + 1 integration, 0 failed**; Python deterministic proof unchanged
|
||||
(off the signal proof path).
|
||||
|
||||
---
|
||||
|
||||
## 5. Summary
|
||||
|
||||
> Big models understand the world. Small ruVector models understand *your room*.
|
||||
|
||||
@@ -231,6 +231,8 @@ Catalogued so nothing is silently dropped. Priority: **P1** correctness-adjacent
|
||||
|
||||
> **Horizon-ledger one-liner.** Milestone-0 DONE: dead CIR gate (FIXED+proved), NaN/inf adversarial bypass (FIXED+proved), divide-by-(n−1) window trio (FIXED+proved), calibration dead-branch (FIXED), PSD FFT-planner cache (MEASURED), DTW band (MEASURED). **Milestone-1 DONE (2026-06-13): all four P1 backlog items cleared — circular phase variance #1 (RESOLVED/MEASURED metric, DATA-GATED threshold), Welford n=0 guard #10 (RESOLVED/MEASURED), threshold magic-constants #9 & #13 (RESOLVED-PARTIAL/DATA-GATED — de-magicked + boundary-tested, values unchanged).** **Milestone-2 DONE (2026-06-13): bench-first P2 perf subset + missing boundary tests cleared — spectrogram per-subcarrier FFT re-plan #20 (MEASURED-HOT, 1.40–1.84×, bit-identical); attention/tomography/Kalman #5/#6/#7 (MEASURED-NULL — benched, not hot, left as-is); field_model eigendecompose #8 (MEASUREMENT-ONLY, BLAS un-buildable on this Windows host, number deferred to a BLAS box, NOT fabricated); fft_operator tolerance #14, phase-align convergence-cap #16, csi-ratio epsilon #19 (RESOLVED, tests added).** **Milestone-3 DONE (2026-06-13): the lumped §7.4 row #21–45 P3 backlog cleared, and with it residual P3 items #2/#12/#17/#18 — 22 magic constants de-magicked into named EMPIRICAL-DEFAULT consts (each pinned == prior literal) + 6 boundary/characterization tests across 11 modules; ~4 doc-only; not-real findings (unreachable attractor_drift div0, non-existent gesture thresholds, proof-path features.rs) reported + skipped, no churn; no operating value changed; workspace 3,275/0, Python proof bit-exact `f8e76f21…`.** **§7.4 deferred backlog is now FULLY CLEARED across M0–M3 — nothing silently dropped.**
|
||||
|
||||
> **Sibling-crate sweep extension (2026-06-14) — `wifi-densepose-geo` + `wifi-densepose-pointcloud`.** The ADR-154-class numerical-robustness sweep (non-finite-input-poisons-persistent-state + divide-by-zero / asin-domain / degenerate-geometry) was extended to two crates *outside* this ADR's signal scope. **Two real `geo` bugs FIXED, each fails-on-old-pinned:** `terrain.rs::parse_hgt` usize-underflow panic on empty/sub-2x2 SRTM data (`1.0/(side-1)` → panic in debug / inf `cell_size_deg` poisoning `ElevationGrid::get` in release — a truncated download / 404 HTML body reaches it; now `bail!`s when `side < 2`); `coord.rs::haversine` `asin(>1)→NaN` for near-antipodal points (`h` rounds to `1.0+4e-16`; clamped to `[0,1]`). The ±90° pole `cos(lat)=0` ENU singularity is pinned no-panic without changing the transform. **`pointcloud` is confirmed-robust (no manufactured finding):** its only persistent auto-accumulating state (`occupancy` EMA + vitals) is fed solely by the integer-rssi/`sqrt`/`atan2` parser (always finite) and is provably self-healing even under an adversarial NaN/inf `CsiFrame` (`motion_score=(NaN/100).min(1.0)→1.0`; breathing `→0→clamp(5,40)→5.0`) — pinned by `nonfinite_frame_does_not_poison_persistent_state` + degenerate-voxel-fusion no-panic tests. `geo` 9→15 lib / 8 integration; `pointcloud` 18→22; 0 failed; workspace green; Python proof bit-exact `f8e76f21…`. See CHANGELOG `[Unreleased] → Fixed`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Consequences
|
||||
|
||||
@@ -102,7 +102,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
| # | Candidate | What | Grade | Verdict |
|
||||
|---|-----------|------|-------|---------|
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **CLAIMED** (author-measured; **not reproduced on our hardware** — reproduction is future work) | **Lead beyond-SOTA candidate for the ruvector ANN path.** Propose as ACCEPTED-future; cite honestly as "claimed by source, reproduction pending." Best fit because the ruvector retrieval path (AETHER re-ID, sketch prefilter) is exactly an ANN problem and SymphonyQG is CPU/edge-portable like our deployment. |
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **MEASURED-direction-tested** (was CLAIMED) — **[ADR-261](ADR-261-ruvector-graph-ann-index.md)** built the missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant and **measured** the ratio on our hardware. | **DONE — direction REFUTED at our scale (honest negative).** ADR-261 built the real HNSW baseline (**~25× QPS over linear scan at recall ≥0.99**, the substrate this row wanted) and a quantized variant. At N=10k the 1-bit Hamming traversal is **too coarse** — its best recall is 0.738, never reaching the ≥0.90 equal-recall point, so **no QPS win over float HNSW** (the SymphonyQG 3.5–17× is *not* reproduced by our 1-bit construction here). Caveat: **our HNSW + our 1-bit quant, not SymphonyQG's system**; expected crossover at large N + a multi-bit code. We did **not** tune to manufacture a speedup. |
|
||||
| **2** | **Multi-bit / Extended RaBitQ + unbiased estimator** | Extends our existing **1-bit** `sketch.rs` (ADR-084): Pass-2 rotation, multi-bit Pass-3, and the **real RaBitQ unbiased distance estimator** (Gao & Long SIGMOD 2024) reranking the candidate set from the 1-bit code + 8 B/vec side info (§11). | **MEASURED-on-our-hardware** (was CLAIMED) — rotation (§10), multi-bit (§10), and the estimator (§11) all implemented + benchmarked. Rotation lifts strict-K 36%→46%; multi-bit (≤4-bit) reaches 74% strict; **the estimator reaches 49.71% strict (cosine rerank), still short of 90%.** All clear 90% only with over-fetch (estimator improves the factor: 95% at candidate_k=24 vs sign 91.6%). | **DONE — RESOLVED-PARTIAL / NEGATIVE.** Rotation (§10) + estimator (§11) built and MEASURED. The honest negative (no strict-bar 90% from rotation, ≤4-bit, **or the unbiased estimator**) is recorded, not hidden. Over-fetch + Pass-2 is the path that meets the bar (ADR-084's "candidate set" pattern); the estimator lowers the over-fetch factor needed. |
|
||||
| **3** | **GraphPose-Fi-style learned antenna-attention + ChebGConv fusion head** | Would replace the current **untrained identity-projection + mean-pool** "attention" (the `CrossViewpointAttention` default is `ProjectionWeights::identity` — not a *learned* attention) with a learned graph fusion head. | **DATA-GATED** (per ADR-152 measurement (b): architecture is **NOT** the current bottleneck — **data is**) | **ACCEPTED-future, data-gated. Do NOT build now.** ADR-152's measured lesson was that swapping architecture without more/better paired data does not move PCK. Building a learned fusion head before the data exists would repeat the mistake ADR-155 §5 also flagged for GraphPose-Fi. |
|
||||
| — | **Cramér-Rao / sensor-placement** (`geometry.rs` CRB) | Investigated for a 2026 advance beating the textbook Fisher-information CRB already implemented. | **Investigated — NO ACTION** | **Cleared honestly.** No 2026 method beats the closed-form Fisher-information CRB for this 2-D bearing problem; our implementation is already correct SOTA. (Recording a negative result is a deliberate anti-slop signal.) The only CRB change this milestone is the §2.3 *GDOP* honesty fix, which is a labelling/quantity correction, not an algorithmic one. |
|
||||
@@ -138,7 +138,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
The review surfaced more than this milestone scoped. Tracked here for a future ADR-156 milestone:
|
||||
|
||||
- **SymphonyQG reproduction** (§5 #1) — reproduce the 3.5–17× QPS-over-HNSW claim on our hardware before integrating into the ruvector ANN path. Currently CLAIMED-only.
|
||||
- **SymphonyQG reproduction** (§5 #1) — **RESOLVED-DIRECTION-TESTED** (see [ADR-261](ADR-261-ruvector-graph-ann-index.md)). The missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant were built and **MEASURED**: float HNSW is ~25× over linear scan at recall ≥0.99 (the baseline this gap needed), but our 1-bit quantized traversal is **too coarse to beat float HNSW at equal recall at N=10k** (best recall 0.738) — the 3.5–17× is **not reproduced** by our construction. Honest negative recorded; expected crossover is large N + a multi-bit traversal code. (Caveat: our HNSW + our 1-bit quant, not SymphonyQG's exact system.)
|
||||
- **Multi-bit / Extended RaBitQ** (§5 #2) — **RESOLVED-PARTIAL** (see §10). Pass-2 randomized rotation (FHT + seeded ±1 sign flips, `src/rotation.rs`) and a multi-bit Pass-3 experiment landed and were MEASURED against the ADR-084 ≥90% bar. **Honest result: rotation helps (+10pp at the strict bar) and Pass-2 reaches 90% with ~3× over-fetch, but NEITHER rotation nor multi-bit (up to 4-bit) clears the strict candidate_k==K 90% bar on the tested anisotropic distribution.** The original `1-bit sign quantization ships first; rotation/more-bits later if benchmark-measured top-K coverage drops below 90%` deferral is therefore retired: the rotation is built, the bar is characterised, and the residual gap is documented rather than deferred.
|
||||
- **Learned cross-viewpoint fusion head** (§5 #3, GraphPose-Fi-style) — **data-gated**: blocked on the paired multi-room data ADR-152 measurement (b) identified as the real bottleneck; do not build the architecture first.
|
||||
- **`CrossViewpointAttention` learned projections** — the default `ProjectionWeights::identity` + mean-pool is honest but unlearned; wiring real learned Q/K/V projections is part of the data-gated item above (no learned weights ⇒ the "attention" is currently a geometric-bias-weighted average, which the code/docs should keep stating plainly).
|
||||
|
||||
@@ -265,3 +265,74 @@ Result at time of writing (all 0 failed):
|
||||
perform (B5).
|
||||
- Files kept under the 500-line guideline (`engine.rs` 462; behavioral tests
|
||||
moved to `tests/engine_behaviors.rs`).
|
||||
|
||||
## Addendum — `homecore-api` follow-up security review (beyond-SOTA pass)
|
||||
|
||||
A later network-facing review of `homecore-api` (the remote REST + WS attack
|
||||
surface) — independent of the ADR-154–159 sweep — found and fixed two real
|
||||
issues the original M7 pass (which focused on the WS auth bypass HC-WS-01, the
|
||||
reply-theater HC-WS-02, and the bin token provisioning HC-WS-08) did not catch.
|
||||
Both are LOW severity and reported at true severity.
|
||||
|
||||
### HC-API-AUTH-01 — `GET /api/` was unauthenticated (FIXED)
|
||||
|
||||
`rest::api_root` took no headers and unconditionally returned
|
||||
`200 {"message":"API running."}`, while every sibling route gates on
|
||||
`BearerAuth::from_headers`. HA's `APIStatusView` inherits `requires_auth = True`,
|
||||
so `/api/` must return **401** for a missing/wrong bearer. HA clients use the
|
||||
status route as a token-validation probe; a 200 told a bad-token client its
|
||||
token was valid and let an unauthenticated party confirm a live endpoint.
|
||||
LOW severity (the body is a static string; no entity/state data leaks).
|
||||
|
||||
**Fix:** `api_root(headers, State)` now validates the bearer like `get_config`.
|
||||
**Pinned by** (fail-on-old, `tests/server_bin_auth.rs`):
|
||||
`api_root_rejects_missing_bearer`, `api_root_rejects_wrong_bearer` (both 200→401),
|
||||
guarded by `api_root_accepts_correct_bearer` (still 200 with a valid token).
|
||||
|
||||
### HC-WS-LAG-01 — `subscribe_events` killed the stream on a broadcast lag (FIXED)
|
||||
|
||||
The per-subscription task matched `Err(_) => break` on both broadcast
|
||||
`recv()` arms. `RecvError::Lagged(n)` (a slow consumer falling
|
||||
>`EVENT_CHANNEL_CAPACITY` = 4,096 events behind) is **recoverable** — the bus
|
||||
doc says "Lagged receivers must re-sync" and HA keeps the subscription alive
|
||||
across a lag. The old code treated the first lag as fatal, so after an event
|
||||
burst the client's stream went permanently silent with no error frame — a
|
||||
self-inflicted event-delivery DoS under load.
|
||||
|
||||
**Fix:** `Lagged(_) => continue` (skip the dropped window, re-sync),
|
||||
`Closed => break`, on both the system and domain arms of the `select!`.
|
||||
**Pinned by** `subscription_survives_broadcast_lag` (`tests/ws_handshake.rs`):
|
||||
subscribes to a filtered event type, floods 6,000 unrelated events past the
|
||||
4,096 capacity to force a `Lagged`, then asserts a subsequent subscribed event
|
||||
is still delivered (old code: 5s-timeout panic).
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **AuthN/AuthZ** — all 7 other REST handlers gate on `BearerAuth::from_headers`
|
||||
→ `LongLivedTokenStore::is_valid` before any work; the WS handshake validates
|
||||
the `auth` token against the same store before the command loop, and
|
||||
privileged commands are unreachable pre-`auth_ok`. Token compare is
|
||||
`HashSet::contains` (content-independent timing — not the byte-`==` oracle of
|
||||
ADR-157 §B4), so no timing-oracle finding. No route skips the gate; no
|
||||
result-ignored check; no default/empty token accepted.
|
||||
- **Path traversal** — no route maps user input to a filesystem path (state is an
|
||||
in-memory `DashMap`); `:entity_id` passes through `EntityId::parse`, a strict
|
||||
`[a-z0-9_]+\.[a-z0-9_]+` ASCII allowlist that rejects `..`, `/`, `\`, and
|
||||
absolute paths. No traversal surface.
|
||||
- **Injection** — no SQL, no shell/subprocess, no `format!`-into-response;
|
||||
service/state bodies are typed `serde_json::Value` handed to the in-process
|
||||
registry (HA-equivalent).
|
||||
- **Info-leak** — `ApiError` maps to fixed status + a typed `{message}`;
|
||||
`ServiceError::HandlerFailed(String)` is integration-controlled (HA surfaces
|
||||
the handler error too), never framework internals/paths/stack-traces — no
|
||||
ADR-080-class leak.
|
||||
- **CORS** — explicit allowlist with `allow_credentials(false)` (HC-05),
|
||||
not `permissive()`.
|
||||
- **De-magic** — no bare security-relevant literals in the crate worth
|
||||
extracting (`EVENT_CHANNEL_CAPACITY` is already named in `homecore`; CORS
|
||||
dev-default ports are documented).
|
||||
|
||||
**Tests:** `homecore-api --no-default-features` **25 → 29** (+2 api-root auth,
|
||||
+1 api-root accept-guard, +1 WS lag-survival), 0 failed. Workspace green.
|
||||
Python deterministic proof unchanged (homecore-api is off the signal proof
|
||||
path).
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
# ADR 260: RuField Multimodal Field Sensing Specification
|
||||
|
||||
Status: Accepted — v0.1 reference stack
|
||||
|
||||
Date: 2026 06 14
|
||||
|
||||
Deciders: rUv
|
||||
|
||||
Tags: sensing, rf, csi, cir, bfld, radar, ultrasonic, infrared, quantum sensing, privacy, provenance, ruvector, ruview
|
||||
|
||||
## 1. Context
|
||||
|
||||
RuView proved that commodity wireless signals can be used as a practical sensing substrate. The next opportunity is larger: define a common specification for multimodal ambient sensing across RF, ultrasonic, subsonic, infrared, radar, and future quantum sensors.
|
||||
|
||||
Existing standards are valuable but fragmented.
|
||||
|
||||
IEEE 802.11bf 2025 standardizes WLAN sensing at the WiFi MAC and PHY layers and was published on September 26, 2025. It is important, but it is WiFi specific.
|
||||
|
||||
Bluetooth Channel Sounding standardizes techniques for obtaining phase and time delay information, but Bluetooth SIG explicitly does not define the distance algorithm. That leaves application level interpretation open.
|
||||
|
||||
IEEE 802.15.4z HRP UWB supports secure ranging using scrambled timestamp sequence waveforms, but UWB remains one modality rather than a universal sensing grammar.
|
||||
|
||||
Matter is a useful smart home interoperability protocol, but it is a device connectivity layer, not a multimodal field sensing specification.
|
||||
|
||||
The gap is clear: there is no open specification that normalizes sensor observations across CSI, CIR, BFLD, radar, ultrasound, subsonic vibration, thermal infrared, and quantum field sensing into one privacy aware, provenance rich, fusion ready event model.
|
||||
|
||||
## 2. Decision
|
||||
|
||||
Create **RuField MFS**, the RuField Multimodal Field Sensing Specification.
|
||||
|
||||
RuField MFS will define a common event, tensor, calibration, confidence, privacy, and provenance model for ambient field sensing.
|
||||
|
||||
It will not replace IEEE 802.11bf, Bluetooth Channel Sounding, UWB, Matter, radar protocols, or device vendor APIs.
|
||||
|
||||
It will sit above them.
|
||||
|
||||
```text
|
||||
WiFi CSI
|
||||
WiFi CIR
|
||||
WiFi BFLD
|
||||
UWB
|
||||
Bluetooth Channel Sounding
|
||||
mmWave radar
|
||||
Ultrasonic
|
||||
Subsonic
|
||||
Infrared
|
||||
Quantum magnetic sensing
|
||||
Quantum inertial sensing
|
||||
|
||||
all emit
|
||||
|
||||
RuField Field Event
|
||||
RuField Field Tensor
|
||||
RuField Fusion Graph
|
||||
RuField Privacy Class
|
||||
RuField Provenance Receipt
|
||||
```
|
||||
|
||||
## 3. Name
|
||||
|
||||
Preferred name: `RuField MFS`
|
||||
|
||||
Full name: `RuField Multimodal Field Sensing Specification`
|
||||
|
||||
Public positioning: `The open specification for camera free field intelligence.`
|
||||
|
||||
## 4. Problem Statement
|
||||
|
||||
Modern sensing systems are locked into modality specific silos: CSI systems produce channel matrices; radar produces range Doppler bins; UWB produces range and time of flight; Bluetooth Channel Sounding produces phase and timing primitives; infrared produces thermal arrays; ultrasonic produces acoustic echoes; subsonic produces structural vibration signatures; quantum sensors produce magnetic, inertial, or optical field traces.
|
||||
|
||||
Each has different sampling, calibration, confidence, privacy, and provenance semantics. This prevents reliable fusion and makes governance weak because raw sensing, derived sensing, biometric inference, and anonymous occupancy are often mixed without explicit boundaries.
|
||||
|
||||
## 5. Goals
|
||||
|
||||
1. Define a common multimodal sensing event format.
|
||||
2. Define a field tensor format spanning time, frequency, phase, amplitude, range, velocity, angle, temperature, vibration, and uncertainty.
|
||||
3. Define a modality registry for RF, acoustic, infrared, radar, and quantum sensing.
|
||||
4. Define privacy classes for raw waveforms, derived features, occupancy, anonymized aggregate state, and biometric inference.
|
||||
5. Define calibration receipts and provenance hashes.
|
||||
6. Define fusion rules for multimodal inference.
|
||||
7. Provide a Rust reference implementation.
|
||||
8. Provide benchmark tasks for camera free room intelligence.
|
||||
9. Make RuView one adapter inside a larger open sensing architecture.
|
||||
|
||||
## 6. Non Goals
|
||||
|
||||
1. Do not define a new wireless PHY.
|
||||
2. Do not replace IEEE 802.11bf.
|
||||
3. Do not replace Bluetooth Channel Sounding.
|
||||
4. Do not replace UWB secure ranging.
|
||||
5. Do not define medical diagnosis.
|
||||
6. Do not transmit speech, images, or raw biometric identity by default.
|
||||
7. Do not require cloud inference.
|
||||
8. Do not require expensive hardware.
|
||||
|
||||
## 7. Core Abstraction — the Field Event
|
||||
|
||||
A Field Event is a timestamped observation from any ambient field sensor.
|
||||
|
||||
```json
|
||||
{
|
||||
"spec": "rufield.mfs.v0.1",
|
||||
"event_id": "01J00000000000000000000000",
|
||||
"timestamp_ns": 1791986400000000000,
|
||||
"sensor": {
|
||||
"modality": "wifi_csi",
|
||||
"vendor": "esp32_c6",
|
||||
"device_id": "sensor_room_01",
|
||||
"placement": "ceiling_corner",
|
||||
"clock_domain": "local_ptp"
|
||||
},
|
||||
"field": {
|
||||
"carrier_hz": 5805000000,
|
||||
"bandwidth_hz": 80000000,
|
||||
"sample_rate_hz": 100,
|
||||
"channels": 234,
|
||||
"features": ["amplitude", "phase", "doppler", "range_proxy"]
|
||||
},
|
||||
"observation": {
|
||||
"space_cell": [4, 2, 1],
|
||||
"range_m": 3.42,
|
||||
"velocity_mps": 0.18,
|
||||
"motion_vector": [0.12, -0.03, 0.00],
|
||||
"confidence": 0.87,
|
||||
"privacy_class": "P2"
|
||||
},
|
||||
"provenance": {
|
||||
"raw_hash": "sha256:raw_measurement_hash",
|
||||
"firmware_hash": "sha256:firmware_hash",
|
||||
"model_id": "ruvector_field_encoder_v1",
|
||||
"calibration_id": "room_cal_2026_06_14"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 8. Modality Registry
|
||||
|
||||
| Code | Modality | Example source |
|
||||
| ---: | -------------------- | --------------------------------------- |
|
||||
| 1 | wifi_csi | ESP32 C6, Intel BE200, AP CSI |
|
||||
| 2 | wifi_cir | channel impulse response |
|
||||
| 3 | wifi_bfld | beamforming feedback |
|
||||
| 4 | uwb_hrp | IEEE 802.15.4z ranging |
|
||||
| 5 | ble_channel_sounding | phase and timing primitives |
|
||||
| 6 | mmwave_radar | range Doppler radar |
|
||||
| 7 | ultrasonic | echo and time of flight |
|
||||
| 8 | subsonic | structural vibration and room resonance |
|
||||
| 9 | infrared_thermal | thermal array or passive IR |
|
||||
| 10 | active_infrared | reflected IR |
|
||||
| 11 | lidar_phase | phase based optical range |
|
||||
| 12 | quantum_magnetic | NV diamond or OPM field trace |
|
||||
| 13 | quantum_inertial | atom interferometer or precision IMU |
|
||||
| 14 | event_camera | optional visual event stream |
|
||||
| 15 | synthetic_sim | simulator or replay source |
|
||||
|
||||
## 9. Field Tensor
|
||||
|
||||
The normalized numeric container (`Modality`, `FieldAxis`, `FieldTensor`) as specified in the implementation crate `rufield-core`.
|
||||
|
||||
## 10. Privacy Classes
|
||||
|
||||
| Class | Description | Example |
|
||||
| ----- | -------------------------------- | ------------------------------- |
|
||||
| P0 | Raw waveform or raw sensor frame | raw CSI, raw radar cube |
|
||||
| P1 | Derived non identity features | Doppler peak, thermal blob |
|
||||
| P2 | Occupancy and motion only | person present, bed exit |
|
||||
| P3 | Anonymous aggregate state | room count, zone activity |
|
||||
| P4 | Biometric or health inference | breathing, gait, sleep, scratch |
|
||||
| P5 | Identity linked inference | named person state |
|
||||
|
||||
Default system policy: edge storage may retain P0 only temporarily; network transmission defaults to P2 or lower; P4 requires explicit consent; P5 requires explicit identity binding and audit log.
|
||||
|
||||
## 11. Provenance Receipt
|
||||
|
||||
Every event must be auditable (`ProvenanceReceipt`). Acceptance invariant: **No fused inference is valid unless every contributing event has a provenance receipt or is explicitly marked synthetic.**
|
||||
|
||||
## 12. Fusion Graph
|
||||
|
||||
Nodes: sensor, event, field_tensor, feature, object, zone, state, inference, receipt.
|
||||
Edges: observed_by, derived_from, calibrated_by, supports, contradicts, fused_into, expires_at, requires_consent.
|
||||
|
||||
## 13. Fusion Rule Format
|
||||
|
||||
Human readable TOML rules (`rule.person_present`, `rule.bed_exit`, `rule.nocturnal_scratch`) with `inputs`, `method`, `threshold`, `privacy_max`, optional `window_ms` and `requires_consent`.
|
||||
|
||||
## 14. Reference Architecture
|
||||
|
||||
Layer 0 physical sensors; Layer 1 native adapters; Layer 2 field tensor normalization; Layer 3 RuVector field embeddings; Layer 4 fusion graph; Layer 5 policy and privacy guard; Layer 6 application event stream; Layer 7 dashboard, API, MCP, Matter bridge.
|
||||
|
||||
## 15. Rust Crate Layout
|
||||
|
||||
`rufield-core`, `rufield-schema`, `rufield-adapters`, `rufield-fusion`, `rufield-privacy`, `rufield-provenance`, `rufield-bench`, `rufield-viewer`.
|
||||
|
||||
## 16. Core Rust Interfaces
|
||||
|
||||
`FieldAdapter`, `FieldEncoder`, `FusionEngine`, `PrivacyGuard` traits as specified in `rufield-core`.
|
||||
|
||||
## 17. MVP Adapters
|
||||
|
||||
v0.1 must support three real modalities: WiFi CSI, mmWave radar, Infrared thermal. Optional: ultrasonic, subsonic, synthetic simulator.
|
||||
|
||||
## 18. Benchmark Suite
|
||||
|
||||
| Task | Metric | Target |
|
||||
| ----------------------- | -------: | -----------: |
|
||||
| Presence detection | F1 | 0.90 |
|
||||
| Room transition | F1 | 0.85 |
|
||||
| Bed exit | F1 | 0.90 |
|
||||
| Breathing detected | F1 | 0.80 |
|
||||
| Nocturnal scratch | F1 | 0.75 |
|
||||
| Fall like event | Recall | 0.95 |
|
||||
| False alarm rate | per hour | below 0.10 |
|
||||
| Event latency | p95 | below 100 ms |
|
||||
| Provenance coverage | percent | 100 |
|
||||
| Privacy violation count | count | 0 |
|
||||
|
||||
## 19. First Viral Demo
|
||||
|
||||
Camera free room intelligence: person enters, sits, breathing detected, sleeps, scratches arm, exits bed, leaves room — no camera, no identity, signed field receipts, live fusion graph, privacy class visible per event.
|
||||
|
||||
## 20. Data Model
|
||||
|
||||
`FieldEvent { spec_version, event_id, timestamp_ns, sensor, tensor, observation, provenance }` and `Observation { zone_id, space_cell, range_m, velocity_mps, motion_vector, confidence, labels, privacy_class }`.
|
||||
|
||||
## 21. Decision Matrix
|
||||
|
||||
| Option | Interop | Novelty | Buildability | Business value | Risk | Score |
|
||||
| --------------------------------------------- | ------: | ------: | -----------: | -------------: | ---: | ----: |
|
||||
| Extend RuView only | 2 | 2 | 5 | 3 | 2 | 14 |
|
||||
| Build proprietary fusion engine | 3 | 3 | 4 | 4 | 3 | 17 |
|
||||
| Create open RuField spec plus reference stack | 5 | 5 | 4 | 5 | 3 | 22 |
|
||||
| Attempt new hardware standard | 5 | 5 | 1 | 4 | 5 | 20 |
|
||||
|
||||
Decision: **Create open RuField spec plus reference stack.** It maximizes credibility, extensibility, and ecosystem pull while avoiding the impossible burden of defining a new physical layer.
|
||||
|
||||
## 22. Security Model
|
||||
|
||||
| Threat | Impact | Mitigation |
|
||||
| ----------------------------------- | ------------------------------- | -------------------------------------- |
|
||||
| Raw waveform leakage | privacy breach | P0 edge only by default |
|
||||
| Biometric inference without consent | legal and trust risk | P4 consent gate |
|
||||
| Sensor spoofing | false occupancy or safety event | signed sensor receipts |
|
||||
| Replay attack | forged event stream | nonce plus timestamp plus hash chain |
|
||||
| Model drift | wrong inference | calibration expiry and benchmark gates |
|
||||
| Overfitting to one room | weak generalization | room split benchmark |
|
||||
| Vendor firmware change | silent degradation | firmware hash in receipt |
|
||||
|
||||
## 23. Calibration Model
|
||||
|
||||
`CalibrationReceipt` is first class. Required calibration tasks: empty room baseline, single person walk path, sit and stand, bed or couch transition, breathing reference, no motion stability period.
|
||||
|
||||
## 24. Inference Semantics
|
||||
|
||||
Every inference must include: label, confidence, supporting events, contradicting events, privacy class, calibration id, model id, expiry time.
|
||||
|
||||
## 25. Consequences
|
||||
|
||||
Positive: RuView becomes part of a larger sensing ecosystem; the spec creates a standards-style wedge without waiting for silicon vendors; multimodal fusion becomes portable; privacy and provenance become differentiators; enterprise deployment becomes easier to justify; benchmark receipts reduce skepticism.
|
||||
|
||||
Negative: broad scope can dilute execution; hardware variability will be painful; calibration is the hardest practical problem; some will claim existing standards already solve parts of this; medical and biometric use cases require careful governance.
|
||||
|
||||
Mitigation: keep v0.1 narrow; ship real adapters; publish benchmark receipts; do not claim medical diagnosis; position RuField above existing standards.
|
||||
|
||||
## 26. Implementation Plan
|
||||
|
||||
Phase 1 spec skeleton; Phase 2 Rust core; Phase 3 adapters; Phase 4 fusion graph; Phase 5 dashboard; Phase 6 benchmark.
|
||||
|
||||
## 27. Acceptance Criteria
|
||||
|
||||
v0.1 is accepted when:
|
||||
|
||||
1. Three modalities stream into one event graph.
|
||||
2. Every event has a privacy class.
|
||||
3. Every event has a provenance receipt.
|
||||
4. Fusion produces at least five room state inferences.
|
||||
5. p95 event pipeline latency is below 100 ms.
|
||||
6. Benchmark runner produces deterministic reports.
|
||||
7. Raw waveform storage is disabled by default.
|
||||
8. P4 inference requires consent policy approval.
|
||||
9. Dashboard shows live camera free room intelligence.
|
||||
10. Spec is readable enough for external implementers.
|
||||
|
||||
## 28. Reference Repository Structure
|
||||
|
||||
Crates under `v2/crates/rufield-*` (workspace members), spec under `docs/rufield/`, benches under `rufield-bench`.
|
||||
|
||||
## 29. Open Questions
|
||||
|
||||
1. JSON Schema first, Protobuf first, or both?
|
||||
2. Default transport: MQTT, NATS, WebSocket, or MCP?
|
||||
3. Matter integration: bridge or first class target?
|
||||
4. P4 health inference disabled by default in public demos?
|
||||
5. Benchmark datasets synthetic first, then real world?
|
||||
6. Include quantum modality IDs even if adapters are synthetic only?
|
||||
|
||||
## 30. Recommendation
|
||||
|
||||
Proceed. Publish RuField as an open specification with a working Rust reference stack and a viral camera free room intelligence demo.
|
||||
|
||||
## 31. Benchmark Acceptance Test
|
||||
|
||||
```text
|
||||
Given a room with WiFi CSI, mmWave radar, and thermal IR sensors
|
||||
When a person enters, sits, breathes, exits bed, and leaves
|
||||
Then RuField emits signed events
|
||||
And classifies room state without a camera
|
||||
And keeps all default network events at P2 or below
|
||||
And produces p95 latency below 100 ms
|
||||
And produces a deterministic benchmark report
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Status (v0.1 reference stack)
|
||||
|
||||
The v0.1 reference stack is implemented as a **standalone Cargo workspace**
|
||||
(`rufield/`, published as `github.com/ruvnet/rufield` and vendored into RuView
|
||||
as a submodule — the `vendor/rvcsi` pattern). It is pure Rust, builds and tests
|
||||
on Windows with no native deps (`ndarray`/`tch`/`openblas` are not used), and
|
||||
depends only on `serde`, `serde_json`, `toml`, `sha2`, and `ed25519-dalek`.
|
||||
|
||||
**All metrics below are SYNTHETIC.** They are scored against the simulator's own
|
||||
ground-truth labels. They demonstrate the pipeline recovers known truth and runs
|
||||
within latency/privacy/provenance budgets — they are **not** field-validated
|
||||
accuracy. There is no hardware in v0.1; real adapters (ESP32 CSI, mmWave, thermal
|
||||
IR) are a documented follow-up (see the repo README "Firmware" section).
|
||||
|
||||
### Crates delivered
|
||||
|
||||
| Crate | Implements |
|
||||
|-------|-----------|
|
||||
| `rufield-core` | §7/§9/§16/§20 data model: `Modality` (15), `FieldAxis`, `FieldTensor` (shape↔values validated), `PrivacyClass` (P0–P5), `SensorDescriptor`, `Observation`, `FieldEvent`, `CalibrationReceipt`, `InferenceQuery`, `FieldInference`, `FieldEmbedding`; `FieldAdapter`/`FieldEncoder`/`FusionEngine`/`PrivacyGuard` traits. §7 JSON example round-trips. |
|
||||
| `rufield-provenance` | Real `sha256` content hashing + deterministic `ed25519` sign/verify; §11 `is_fusable` invariant. Tests: tamper → verify fails; synthetic event fusable without signer. |
|
||||
| `rufield-privacy` | §10 default policy + `DefaultPrivacyGuard` (`authorize` → Allow/Deny/RequiresConsent). Tests: P0 transmit denied; P4 no-consent → RequiresConsent; P4 consent → Allow; P2 → Allow; P5 needs identity binding. |
|
||||
| `rufield-adapters` | Deterministic seeded `SyntheticSim` emitting the §19 sequence across 3 modalities (wifi_csi, mmwave_radar, infrared_thermal). Same seed ⇒ identical signed event stream with ground-truth labels. |
|
||||
| `rufield-fusion` | `FusionGraph` (§12) + `RuFieldFusion` engine; TOML rules (§13, ≥5 inferences: person_present, sitting, sleeping, breathing, nocturnal_scratch, bed_exit, room_transition); weighted-Bayes + temporal-window; rejects non-fusable events; `FieldInference` with §24 fields. |
|
||||
| `rufield-bench` | Deterministic runner: F1 per task (SYNTHETIC), p95 latency, provenance coverage, privacy violations; JSON + human table; §31 acceptance test as `#[test]`. |
|
||||
|
||||
Total test count across the workspace: **60 tests, 0 failed**.
|
||||
`cargo clippy --workspace` is clean.
|
||||
|
||||
### §27 acceptance-criteria scorecard
|
||||
|
||||
| # | Criterion | Status |
|
||||
|---|-----------|--------|
|
||||
| 1 | Three modalities stream into one event graph | **PASS** — wifi_csi, mmwave_radar, infrared_thermal |
|
||||
| 2 | Every event has a privacy class | **PASS** — `Observation.privacy_class` (non-optional), default ≤ P2 |
|
||||
| 3 | Every event has a provenance receipt | **PASS** — every event is ed25519-signed and verifies; coverage 100% |
|
||||
| 4 | Fusion produces ≥ 5 room-state inferences | **PASS** — 7 distinct inferences produced |
|
||||
| 5 | p95 event pipeline latency < 100 ms | **PASS** — p95 ≈ 0.01 ms (in-process) |
|
||||
| 6 | Benchmark runner produces deterministic reports | **PASS** — identical report across runs (latency is the only wall-clock field) |
|
||||
| 7 | Raw waveform storage disabled by default | **PASS** — P0 network transmission denied by default policy |
|
||||
| 8 | P4 inference requires consent policy approval | **PASS** — P4 without consent → RequiresConsent; breathing/scratch rules carry `requires_consent = true` |
|
||||
| 9 | Dashboard shows live camera-free room intelligence | **PASS** — `rufield-viewer` (Axum + vanilla JS) streams the deterministic SyntheticSim→fusion demo: live room state, privacy-badged (P0–P5) event log, fusion graph, click-to-open signed-receipt modal, behind a permanent `SYNTHETIC — simulated sensors, no hardware` banner. `cargo run -p rufield-viewer`. Read-only demo viewer (not a device-management console — that's the real-adapter milestone). |
|
||||
| 10 | Spec readable for external implementers | **PASS** — ADR-260 + detailed standalone README with compiling usage examples |
|
||||
|
||||
**Decision:** **all §27 criteria 1–10 PASS** (criterion 9, the live dashboard,
|
||||
was completed by `rufield-viewer`). Status is **Accepted — v0.1 reference stack**.
|
||||
|
||||
### Deterministic benchmark report (SYNTHETIC, seed = 2026)
|
||||
|
||||
```text
|
||||
TASK (SYNTHETIC) METRIC VALUE TARGET MEETS
|
||||
presence f1 1.000 0.900 yes
|
||||
breathing f1 1.000 0.800 yes
|
||||
nocturnal_scratch f1 0.923 0.750 yes
|
||||
bed_exit f1 1.000 0.900 yes
|
||||
room_transition f1 1.000 0.850 yes
|
||||
-----------------------------------------------------------------------------------
|
||||
p50 latency: 0.0097 ms
|
||||
p95 latency: 0.0123 ms (target < 100 ms: PASS)
|
||||
provenance coverage: 100.0 % (target 100%: PASS)
|
||||
privacy violations: 0 (target 0: PASS)
|
||||
events=216 modalities=3 distinct_inferences=7
|
||||
```
|
||||
|
||||
All five scored §18 tasks meet their F1 targets **on synthetic ground truth**.
|
||||
`nocturnal_scratch` is 0.923 (one borderline noise tick at this seed) — reported
|
||||
honestly rather than tuned to 1.0. The fall-like / false-alarm-rate §18 rows are
|
||||
not scored in v0.1 (no fall is in the demo sequence) and are a follow-up. These
|
||||
numbers prove the fusion pipeline scores correctly against known truth; they say
|
||||
**nothing** about real-world accuracy, which requires the hardware adapters that
|
||||
v0.1 deliberately does not ship.
|
||||
|
||||
### Honest statement
|
||||
|
||||
Every metric here is simulator-based. No ESP32 CSI, mmWave, or thermal capture
|
||||
was used. RuField v0.1 is a working, honestly-measured reference pipeline —
|
||||
data model, provenance, privacy, fusion, and a deterministic benchmark — pending
|
||||
real hardware adapters.
|
||||
@@ -0,0 +1,200 @@
|
||||
# ADR-261: RuVector Graph-ANN Index — a real HNSW baseline + a SymphonyQG-style quantized variant, MEASURED
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | `wifi-densepose-ruvector` — `hnsw.rs`, `hnsw_quantized.rs`, `ann_measure.rs`, `benches/ann_bench.rs`, docs |
|
||||
| **Relates to** | ADR-084 (RaBitQ similarity sensor — 1-bit sketch), ADR-156 (RuVector beyond-SOTA sweep — §5 #1 SymphonyQG, §8/§10/§11 RaBitQ Pass-2/multi-bit/estimator), ADR-024 (AETHER re-ID), ADR-016/017 (RuVector integration) |
|
||||
| **Scope** | Build the **missing HNSW graph-ANN baseline** in the ruvector retrieval path, build a **SymphonyQG-style quantized-traversal variant** on the same graph, and **MEASURE** the real recall/QPS ratio between them — closing the ADR-156 §5 #1 gap honestly. Resolves ADR-156 §8 backlog item **"SymphonyQG reproduction"** from **CLAIMED-only** to **MEASURED-direction-tested**. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — the same contract as ADR-154/156:
|
||||
|
||||
- The HNSW index ships a **committed recall@10 correctness gate** (≥ 0.95 vs brute force on a planted-cluster fixture). Low recall means a graph bug; the gate is wired to fail in that case. It **did** fail first — and caught a real index-out-of-bounds bug in the insert path (§4) — which is exactly what a real gate is for.
|
||||
- Every QPS/recall number below is **MEASURED** on this box with a committed, deterministic, `--no-default-features`-runnable measurement (`src/ann_measure.rs`, `ann_bench_report`) and a committed criterion bench (`benches/ann_bench.rs`). Both call **one** shared fixture/measurement module, so the bench and the report can never measure different graphs.
|
||||
- The **headline result is an honest negative**: at our test scale the SymphonyQG-style quantized variant **does not beat float HNSW at equal recall** — the 1-bit Hamming traversal is too coarse to keep recall up. We report the real numbers, explain *why*, and state the expected large-N crossover. **We did not tune the quantized path to manufacture the 3.5–17× the source claims.** A measured negative + a scale caveat is a valid, publishable result.
|
||||
- We are explicit that this is **OUR HNSW + OUR 1-bit quantization, not SymphonyQG's exact system**. It tests the **direction** of the claim on our hardware/data, not a 1:1 reproduction.
|
||||
|
||||
Test machine: Windows 11, `cargo test --release`, `std::time::Instant` wall-clock. Numbers are warm medians on this box; the **ratio** is the claim, not the absolute QPS.
|
||||
|
||||
Reproduce:
|
||||
```bash
|
||||
cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
ann_bench_report -- --nocapture
|
||||
# Larger N: ANN_BENCH_N=50000 cargo test ... --release ann_bench_report -- --nocapture
|
||||
cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
The ruvector crate's retrieval path — AETHER re-ID hot-cache (ADR-024), the `sketch.rs` 1-bit prefilter (ADR-084), room fingerprinting — is, at its core, an **approximate nearest-neighbour (ANN)** problem: dense float embedding in, top-K similar ids out. But **the crate had no graph index**. Every `topk` was either a linear scan (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is `O(N)` per query and does not scale.
|
||||
|
||||
[ADR-156 §5 #1](ADR-156-ruvector-fusion-beyond-sota.md) graded **SymphonyQG** (SIGMOD 2025) the **lead beyond-SOTA ANN candidate**, citing the source's claim of **3.5–17× QPS over HNSW at equal recall**, but marked it **CLAIMED**:
|
||||
|
||||
> *"author-measured; **not reproduced on our hardware** — reproduction is future work."*
|
||||
|
||||
And ADR-156 §8 was blunt about *why* it could not be reproduced: **there was no HNSW baseline to compare against.** You cannot measure a ratio against a baseline that does not exist. This ADR builds that missing baseline, builds the quantized variant that tests the direction of the SymphonyQG bet, and measures the real ratio.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. Add a correct, dependency-free **float HNSW** graph index (`hnsw.rs`): the real Malkov & Yashunin (TPAMI 2018) algorithm — multi-layer navigable small-world graph, `ef_construction` / `ef_search`, the Algorithm-4 neighbour-selection heuristic, seeded-deterministic level assignment, L2 + cosine. This is the **baseline** ADR-156 said was missing.
|
||||
2. Add a **SymphonyQG-style quantized-traversal variant** (`hnsw_quantized.rs`): the *same* graph (same seed, same structure), but the beam search scores candidates with a **cheap 1-bit Hamming distance** over the RaBitQ Pass-2 rotated sign code (reusing `rotation.rs` + the sign-quantization of `sketch.rs`), then **exact-float reranks** the final candidate set. This is the SymphonyQG bet — cheaper per-node scoring, recovered by a final exact rerank.
|
||||
3. **Measure** linear vs float-HNSW vs quantized-HNSW (recall@10, QPS, equal-recall ratios) on one deterministic planted-cluster fixture, and record the honest verdict against the SymphonyQG 3.5–17× claim.
|
||||
|
||||
### Why 1-bit Hamming for the quantized traversal
|
||||
|
||||
The crate already had the exact pieces SymphonyQG fuses: a deterministic orthogonal rotation (`rotation.rs`, RaBitQ Pass-2) and sign-quantization (`sketch.rs`). A 1-bit code compares by POPCNT Hamming — a few machine words, no per-dimension float work — so it is the cheapest possible traversal score and the most direct test of "can a quantized score keep the beam on the right path." The cost (measured below): the 1-bit code is a *coarse* angle proxy (ADR-156 §10 measured ~46% strict-K coverage for sign-only), and that coarseness is what limits recall here.
|
||||
|
||||
---
|
||||
|
||||
## 3. Design
|
||||
|
||||
### 3.1 `hnsw.rs` — float HNSW (the baseline)
|
||||
|
||||
- **Graph.** `links[id][layer]` adjacency; layer 0 holds every node, higher layers exponentially sparser. `m_max` is `2·M` on layer 0, `M` above (the paper's asymmetric degree cap).
|
||||
- **Insert.** Greedy-descend the upper layers to a good entry point, then for each layer from the node's level down to 0: `search_layer` for `ef_construction` candidates, `select_neighbours` (Algorithm 4 — keep a candidate only if it is closer to the new node than to any already-selected neighbour, giving diverse navigable edges), wire bidirectional edges, re-prune any neighbour that overflows `m_max`. The node is pushed into the arrays **before** wiring so every `links[*]` index is valid mid-insert (§4 — the bug the gate caught).
|
||||
- **Search.** Greedy-descend layers `>0`, then best-first beam search of width `ef` on layer 0; return the closest `k`. Iterative (explicit heaps + visited set) — **no recursion**, bounded by the beam and the visited set.
|
||||
- **Determinism.** Level assignment is the only randomness and is driven by a **seeded SplitMix64** (the exact pattern from `rotation.rs`) — never `Date::now`/OS RNG/unseeded `rand`. Same `(seed, params, insertion order)` ⇒ bit-identical graph and search (pinned by `hnsw_is_deterministic_for_seed`).
|
||||
- **Robustness.** Empty index, `k==0`, `k>n`, single node, zero-dim, ragged query, `ef<k` all return cleanly — pinned by `*_no_panic` tests.
|
||||
|
||||
### 3.2 `hnsw_quantized.rs` — the SymphonyQG-style variant
|
||||
|
||||
Same graph as the float index (identical seed/structure — the **only** variable is the scoring), plus a per-node `ceil(D/8)`-byte 1-bit Pass-2 sign code (`D = next_pow2(dim)`). `search_quantized(query, k, ef, rerank)`:
|
||||
1. Encode the query to its 1-bit code (one rotation + sign pack).
|
||||
2. Greedy-descend + beam-search the graph scoring every visited node by **POPCNT Hamming** (query-code XOR node-code) — no per-dim float work.
|
||||
3. **Exact-float rerank** the top `rerank` Hamming candidates with the true L2/cosine metric, return the best `k`.
|
||||
|
||||
### 3.3 Security / robustness
|
||||
|
||||
Both indices: bounded **iterative** traversal (no unbounded recursion), no panic on empty/degenerate/ragged/zero-dim input (the metric compares over the shorter prefix; zero-norm cosine returns max distance, not NaN). The 1-bit encode handles padded dims via the existing `Rotation::apply_padded`.
|
||||
|
||||
---
|
||||
|
||||
## 4. The bug the correctness gate caught (disclosed, not hidden)
|
||||
|
||||
The first run of the recall@10 gate **panicked**: `index out of bounds: the len is 33 but the index is 33` in `search_layer`. Root cause: `insert` wired bidirectional edges (`links[nbr][l].push(id)`) **before** pushing the new node's own `links[id]` row into the array. A later traversal step in the *same* insert could hop to a neighbour that now pointed at `id` and read `links[id]` — which did not exist yet. Fix: push the node (with empty per-layer link lists) into `vectors`/`links`/`levels` **up front**, then wire edges into its existing slot. The new node has no incoming edges and empty outgoing lists until wiring, so it is unreachable by the searches that run first — pushing early is safe and keeps every index valid. This is exactly why the recall gate exists: a silent low-recall graph and an out-of-bounds panic are both "slop" the gate forces into the open.
|
||||
|
||||
---
|
||||
|
||||
## 5. The SymphonyQG claim being tested
|
||||
|
||||
| Source | Claim | Grade (before this ADR) |
|
||||
|--------|-------|-------------------------|
|
||||
| SymphonyQG, SIGMOD 2025 | **3.5–17× QPS over HNSW at equal recall**, via quantization unified with graph traversal, pure-CPU/edge-portable | **CLAIMED** — author-measured, *not reproduced on our hardware (no HNSW baseline existed)* |
|
||||
|
||||
The bet: a quantized traversal score is cheap enough — and accurate enough to keep the beam on-path — that you pay far less per visited node and recover the small recall loss with a final exact rerank.
|
||||
|
||||
---
|
||||
|
||||
## 6. MEASURED results
|
||||
|
||||
Fixture: planted-cluster synthetic, **dim=128, N=10,000, 64 clusters, 200 queries, K=10, noise=0.35**, L2 metric, `M=16`, `ef_construction=200`. Graph seed `0x6261524741484E53`, rotation seed `0x5EEDC0DE12345678`. `--release`, warm wall-clock on the test machine. (The fixture and both indices are shared by the criterion bench.)
|
||||
|
||||
| Method | recall@10 | QPS | latency (µs) |
|
||||
|--------|-----------|-----|--------------|
|
||||
| **linear scan (brute force)** | 1.0000 | 1,022 | 978 |
|
||||
| **float-HNSW** ef=16 | 0.9945 | **25,744** | 39 |
|
||||
| float-HNSW ef=32 | 0.9990 | 21,470 | 47 |
|
||||
| float-HNSW ef=64 | 1.0000 | 18,779 | 53 |
|
||||
| float-HNSW ef=128 | 1.0000 | 12,722 | 79 |
|
||||
| float-HNSW ef=256 | 1.0000 | 5,742 | 174 |
|
||||
| quant-HNSW ef=32 rr=20 | 0.1620 | 30,005 | 33 |
|
||||
| quant-HNSW ef=32 rr=100 | 0.2615 | 36,388 | 28 |
|
||||
| quant-HNSW ef=64 rr=100 | 0.4865 | 20,603 | 49 |
|
||||
| quant-HNSW ef=128 rr=100 | 0.6785 | 13,718 | 73 |
|
||||
| quant-HNSW ef=256 rr=100 | **0.7380** | 6,578 | 152 |
|
||||
|
||||
### Equal-recall QPS ratios
|
||||
|
||||
| Target recall | Fastest float-HNSW | Fastest quant-HNSW meeting it | quant/float | float/linear |
|
||||
|---------------|--------------------|-------------------------------|-------------|--------------|
|
||||
| ≥ 0.90 | ef=16 → 25,744 QPS | **none** (best quant recall = 0.738) | — | **25.19×** |
|
||||
| ≥ 0.95 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
| ≥ 0.99 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
|
||||
---
|
||||
|
||||
## 7. Honest verdict
|
||||
|
||||
**The HNSW baseline is a decisive win over linear scan: ~25× QPS at recall ≥ 0.99** (ef=16: 0.9945 recall, 25,744 QPS vs linear 1,022 QPS). The correctness gate (recall@10 ≥ 0.95 vs brute force, both L2 and cosine) holds. This is the baseline ADR-156 §5 #1 said did not exist — it now does.
|
||||
|
||||
**The SymphonyQG-style quantized variant does NOT beat float HNSW at our scale — direction REFUTED at N=10k.** The 1-bit Hamming traversal is too coarse: its best achievable recall is **0.738** (ef=256, rr=100), and it never reaches even the 0.90 equal-recall point where a fair QPS comparison could be made. Where the quantized score *is* faster (ef=32: ~30–36k QPS, beating float's 25.7k), its recall collapses to 0.16–0.26 — a meaningless win. There is **no equal-recall operating point** at which quantized is faster, so the SymphonyQG 3.5–17× claim is **not reproduced** by our 1-bit construction here.
|
||||
|
||||
**Why** (so the negative is understood, not just stated):
|
||||
1. The 1-bit sign code is a **coarse angle proxy** — ADR-156 §10 already measured it at ~46% strict-K coverage. Driving graph *traversal* by that coarse score steers the beam onto the wrong nodes, and the exact-float rerank can only recover what the beam actually visited. At N=10k, near-neighbours have nearly-identical sign codes, so Hamming cannot separate them.
|
||||
2. At this scale **float distance is already cheap**: one 128-d L2 is a handful of µs; the per-node float compute the quantization saves is small relative to the recall it costs. SymphonyQG's win shows up at **much larger N** (millions), where (a) the float-distance fraction of query time dominates and (b) their *multi-bit RaBitQ-fused* code (not our 1-bit sign code) keeps recall high. **Expected crossover: large N + a higher-bit code.** ADR-156 §10 already measured that a ≤4-bit code reaches ~74% strict coverage vs 1-bit's ~46%, so a multi-bit traversal score is the obvious next lever — deferred, not claimed.
|
||||
|
||||
**Caveat (stated plainly):** this is **our** HNSW + **our** 1-bit quantization, not SymphonyQG's system. We tested the *direction* of the claim ("does quantized traversal + rerank beat float HNSW at equal recall?") on our hardware/data and got a **measured no at N=10k**. That neither confirms nor refutes SymphonyQG's own published numbers on their system/scale — it refutes the direction *for our construction at our scale*, and identifies the two levers (scale, code bit-depth) a real reproduction would need.
|
||||
|
||||
---
|
||||
|
||||
## 8. Validation
|
||||
|
||||
- **`cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --lib`** — **156 passed / 0 failed, 1 ignored** (M1 added 20: 10 `hnsw`, 7 `hnsw_quantized`, 3 `ann_measure`; M2 added 5 multi-bit/scaling tests; `scaling_report` is the `#[ignore]` measurement that produced the §11 table).
|
||||
- **`cargo test --workspace --no-default-features`** — GREEN (see §10 for the count).
|
||||
- **Correctness gate verified to bite:** the recall@10 gate **panicked** on the first (buggy) insert path (§4); after the fix it passes at 0.99+ recall (L2 and cosine).
|
||||
- **`cargo test -p wifi-densepose-ruvector --no-default-features --release ann_bench_report -- --nocapture`** — prints the §6 table; the numbers above are copied verbatim from that run.
|
||||
- **`cargo bench -p wifi-densepose-ruvector --bench ann_bench`** — compiles and runs the same fixture through criterion.
|
||||
- **`python archive/v1/data/proof/verify.py`** — **VERDICT: PASS** (the Rust ANN work is independent of the Python signal-proof pipeline; hash unchanged).
|
||||
|
||||
---
|
||||
|
||||
## 9. Consequences
|
||||
|
||||
**Positive.** ruvector now has a real, deterministic, pure-Rust HNSW graph index (25× over linear scan at high recall) usable by the AETHER re-ID / sketch-prefilter path — the ANN substrate ADR-156 §5 #1 wanted. The SymphonyQG claim is no longer CLAIMED-only: we built the missing baseline and **measured** the direction, with the bug-caught-by-the-gate disclosed.
|
||||
|
||||
**Negative / honest.** The 1-bit quantized variant is **not** an equal-recall QPS win at our scale; it is shipped as a measured experiment with a clearly-stated ceiling, not as a recommended default. Anyone reaching for it must read §7.
|
||||
|
||||
**Resolved by Milestone-2 (§11, MEASURED — no longer deferred).**
|
||||
- **Multi-bit traversal score** — implemented (`b ∈ {1,2,4}` bits/dim over the Pass-2 rotated coordinates) and measured. It *does* lift quantized recall (at N=10k, b=4 reaches the 0.90 equal-recall regime where 1-bit could not), but still does not beat float HNSW QPS.
|
||||
- **Large-N crossover measurement** — measured at N ∈ {10k, 100k, 250k}. **The predicted large-N crossover did NOT materialize — it moved the wrong way** (quant recall *collapses* as N grows). See §11.
|
||||
|
||||
**Deferred (not silently dropped).**
|
||||
- **Wiring HNSW into the live re-ID path** (AETHER hot-cache / sketch prefilter) behind a flag.
|
||||
- **N ≥ 1M + SymphonyQG's exact RaBitQ-fused construction** — our impl refutes the *direction* at ≤250k; a true 1:1 reproduction at million-scale with their fused codes remains a separate, larger build.
|
||||
|
||||
---
|
||||
|
||||
## 10. What changed, file by file
|
||||
|
||||
- `hnsw.rs` (new) — float HNSW: graph, seeded-deterministic level assignment, Algorithm-2 beam search, Algorithm-4 neighbour selection, L2/cosine, brute-force ground truth, full degenerate-case guards; 10 tests incl. the recall@10 correctness gate (L2 + cosine) and determinism. The insert-order bug fix (§4).
|
||||
- `hnsw_quantized.rs` (new) — SymphonyQG-style quantized-traversal index over the shared graph: 1-bit Pass-2 code per node, Hamming-scored greedy + beam, exact-float rerank; 7 tests incl. the rerank-recall gate and determinism.
|
||||
- `ann_measure.rs` (new) — shared deterministic fixture + recall/QPS measurement for linear / float-HNSW / quant-HNSW, the `ann_bench_report` test (the §6 source of truth), `ANN_BENCH_N` override.
|
||||
- `benches/ann_bench.rs` (new) + `Cargo.toml` `[[bench]]` — criterion bench over the same fixture/indices.
|
||||
- `lib.rs` — `pub mod hnsw / hnsw_quantized / ann_measure`; re-export `HnswIndex`, `HnswParams`, `Metric`, `QuantizedHnswIndex`.
|
||||
- `ADR-156-ruvector-fusion-beyond-sota.md` §5 #1 + §8 backlog — SymphonyQG regraded **CLAIMED → MEASURED-direction-tested (refuted at N=10k for our 1-bit construction)**, pointing here.
|
||||
- `CHANGELOG.md` — `[Unreleased]` entry.
|
||||
|
||||
---
|
||||
|
||||
## 11. Milestone-2 — multi-bit traversal + large-N scaling study (MEASURED)
|
||||
|
||||
M1 (§7) refuted the SymphonyQG direction at N=10k with a 1-bit code, and *predicted* a crossover at "large N + a higher-bit code." M2 builds both levers and measures them — so the prediction is tested, not assumed.
|
||||
|
||||
**Built:** `hnsw_quantized.rs` generalized from 1-bit to a **`b`-bit-per-dimension** code (`b ∈ {1,2,4}`, a mid-rise quantizer over the same `RANGE=3.0` rotated coordinates as ADR-156 §10's `measure_multibit`); `ann_measure.rs` gained `run_scaling_study` / `best_float_op` / `best_quant_op` + a deterministic `scaling_report` (`#[ignore]`, `--release`) and a CI-safe `scaling_study_small_is_consistent`. Memory: **16 / 32 / 64 bytes/node** for b = 1 / 2 / 4.
|
||||
|
||||
**MEASURED** (dim=128, 64 clusters, 200 queries, K=10, L2, M=16, ef_construction=200, seeded, `--release`, this box; target recall ≥ 0.90):
|
||||
|
||||
| N | bits | B/node | quant best recall | float @ target | quant @ target | quant/float |
|
||||
|--:|--:|--:|--:|--|--|--:|
|
||||
| 10,000 | 1 | 16 | 1.000 | 23,155 QPS @ r=0.995 | 4,482 QPS @ r=0.965 | **0.19×** |
|
||||
| 10,000 | 2 | 32 | 1.000 | 23,155 QPS @ r=0.995 | 10,658 QPS @ r=0.908 | **0.46×** |
|
||||
| 10,000 | 4 | 64 | 1.000 | 23,155 QPS @ r=0.995 | 11,217 QPS @ r=0.946 | **0.48×** |
|
||||
| 100,000 | 1 / 2 / 4 | 16/32/64 | 0.207 / 0.346 / 0.788 | 2,493 QPS @ r=0.938 | none (never ≥ 0.90) | — |
|
||||
| 250,000 | 1 / 2 / 4 | 16/32/64 | 0.108 / 0.210 / 0.624 | 1,593 QPS @ r=0.925 | none | — |
|
||||
|
||||
**Verdict — NO crossover at any measured (N, b) up to 250k, and the trend REFUTES the large-N prediction:**
|
||||
1. **Multi-bit helps at small N but not enough.** At N=10k, more bits lift the equal-recall QPS ratio 0.19× → 0.46× → 0.48× (and let b≥2 actually *reach* the 0.90 bar that 1-bit missed) — but quant stays **below 1.0×**, i.e. slower than float HNSW at equal recall.
|
||||
2. **The predicted large-N crossover moved the wrong way.** As N grows 10k → 100k → 250k, quant's best achievable recall **collapses** (b=4: 1.000 → 0.788 → 0.624) and never reaches the 0.90 comparison point, while float HNSW holds ≥0.92. A denser graph packs near-neighbours whose low-bit codes are nearly identical, so the approximate score steers the beam off-path faster than the bigger float-distance savings can repay. The "crossover at millions" intuition is **not supported by our construction's trend** — if anything it diverges.
|
||||
3. **Caveat unchanged:** this is our HNSW + our per-node multi-bit code, not SymphonyQG's RaBitQ-fused graph. The result refutes the *direction* for our construction at ≤250k; it does not disprove their published numbers on their system at their scale. A real 1:1 reproduction is the deferred million-scale build.
|
||||
|
||||
This is a **published negative with the mechanism explained** — the multi-bit + scaling levers were built and measured rather than asserted, and the honest outcome (no crossover, trend diverging) is recorded, not hidden.
|
||||
@@ -0,0 +1,207 @@
|
||||
# ADR-262: RuField MFS ↔ RuView integration — a live SensingServerAdapter, a privacy/provenance bridge, MAPPED not papered-over
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed — **P1 + P3 implemented** (live `/api/field` + `/ws/field`; P3 signs with a **dedicated dev/sensing key**, deferring the §8 Q1 `cog-ha-matter` key-ownership decision to P2) |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | New thin bridge crate `wifi-densepose-rufield` (v2 workspace member); taps `wifi-densepose-sensing-server` emit path + `wifi-densepose-engine` `TrustedOutput`; depends on `vendor/rufield/crates/rufield-*` via path (the `vendor/rvcsi` pattern) |
|
||||
| **Relates to** | ADR-260 (RuField MFS spec + v0.1 reference stack), ADR-261 (RuVector graph-ANN), ADR-141 (BFLD privacy control-plane / modes / attestation), ADR-137 (fusion-engine quality scoring / contradiction), ADR-032 (multistatic mesh security hardening / witness), ADR-116 (cog tamper-evident audit log — `cog-ha-matter` SHA-256+Ed25519), ADR-095/096 (`rvcsi` vendored-submodule precedent) |
|
||||
| **Scope** | Decide **how** RuView's live WiFi-CSI sensing-server emits RuField `FieldEvent`s, **whether** RuView's ruvsense fusion composes with or is wrapped by rufield-fusion, and **how** to reconcile RuView's existing privacy/witness/provenance machinery with RuField's P0–P5 + ed25519 `ProvenanceReceipt`. The privacy/provenance reconciliation is the crux. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — every "RuView already does X" carries a `file:line`, and every external/SOTA claim is graded.
|
||||
|
||||
- **No accuracy is claimed.** RuField v0.1 is **SYNTHETIC** end-to-end by its own admission (ADR-260 "Honest statement", line 386–390: *"Every metric here is simulator-based. No ESP32 CSI, mmWave, or thermal capture was used."*). RuView's only real-CSI rufield path today would be **replay of recorded `.csi.jsonl`, unlabeled** — `rufield-adapters::CsiReplayAdapter`'s own module doc (`vendor/rufield/crates/rufield-adapters/src/csi_replay.rs:19-31`) states it is *"real signal, replay from file not live hardware, unlabeled ⇒ proxy not validated accuracy."* This ADR therefore proposes **plumbing**, and grades its own claims as "ARCHITECTURE" (a design decision, testable by a round-trip/compile gate) vs "ACCURACY" (which it explicitly does not assert).
|
||||
- The privacy/provenance section reports an **honest conflict**: RuView has **three** witness mechanisms across two hash algorithms, and **two** privacy enums, none of which map 1:1 onto RuField's P0–P5. We map them and recommend the cleanest reconciliation rather than asserting they already align.
|
||||
- Each phase below ships an **independently testable gate** (a round-trip test, a privacy-monotonicity test, a signature-verify test) so the integration is provable, not aspirational.
|
||||
|
||||
---
|
||||
|
||||
## 0.1 Implementation status
|
||||
|
||||
**P1 (§4) is implemented** as the `wifi-densepose-rufield` bridge crate (`v2/crates/wifi-densepose-rufield/`, a new v2 workspace member; path-deps the `vendor/rufield` submodule per §5.4):
|
||||
|
||||
- **Input** — `SensingSnapshot` (owned primitives mirroring `SensingUpdate` features/classification/signal_field joined with the `TrustedOutput` `trust_class`/`demoted`/`identity_bound`); the bridge does **not** depend on `wifi-densepose-sensing-server` (anti-corruption layer).
|
||||
- **Conversion** — `snapshot_to_field_event(&snap, &Signer)` emits a signed `FieldEvent` (`Modality::WifiCsi`, axis `[Frequency]`, real `timestamp_ns`); position derived from the signal-field peak when present (never fabricated); real sha256 `ProvenanceRef` + ed25519 signature, `synthetic = false`.
|
||||
- **Privacy (§3.3 crux)** — `map_privacy()` maps by information content, **fail-closed**: `Raw → P0`, `Derived → P4` (or `P5` if identity-bound — **never P1**), `Anonymous → P2`, `Restricted → P2`; a `demoted` cycle floors egress to ≥ P2.
|
||||
- **Gates that pass** (`tests/p1_gates.rs`, 15 tests / 0 failed = 5 unit + 9 integration + 1 doc): round-trip (snapshot → `FieldEvent` → serde → equal); `is_fusable` (verified ed25519 receipt); `RuFieldFusion::ingest` accept + `infer()` runs; **privacy-safety** (`gate_privacy_safety_derived_never_maps_to_low_privacy` — `Derived → P4/P5`, never P1; full §3.3 table; fail-closed demotion); determinism (same snapshot + same signer seed → byte-identical event).
|
||||
|
||||
**P3 (§4) is implemented** as the live RuField surface in `wifi-densepose-sensing-server` (the bridge is now wired into the running server):
|
||||
|
||||
- **Tap** — at the ESP32 governed-trust cycle (`main.rs` `observe_cycle` ~`:5886` / `SensingUpdate` build ~`:5938`), a new `emit_rufield_event` joins the cycle's `SensingUpdate` (features / classification / signal_field) with the engine's recorded `effective_class` / `demoted` trust state into a `wifi_densepose_rufield::SensingSnapshot`, then `snapshot_to_field_event(&snap, &signer)`. Existing endpoints (`/ws/sensing` etc.) are **unchanged** — purely additive.
|
||||
- **Surface** — `GET /api/field` (latest signed `FieldEvent`s + signer pubkey + a `dev_signing_key` flag) and `GET /ws/field` (broadcast stream, mirroring `/ws/sensing`), both mounted on the HTTP port and `/ws/field` also on the WS port. A small bounded ring buffer (`FIELD_RING_CAPACITY = 64`) holds recent **network-surfaced** events. New handler code lives in `src/rufield_surface.rs`, not in the 8k-line `main.rs`.
|
||||
- **Signer (defers the P2 key decision)** — a **dedicated standalone `Signer`** held in server state, seeded from `WDP_RUFIELD_SIGNING_SEED` (64-hex or ≥32-byte value), else a deterministic dev default with a logged `WARN`. Reusing the `cog-ha-matter` Ed25519 key (§8 Q1) is the **deferred P2** decision — P3 uses a standalone sensing key so it does not pre-empt that call.
|
||||
- **Egress privacy (fail-closed)** — `network_egress_allowed` is *stricter* than `DefaultPrivacyGuard` for an unattended live surface: only **P1/P2** leave the box; P0 (raw) and P3/P4/P5 (identity/biometric/aggregate above the default P2 ceiling) are held edge-local. A `Derived` cycle maps to P4/P5 and is therefore **never** surfaced. No-presence cycles emit nothing (no phantom events).
|
||||
- **Gates that pass** (`tests/rufield_surface_test.rs`, 4 integration via `tower::oneshot` + 4 module unit, 0 failed): a well-formed **signed** event (`Modality::WifiCsi`, P2 not P1, `is_fusable` ed25519-verified, real timestamp); **empty cycle → no phantom**; **privacy-safety** — an injected `Derived` trust never surfaces on `/api/field`; a mixed stream surfaces only egress-safe events.
|
||||
|
||||
**Deferred:** the §3.3 *provenance carrier* recommendation (reuse the `cog-ha-matter` SHA-256+Ed25519 chain + embed the BLAKE3 engine witness) is **not** in P1/P3 — both take a dedicated `Signer` (the §8 open question 1 key-ownership decision is unresolved; P3 uses a standalone dev/sensing key precisely so it does not pre-empt P2). P2's `cog-ha-matter` key reuse + BLAKE3-embed, and P4 (multi-modality), remain future work. **No accuracy is claimed** (§0 / §6) — P1/P3 are tested plumbing on a live endpoint + a safe privacy mapping; the live surface is single-link CSI with its existing caveats (no validated room-coordinate accuracy — `field_localize`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Context — two architectures, mapped
|
||||
|
||||
### 1.1 RuField MFS (ADR-260, `vendor/rufield/`)
|
||||
|
||||
A standalone pure-Rust Cargo workspace (serde, serde_json, toml, sha2, ed25519-dalek; **no tch/ndarray/candle**), vendored here as a git submodule (`git submodule status vendor/rufield` → `ba66e2e…`), **not** a v2 workspace member — exactly the `vendor/rvcsi` precedent (ADR-095/096). **Not published to crates.io**: every internal dep is a path dep with a nominal `version = "0.1.0"` (`vendor/rufield/Cargo.toml:31-37`); the `docs.rs/rufield-*` URLs are aspirational.
|
||||
|
||||
The data model (graded ARCHITECTURE, evidence read directly):
|
||||
|
||||
- **`FieldEvent`** (`vendor/rufield/crates/rufield-core/src/event.rs:96-112`): `spec_version, event_id, timestamp_ns: u64, sensor: SensorDescriptor, tensor: FieldTensor, observation: Observation, provenance: ProvenanceRef`.
|
||||
- **`Observation`** (`event.rs:25-51`): `zone_id, space_cell, range_m, velocity_mps, motion_vector, confidence: f32, features: BTreeMap<String,f32>` (the derived P1 scalars the fusion engine actually reads), `labels: Vec<String>` (ground-truth, **never read by fusion**), `privacy_class: PrivacyClass`.
|
||||
- **`PrivacyClass`** (`rufield-core/src/privacy.rs:8-25`): `P0..P5`, `#[serde(rename_all="UPPERCASE")]`, `Ord` by declaration order so **P0 < P1 < … < P5** — higher = more private; `level()->u8` returns 0..=5 (`privacy.rs:27-40`).
|
||||
- **`ProvenanceRef`** (on-wire, `event.rs:73-93`): `raw_hash, firmware_hash` (`sha256:…`), `model_id, calibration_id, synthetic: bool`, optional `signature_hex` / `signer_pubkey_hex` (detached ed25519).
|
||||
- The four traits (`rufield-core/src/traits.rs`): **`FieldAdapter`** (`:26-38`, `next_event() -> Result<Option<FieldEvent>>`), **`FieldEncoder`** (`:41-51`, **unimplemented in v0.1** — an open seam), **`FusionEngine`** (`:54-63`, `ingest(event)` + `infer(&query)`), **`PrivacyGuard`** (`:86-97`, `authorize(class, Destination, consent, identity_bound) -> PrivacyDecision{Allow|Deny|RequiresConsent}`).
|
||||
- **`CsiReplayAdapter`** (`rufield-adapters/src/csi_replay.rs`): constructed from **already-loaded text** (`from_jsonl(&str)` `:249-251`; `from_jsonl_with(text, device_id, &[u8;32])` `:254-323`) — **not** a path/`Read`/`Iterator`. Deserializes `CsiFrameRecord { timestamp: f64 (seconds), subcarriers: Vec<f64> }` (`:74-80`), buffers all frames into a `Vec<CsiFrame>`, then streams via a cursor (`next_event` `:550-557`). Maps each frame → `FieldEvent` with `Modality::WifiCsi`, axes `[Frequency]`, a Welford motion proxy, observation `privacy_class = P2 if presence else P1` (`:439-443`), real `sha256` raw-hash, and a **real ed25519 signature** (`signer.sign_event` `:507-510`). `max_privacy_class = P2`.
|
||||
- **`RuFieldFusion`** (`rufield-fusion/src/engine.rs:55-78`): `ingest()` **rejects non-fusable events on its first line** — `if !is_fusable(&event) { return Err(NotFusable) }` (`:212-215`) — then reads `event.observation.features` into a bounded temporal window; `infer()` applies TOML rules (`WeightedBayes` noisy-OR / `TemporalWindow`) → `Vec<FieldInference>`. TOML rule struct: `inputs, method, feature, threshold, privacy_max, window_ms, requires_consent` (`rules.rs:17-35`).
|
||||
- **`is_fusable`** (`rufield-provenance/src/lib.rs:179-184`): `synthetic == true` **OR** `verify_event().is_ok()` — the §11 invariant. Signing key is `ed25519_dalek 2.1`, deterministic from a 32-byte seed; raw hash is `sha256_hex` → `"sha256:<hex>"` (`:26-35`).
|
||||
- **`DefaultPrivacyGuard`** (`rufield-privacy/src/lib.rs:38-110`): default `network_max = P2`, `allow_p0_network = false`. P5-no-identity → `Deny`; P4-no-consent → `RequiresConsent`; `EdgeLocal` → `Allow`; `Network` denies P0 and `class > network_max`.
|
||||
- **`rufield-viewer`** (Axum 0.7): **self-contained, consumes `SyntheticSim` only** — all routes are read-only GET/SSE (`GET /api/run`, `GET /events`); **there is no ingest endpoint** (`vendor/rufield/crates/rufield-viewer/src/server.rs:63-72`). Feeding it a live stream requires adding a route.
|
||||
|
||||
### 1.2 RuView (the integration target)
|
||||
|
||||
- **Sensing-server is Axum** (`v2/crates/wifi-densepose-sensing-server/src/main.rs:7498-7629`), two listeners (WS `:8765`, HTTP). CSI does **not** arrive over WS/HTTP — it arrives over **UDP** from ESP32 nodes (`use tokio::net::UdpSocket`, `main.rs:53`; `recv_from` loop `main.rs:5286-5299`), parsed by magic `0xC511_0001` → **`Esp32Frame`** (`types.rs:84-100`: `node_id, n_subcarriers, ppdu_type, amplitudes: Vec<f64>, phases: Vec<f64>`, rssi/freq/sequence) → pushed into per-node `NodeState.frame_history: VecDeque<Vec<f64>>` (`main.rs:441-497`).
|
||||
- **`/ws/sensing` emits a `SensingUpdate`** (`main.rs:267-317`), broadcast over a `tokio::sync::broadcast` channel (`s.tx.send(json)` `main.rs:5938-5991`; the WS handler just subscribes and forwards, `main.rs:3021-3073`). `SensingUpdate` carries `nodes`, `features`, `classification {motion_level, presence, confidence}`, `signal_field`, `persons: Vec<PersonDetection>` (17 COCO keypoints + `position:[f64;3]` from `field_localize`, `main.rs:403-428`), pose, vitals. **`field_localize` (PR #1050) is a module, not a route** (`mod field_localize` `main.rs:17`; honesty caveat `field_localize.rs:16-27` — a single ESP32 link cannot resolve true room position, `position` is "strongest field peak").
|
||||
- **ruvsense fusion is strictly WITHIN-WiFi-modality.** `MultistaticFuser::fuse(&[MultiBandCsiFrame]) -> FusedSensingFrame` (`v2/crates/wifi-densepose-signal/src/ruvsense/multistatic.rs:285-288`) attention-weights **multiple WiFi CSI nodes/viewpoints** (every input is ESP32 CSI; `multistatic_bridge.rs:50-62` builds the frames from `NodeState` amplitude with `HardwareType::Esp32S3`). `coherence_gate.rs:18-37` is the `GateDecision{Accept|PredictOnly|Reject|Recalibrate}`; `pose_tracker.rs:255-263` is the 17-keypoint Kalman tracker with 128-dim AETHER re-ID; `field_model.rs:301-308` does SVD room-eigenstructure perturbation extraction. **No camera/mmWave/audio enters this path** — ruvsense is a multi-link WiFi-CSI fuser.
|
||||
- **The governed-trust cycle** runs in the separate **`wifi-densepose-engine`** crate. `StreamingEngine::process_cycle` (`v2/crates/wifi-densepose-engine/src/lib.rs:409`, `run_cycle` `:434-533`) produces **`TrustedOutput`** (`:82-112`): `semantic_id, quality: QualityScore, effective_class: PrivacyClass, demoted: bool, provenance: SemanticProvenance, witness: [u8;32]` (BLAKE3 over `evidence‖model‖calibration‖privacy_decision‖class`, `witness_of` `:598-613`), `recalibration_recommended`. **Crucially, none of this trust metadata is on the `SensingUpdate` wire today** — it is exposed only out-of-band on `GET /api/v1/status` (`main.rs:4173-4178`) and as a single live effect: `EngineBridge::suppress_raw_outputs()` strips per-node amplitude when `effective_class >= Restricted` (`engine_bridge.rs:240-243`, applied `main.rs:5908-5932`). The honest scope is stated in `engine_bridge.rs:14-27`: the governed engine runs *alongside* the bare fusion path; derived outputs are "published ungoverned."
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. **Build a thin RuView-side bridge crate `wifi-densepose-rufield`** (a new v2 workspace member) that depends on `vendor/rufield/crates/rufield-core` (+ `rufield-provenance`, `rufield-privacy`, `rufield-fusion`) **via path** — mirroring the `vendor/rvcsi` pattern. RuView does **not** depend on published rufield crates (there are none) and does **not** vendor rufield into the v2 workspace; rufield stays a standalone submodule and the bridge is the only coupling point (an anti-corruption layer).
|
||||
2. **Emit `FieldEvent`s from the live server via an in-process `SensingServerAdapter`**, not by re-using the file-based `CsiReplayAdapter` on the hot path. The bridge taps the existing `SensingUpdate` build site and the `EngineBridge` trust state, joins them, and emits one signed `FieldEvent` per cycle on a new `tokio::broadcast` topic / optional `/ws/field` endpoint. `CsiReplayAdapter` is retained for the **offline/replay** path (recorded `.csi.jsonl` → events) because it already reads RuView's recording format (`recording.rs` writes `{session}.csi.jsonl`).
|
||||
3. **Compose the two fusion engines vertically, do not merge them.** ruvsense stays the **WiFi-modality node** (multi-link fusion → one fused WiFi belief); rufield-fusion sits **above** it as the **cross-modality** graph. ruvsense's `FusedSensingFrame`/`TrustedOutput` becomes one `FieldEvent` (modality `wifi_csi`); rufield fuses it against future mmWave/thermal/`rvcsi` events. They do not conflict because ruvsense has no cross-modality fusion to collide with (§1.2 evidence).
|
||||
4. **Reconcile privacy/provenance with ONE canonical model + a documented mapping** (§3, the crux): RuView's `effective_class` is the **source of truth**, mapped onto RuField `PrivacyClass` at the bridge; RuView's existing **`cog-ha-matter` SHA-256+Ed25519 witness chain** (already RuField's exact crypto) is adopted as the carrier for RuField `ProvenanceReceipt`, with the live BLAKE3 engine witness embedded as a hashed field. We do **not** maintain two parallel signed-receipt systems.
|
||||
|
||||
---
|
||||
|
||||
## 3. Privacy & provenance reconciliation (the crux)
|
||||
|
||||
This is the most important section. RuView and RuField genuinely **overlap and partially conflict**. We map both honestly.
|
||||
|
||||
### 3.1 What RuView actually has (implemented, with evidence)
|
||||
|
||||
- **TWO privacy enums, not one ladder.** `PrivacyClass` — **4 variants** `Raw=0, Derived=1, Anonymous=2, Restricted=3` (`v2/crates/wifi-densepose-bfld/src/lib.rs:103-116`, `#[repr(u8)]`, higher byte = more private, **non-monotonic in information** — `Derived=1` carries *more* identity than `Anonymous=2`). And `PrivacyMode` — **5 variants** `RawResearch, PrivateHome, EnterpriseAnonymous, CareWithConsent, StrictNoIdentity` (`bfld/src/privacy_mode.rs:18-31`), each mapping to a `PrivacyClass` via `target_class()` (`:63-70`; two modes collapse to `Anonymous`).
|
||||
- **THREE witness mechanisms across TWO hash algorithms:**
|
||||
- BFLD `PrivacyAttestationProof` — **BLAKE3, unsigned**, attests mode/class continuity only; **built but NOT on the live path** (ADR-141 status line ~597; `bfld/src/privacy_mode.rs:121-148`).
|
||||
- Engine-cycle `TrustedOutput.witness: [u8;32]` — **BLAKE3, unsigned**, over the full trust decision; **LIVE every cycle** (`wifi-densepose-engine/src/lib.rs:598-613`).
|
||||
- `cog-ha-matter::WitnessChain` — **SHA-256 hash chain + Ed25519 signatures** (`v2/crates/cog-ha-matter/src/witness.rs:138-151`; `witness_signing.rs:39-76`), JSONL-persisted, `verify()` + `verify_signature()`. Implemented for ADR-116 (cog/Matter audit log); **standalone, not wired to BFLD/engine**. Its `WitnessHash` newtype doc explicitly anticipates a hash-algo migration (`witness.rs:37-41`).
|
||||
- **No numeric trust score.** "Trust" in code = `base_coherence: f32∈[0,1]` + `penalized_coherence()` (`signal/.../fusion_quality.rs:99,122-126`) + a **boolean** `forces_privacy_demotion()` (`:116`). Demotion is monotonic and irreversible (`demote_one` clamps at Restricted, `engine/src/lib.rs:617-619`).
|
||||
- **Structured provenance exists, but no signed "receipt" on the sensing path.** `SemanticProvenance { evidence, model_version, calibration_version, privacy_decision }` (`v2/crates/wifi-densepose-worldgraph/src/model.rs:137-147`) is attached to every belief and is the *input* to the BLAKE3 witness — but it is unsigned and not called a receipt.
|
||||
|
||||
### 3.2 Side-by-side, graded
|
||||
|
||||
| Dimension | RuView (file:line) | RuField | Alignment |
|
||||
|---|---|---|---|
|
||||
| Privacy ladder | `PrivacyClass` 4 (`bfld/lib.rs:103`) **or** `PrivacyMode` 5 (`bfld/privacy_mode.rs:18`) | `PrivacyClass` 6 (P0–P5, `rufield-core/privacy.rs:8`) | **PARTIAL→CONFLICT** — no clean 1:1; counts differ (4/5 vs 6); RuView class ordering non-monotonic |
|
||||
| Demotion direction | higher = more private, irreversible (`engine/lib.rs:617`) | higher P# = more private, `Ord` by decl order (`privacy.rs:8-25`) | **STRONG** (same direction) |
|
||||
| Provenance receipt | `SemanticProvenance` unsigned (`worldgraph/model.rs:137`) | `ProvenanceRef` + ed25519 (`event.rs:73`) | **PARTIAL** — structured but unsigned |
|
||||
| Witness crypto (live path) | BLAKE3 `[u8;32]`, unsigned (`engine/lib.rs:598`) | sha256 + ed25519 (`rufield-provenance/lib.rs:26,135`) | **CONFLICT** (algo + signing) |
|
||||
| Witness crypto (cog-ha-matter) | **SHA-256 + Ed25519** (`cog-ha-matter/witness.rs`, `witness_signing.rs`) | **sha256 + ed25519** | **STRONG** — RuField's exact crypto, already in-repo, but unwired and in another bounded context |
|
||||
| Trust / confidence | `penalized_coherence: f32` + boolean demote (`fusion_quality.rs:122`) | `confidence: f32` per observation | **WEAK** — RuView has no graded trust object; confidence maps, demotion is binary |
|
||||
|
||||
### 3.3 The recommendation (the key call)
|
||||
|
||||
**Adopt ONE canonical model with a documented, lossy-but-monotonic mapping — do not run two parallel schemes.** Concretely:
|
||||
|
||||
1. **Privacy: RuView `effective_class` is the source of truth; the bridge maps it onto RuField `PrivacyClass`** at the egress boundary. The honest mapping (graded ARCHITECTURE — it is a *policy* decision, and it is **monotonicity-testable**, not an accuracy claim):
|
||||
|
||||
| RuView `PrivacyClass` | → RuField | Rationale |
|
||||
|---|---|---|
|
||||
| `Raw` (raw CSI amplitude) | `P0` | raw waveform |
|
||||
| `Derived` (identity embedding, LAN-only) | `P4` *(or P5 if identity-bound)* | derived **identity** features ⇒ biometric/identity tier, **not** P1 — RuView's non-monotonic `Derived=1` is the trap; map by *information content*, not byte value |
|
||||
| `Anonymous` (occupancy/aggregate) | `P2`/`P3` | occupancy → P2, room-count aggregate → P3 |
|
||||
| `Restricted` (zeroized) | `P2`-capped, raw suppressed | matches `suppress_raw_outputs` (`engine_bridge.rs:240`) |
|
||||
|
||||
The bridge **must** map `Derived → P4/P5`, never P1, because RuView's `Derived` carries `identity_embedding` (§3.1) — this is the single most dangerous mapping mistake and gets a dedicated test (P2 in §4). `PrivacyMode` (5) is the better *operator-facing* join to RuField's 6 levels but the **class** is what gates egress, so the class mapping is canonical.
|
||||
|
||||
2. **Provenance: adopt `cog-ha-matter`'s SHA-256+Ed25519 chain as the carrier for RuField `ProvenanceReceipt`** — it is already RuField's exact crypto (graded STRONG above), already implemented, already tamper-evident. The bridge constructs the RuField `ProvenanceRef` by: `raw_hash = sha256(csi bytes)`, `model_id`/`calibration_id` from `SemanticProvenance`, and **embeds the live BLAKE3 engine witness `[u8;32]` as a hashed provenance field** (it is already computed every cycle — do not throw it away), then **signs with ed25519** so `is_fusable` passes for live (non-synthetic) events. We do **not** add a second BLAKE3-vs-ed25519 argument: BLAKE3 stays RuView's internal fast cycle-fingerprint; ed25519 is the *external* attestation RuField requires. One signer, one chain.
|
||||
|
||||
3. **Trust: map `penalized_coherence` → `Observation.confidence`; keep demotion binary.** RuView has no graded trust object to reconcile; the coherence scalar is the honest analog and the demotion boolean already drives `effective_class`.
|
||||
|
||||
This is a **bridge-with-canonical-source**, not "keep both forever." RuView owns the privacy decision (it has the live governed cycle); RuField owns the *external wire shape* (P0–P5 + signed receipt). The bridge is the one-directional translation, and it is the only place the two schemes meet.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phased plan (each phase independently shippable + testable)
|
||||
|
||||
**P1 — `SensingServerAdapter` emitting `FieldEvent`s (ARCHITECTURE).**
|
||||
New crate `wifi-densepose-rufield` with a `SensingServerAdapter` that consumes a `(SensingUpdate, TrustedOutput)` pair (tapped at `main.rs:5886`/`:5938`) and emits a signed `FieldEvent` (`Modality::WifiCsi`, axes `[Frequency]`, observation features from `SensingUpdate.features`, `confidence` from `penalized_coherence`). Offline path: keep `CsiReplayAdapter` for recorded `.csi.jsonl`. **Gate:** a round-trip test — emit a `FieldEvent` from a fixture `SensingUpdate`, assert it serializes, `is_fusable` passes (ed25519-signed), and `RuFieldFusion::ingest` accepts it. No server changes required beyond exposing the tap; the adapter is a library.
|
||||
|
||||
**P2 — privacy/provenance bridge (the crux, ARCHITECTURE).**
|
||||
Implement the §3.3 mapping: `effective_class → PrivacyClass`, `cog-ha-matter` ed25519 signer for the receipt, BLAKE3 witness embedded. **Gates (three, all monotonicity/safety, not accuracy):** (a) `Derived → P4|P5` never P1 (the dangerous-mapping test); (b) privacy monotonicity — `demoted == true` ⇒ emitted `PrivacyClass >= P2` and raw suppressed; (c) signature round-trip — sign with the cog-ha-matter key, `rufield_provenance::verify_event` passes. This phase is shippable without P3 (events emitted on an internal topic, not yet on the public wire).
|
||||
|
||||
**P3 — surface in `/ws` + viewer (ARCHITECTURE).**
|
||||
Add an opt-in `/ws/field` endpoint (or a `field_events` array on `SensingUpdate` behind a flag) carrying the signed `FieldEvent` + a privacy badge. Add an ingest route to `rufield-viewer` (it has none today — `server.rs:63-72`) so it can replay RuView's live feed instead of only `SyntheticSim`. **Gate:** a WS integration test asserting a connected client receives a privacy-badged, signature-verifiable `FieldEvent`; a viewer test asserting the new ingest route renders a live event. The `cognitum` appliance can speak RuField by consuming this endpoint (it already runs `ruview-vitals-worker`); deferred to its own ADR.
|
||||
|
||||
**P4 — fusion composition + multi-modality (ARCHITECTURE, optional).**
|
||||
Wire a second modality (cheapest: an `rvcsi`-sourced event, or recorded mmWave) into `RuFieldFusion` alongside the WiFi event, proving cross-modality fusion above ruvsense. **Gate:** a fusion test with two modalities producing ≥1 cross-modal inference, with provenance coverage 100%.
|
||||
|
||||
---
|
||||
|
||||
## 5. Decision matrix
|
||||
|
||||
### 5.1 Data-path emission (P1)
|
||||
|
||||
| Option | Latency | Reuse | Live-fit | Risk | Verdict |
|
||||
|---|---|---|---|---|---|
|
||||
| Re-use `CsiReplayAdapter` on hot path | poor (file buffer, `&str` ctor) | high | **bad** — it's a file-cursor, not a live source | low | **Reject for live** (keep for replay) |
|
||||
| In-process `SensingServerAdapter` (tap `SensingUpdate`+`TrustedOutput`) | good | medium | **good** — taps the real emit + real trust state | low | **CHOSEN** |
|
||||
| Server publishes `FieldEvent` on its own topic (no adapter trait) | good | low | good | medium (bypasses `FieldAdapter` contract) | Reject — loses the trait seam |
|
||||
|
||||
### 5.2 Fusion relationship (P3/P4)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Merge ruvsense into rufield-fusion | **Reject** — different scopes; ruvsense is within-WiFi multi-link, rufield is cross-modality |
|
||||
| rufield-fusion wraps ruvsense (vertical compose) | **CHOSEN** — ruvsense → one WiFi `FieldEvent` → rufield cross-modality graph |
|
||||
| Run both as peers, reconcile after | Reject — duplicates fusion semantics, two contradiction models |
|
||||
|
||||
### 5.3 Privacy/provenance reconciliation (P2)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| (a) Map RuView classes onto RuField P0–P5, RuView canonical | **CHOSEN (privacy)** — `effective_class` is the live source of truth |
|
||||
| (b) Adopt RuField ed25519 receipts as RuView's provenance | **CHOSEN (provenance)** — via the already-present `cog-ha-matter` SHA-256+Ed25519 chain |
|
||||
| (c) Keep both schemes with a permanent bridge | **Reject** — two signed-receipt systems is the duplication we must not ship |
|
||||
|
||||
### 5.4 Dependency direction
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Depend on published rufield crates | **Reject** — not published (`vendor/rufield/Cargo.toml:31-37`) |
|
||||
| Make rufield a v2 workspace member | **Reject** — breaks the standalone-spec/`rvcsi` precedent |
|
||||
| Thin `wifi-densepose-rufield` bridge → path deps on submodule | **CHOSEN** — anti-corruption layer, single coupling point |
|
||||
|
||||
---
|
||||
|
||||
## 6. Security & honesty notes
|
||||
|
||||
- **No accuracy claim.** Live RuField events from RuView are derived from the same single-link CSI whose own caveats are on record (`field_localize.rs:16-27`); the offline path is unlabeled replay (`csi_replay.rs:19-31`). This ADR ships **plumbing with monotonicity/signature gates**, not validated F1.
|
||||
- **The dangerous mapping is `Derived → P1`.** RuView's `Derived` byte value (1) is numerically below `Anonymous` (2) but carries identity (`bfld/lib.rs`); a naive byte-mapping would leak identity-bearing features as low-privacy P1. P2's gate (a) exists specifically to prevent this.
|
||||
- **One signer, not two.** Adding a second ed25519 keypair alongside `cog-ha-matter`'s would create two roots of trust. The bridge reuses the cog-ha-matter signing key (`witness_signing.rs`).
|
||||
- **`is_fusable` is a real gate, not decoration** (`rufield-provenance/lib.rs:179-184`): live events that fail to sign are rejected by `RuFieldFusion::ingest` — we must not paper over a signing failure with `synthetic = true` on a real event (that would be the §11 invariant violation the spec forbids).
|
||||
- BLAKE3 stays internal; ed25519 is the external attestation. We do not relitigate RuView's BLAKE3 cycle-witness — it is embedded, not replaced.
|
||||
|
||||
## 7. Consequences
|
||||
|
||||
**Positive:** RuView becomes one honest adapter in the larger RuField ecosystem (ADR-260 goal §9) without forking its fusion or privacy engine; the three witness mechanisms get a single external attestation path; cross-modality fusion becomes possible above the existing WiFi fusion; the `cognitum` appliance gains a standard wire format. The bridge is the only coupling point, so rufield can evolve as a standalone spec.
|
||||
|
||||
**Negative:** a fourth crate to maintain; the privacy mapping is lossy (4/5 → 6) and must be kept honest by tests; reusing the `cog-ha-matter` key crosses a bounded-context boundary (cog/Matter ↔ sensing) that ADR-116 kept separate — that coupling needs review. The live trust metadata (`witness`, `effective_class`) is **currently decoupled** from `SensingUpdate` (§1.2), so P1 must do real join work, not a field read.
|
||||
|
||||
## 8. Open questions
|
||||
|
||||
1. **Signer ownership:** should the bridge reuse the `cog-ha-matter` Ed25519 key, or mint a dedicated RuView-sensing key with its own rotation? (Reuse couples bounded contexts; a new key adds a second root of trust.)
|
||||
2. **`PrivacyMode` vs `PrivacyClass` as the canonical map target:** class gates egress (chosen), but the 5-mode ladder is the cleaner join to 6 levels — do we expose mode in the receipt too?
|
||||
3. **Where does the BLAKE3 engine witness live in the RuField receipt** — a `firmware_hash`-style field, an extension field, or a `CalibrationReceipt.data_hash`? (RuField's `ProvenanceRef` has no spare slot; needs a spec extension or reuse of `model_id`.)
|
||||
4. **Should `field_localize` positions ride in `Observation.space_cell`/`motion_vector`** given the explicit single-link caveat, or stay RuView-only until multi-node calibration lands?
|
||||
5. **`rvcsi` relationship:** `rvcsi` has its own `CsiFrame`/`CsiWindow` and could implement `FieldAdapter` directly — should the second modality in P4 be `rvcsi`, making RuField the convergence point for *both* vendored sensing runtimes?
|
||||
6. **Transport:** RuField ADR-260 §29 leaves default transport open (MQTT/NATS/WS/MCP). RuView is WS + UDP + broadcast; does `/ws/field` suffice, or does the appliance need MQTT to match the cog stack?
|
||||
|
||||
## 9. Recommendation
|
||||
|
||||
Proceed with P1+P2 behind a feature flag. They are independently shippable, carry real gates (round-trip, monotonicity, signature-verify), and require no change to RuView's fusion or privacy engine — only a tap and a translation. Defer P3/P4 and the appliance/transport questions to follow-up ADRs once the bridge round-trips on recorded `.csi.jsonl` and on one live cycle.
|
||||
@@ -522,6 +522,25 @@ Base URL: `http://localhost:3000` (Docker) or `http://localhost:8080` (binary de
|
||||
| `GET` | `/api/v1/mesh` | ADR-110 fleet-wide mesh sync map ([iter 29](adr/ADR-110-esp32-c6-firmware-extension.md)) | `{"nodes":{"9":{...},"12":{...}},"total":2}` |
|
||||
| `GET` | `/api/v1/nodes/:id/sync` | Single-node mesh sync snapshot (or 404) | `{"offset_us":1163565,"is_leader":false,...}` |
|
||||
| `GET` | `/api/v1/mesh/metrics` | ADR-110 mesh state in Prometheus exposition format ([iter 36](adr/ADR-110-esp32-c6-firmware-extension.md)) | `wifi_densepose_mesh_offset_us{node="9"} 1163565\n…` |
|
||||
| `GET` | `/api/field` | ADR-262 P3 — latest **signed RuField `FieldEvent`s** from the live sensing cycle, plus the signer pubkey + a `dev_signing_key` flag. Only egress-safe (P1/P2) events are surfaced; identity/biometric (P4/P5) and raw (P0) are held edge-local | `{"spec":"rufield","signer_pubkey_hex":"…","dev_signing_key":true,"events":[…]}` |
|
||||
|
||||
### RuField surface (ADR-262 P3)
|
||||
|
||||
RuView's live WiFi-CSI sensing now also speaks the standalone **RuField MFS** wire format. Each governed sensing cycle is converted (via the `wifi-densepose-rufield` anti-corruption bridge) into a **signed** `FieldEvent` (`Modality::WifiCsi`, ed25519 `ProvenanceRef`) and surfaced on two additive endpoints:
|
||||
|
||||
- `GET /api/field` — the most recent signed events (JSON).
|
||||
- `GET /ws/field` — a WebSocket that streams each cycle's signed event (mirrors `/ws/sensing`).
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/field | python -m json.tool # latest signed FieldEvents
|
||||
python -c "import asyncio,websockets; asyncio.run((lambda: websockets.connect('ws://localhost:8765/ws/field'))())" # stream
|
||||
```
|
||||
|
||||
Privacy is fail-closed: only egress-safe **P1/P2** events leave the box — raw (P0) and identity/biometric/aggregate (P3–P5) cycles are held **edge-local** and never appear on these endpoints; a no-presence cycle emits **no event**.
|
||||
|
||||
**Signing key:** the surface signs with a **dedicated dev/sensing key**, seeded from `WDP_RUFIELD_SIGNING_SEED` (a 64-char hex string or a ≥32-byte value); when unset it falls back to a deterministic dev default and logs a `WARN` (the `dev_signing_key` flag in `/api/field` reflects this). This is a standalone key pending the ADR-262 §8 Q1 key-ownership decision — set `WDP_RUFIELD_SIGNING_SEED` for any real deployment.
|
||||
|
||||
> **Honesty (ADR-262 §0/§6):** this is real plumbing on a live endpoint, **not an accuracy claim.** It is the single-link CSI sensing with its existing caveats (no validated room-coordinate accuracy — positions are the "strongest field peak", not calibrated triangulation).
|
||||
|
||||
### Example: Get fleet mesh state (ADR-110)
|
||||
|
||||
|
||||
Generated
+49
@@ -7085,6 +7085,42 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-fusion"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"toml 0.8.23",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-privacy"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-provenance"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"rufield-core",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc"
|
||||
version = "0.24.0"
|
||||
@@ -11045,6 +11081,18 @@ dependencies = [
|
||||
"tower-http",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-fusion",
|
||||
"rufield-privacy",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-ruvector"
|
||||
version = "0.3.2"
|
||||
@@ -11094,6 +11142,7 @@ dependencies = [
|
||||
"wifi-densepose-engine",
|
||||
"wifi-densepose-geo",
|
||||
"wifi-densepose-hardware",
|
||||
"wifi-densepose-rufield",
|
||||
"wifi-densepose-signal",
|
||||
"wifi-densepose-wifiscan",
|
||||
"wifi-densepose-worldgraph",
|
||||
|
||||
@@ -72,6 +72,11 @@ members = [
|
||||
"crates/homecore-assist", # ADR-133 — HOMECORE voice assistant + ruflo bridge
|
||||
"crates/homecore-server", # iter-9 — HOMECORE integration binary (all 8 crates wired together)
|
||||
"crates/ruview-swarm", # ADR-148 — drone swarm control system
|
||||
# ADR-262 P1 — anti-corruption bridge converting RuView WiFi-CSI sensing
|
||||
# output into signed RuField FieldEvents. Path-deps the `vendor/rufield`
|
||||
# submodule crates (rufield-core/-provenance/-privacy/-fusion); single
|
||||
# coupling point between RuView and the standalone RuField MFS spec.
|
||||
"crates/wifi-densepose-rufield",
|
||||
]
|
||||
# ADR-040: WASM edge crate targets wasm32-unknown-unknown (no_std),
|
||||
# excluded from workspace to avoid breaking `cargo test --workspace`.
|
||||
|
||||
@@ -102,19 +102,43 @@ pub struct WitnessEvent {
|
||||
pub this_hash: WitnessHash,
|
||||
}
|
||||
|
||||
/// Domain-separation tag prefixing every witness canonical message.
|
||||
///
|
||||
/// This is the *domain tag* half of the "domain-tag + length-prefix"
|
||||
/// rule for any hashed/signed message whose fields are
|
||||
/// operator-influenceable. The witness chain already length-prefixes
|
||||
/// `kind` and `payload` (preventing intra-protocol concatenation
|
||||
/// forgery); the tag adds cross-protocol separation so a SHA-256
|
||||
/// preimage / Ed25519 message produced here can never be re-interpreted
|
||||
/// as a message from another signing context that shares key
|
||||
/// infrastructure — notably ADR-116's *manifest* `binary_signature`
|
||||
/// (Ed25519 over `binary_sha256`), which ADR-262 P2 reuses this exact
|
||||
/// chain for. A signature is only ever valid for the one domain whose
|
||||
/// tag it commits to.
|
||||
///
|
||||
/// The trailing NUL terminates the version string so a future
|
||||
/// migration (Blake3, extra fields, Merkle tier) bumps the tag instead
|
||||
/// of silently colliding with v1 bundles.
|
||||
pub const WITNESS_DOMAIN_TAG: &[u8] = b"cog-ha-matter/witness-event/v1\x00";
|
||||
|
||||
/// Compute the canonical-bytes form an event is hashed over.
|
||||
///
|
||||
/// The format is intentionally simple and length-prefixed so a
|
||||
/// future migration can be staged with a `version` byte in front
|
||||
/// without ambiguity:
|
||||
/// The format is domain-tagged and length-prefixed:
|
||||
///
|
||||
/// ```text
|
||||
/// prev_hash[32] | seq:u64-be | ts:u64-be | kind_len:u32-be | kind | payload_len:u32-be | payload
|
||||
/// DOMAIN_TAG | prev_hash[32] | seq:u64-be | ts:u64-be
|
||||
/// | kind_len:u32-be | kind | payload_len:u32-be | payload
|
||||
/// ```
|
||||
///
|
||||
/// Length-prefixing prevents the classic "concatenation forgery"
|
||||
/// attack where `"abc" + "def"` and `"ab" + "cdef"` would hash the
|
||||
/// same.
|
||||
/// * The leading [`WITNESS_DOMAIN_TAG`] gives cross-protocol
|
||||
/// separation: bytes signed/hashed here cannot be replayed as a
|
||||
/// message for another Ed25519 context in the same trust chain
|
||||
/// (e.g. the manifest `binary_signature`). It also carries a format
|
||||
/// version for staged migrations.
|
||||
/// * Length-prefixing `kind` and `payload` prevents the classic
|
||||
/// "concatenation forgery" where `"abc" + "def"` and `"ab" + "cdef"`
|
||||
/// would hash the same. The fixed-width `prev_hash`/`seq`/`ts`
|
||||
/// fields are self-delimiting.
|
||||
pub fn canonical_bytes(
|
||||
prev_hash: WitnessHash,
|
||||
seq: u64,
|
||||
@@ -123,7 +147,10 @@ pub fn canonical_bytes(
|
||||
payload: &[u8],
|
||||
) -> Vec<u8> {
|
||||
let kind_bytes = kind.as_bytes();
|
||||
let mut out = Vec::with_capacity(32 + 8 + 8 + 4 + kind_bytes.len() + 4 + payload.len());
|
||||
let mut out = Vec::with_capacity(
|
||||
WITNESS_DOMAIN_TAG.len() + 32 + 8 + 8 + 4 + kind_bytes.len() + 4 + payload.len(),
|
||||
);
|
||||
out.extend_from_slice(WITNESS_DOMAIN_TAG);
|
||||
out.extend_from_slice(&prev_hash.0);
|
||||
out.extend_from_slice(&seq.to_be_bytes());
|
||||
out.extend_from_slice(×tamp_unix_s.to_be_bytes());
|
||||
@@ -466,11 +493,51 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_bytes_starts_with_prev_hash() {
|
||||
fn canonical_bytes_starts_with_domain_tag_then_prev_hash() {
|
||||
// Locks the on-wire format. A future migration that flips
|
||||
// field order must bump a version byte and update this test.
|
||||
// field order must bump the domain tag and update this test.
|
||||
let bytes = canonical_bytes(WitnessHash([7u8; 32]), 1, 2, "k", b"p");
|
||||
assert_eq!(&bytes[..32], &[7u8; 32]);
|
||||
let tag = WITNESS_DOMAIN_TAG.len();
|
||||
assert_eq!(&bytes[..tag], WITNESS_DOMAIN_TAG);
|
||||
assert_eq!(&bytes[tag..tag + 32], &[7u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_bytes_is_domain_separated() {
|
||||
// Cross-protocol separation: the witness preimage must begin
|
||||
// with the domain tag so its SHA-256 / Ed25519 message can
|
||||
// never be reinterpreted as a message from another signing
|
||||
// context that shares key infrastructure (e.g. the manifest
|
||||
// `binary_signature` over `binary_sha256`). Fails on the old
|
||||
// un-tagged encoding, which began directly with `prev_hash`.
|
||||
let bytes = canonical_bytes(WitnessHash::GENESIS, 0, 0, "k", b"p");
|
||||
assert!(
|
||||
bytes.starts_with(WITNESS_DOMAIN_TAG),
|
||||
"canonical message is not domain-separated"
|
||||
);
|
||||
// The tag is versioned and NUL-terminated.
|
||||
assert!(WITNESS_DOMAIN_TAG.ends_with(b"\x00"));
|
||||
assert!(WITNESS_DOMAIN_TAG.windows(2).any(|w| w == b"v1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn witness_preimage_cannot_collide_with_a_bare_manifest_digest() {
|
||||
// The manifest `binary_signature` signs a bare 64-byte
|
||||
// SHA-256 hex string. A witness preimage must never *equal*
|
||||
// such a string, even if an operator crafted kind/payload to
|
||||
// try — the domain tag (33 bytes) + fixed 48-byte prefix make
|
||||
// the witness message structurally longer and tag-distinct.
|
||||
// Fails on the old encoding only if it could ever produce a
|
||||
// 64-byte all-hex message; the tag makes the impossibility
|
||||
// explicit and regression-guarded.
|
||||
let manifest_digest_msg = "a".repeat(64); // 64 ASCII hex bytes
|
||||
let witness = canonical_bytes(WitnessHash::GENESIS, 0, 0, "", b"");
|
||||
assert_ne!(witness.as_slice(), manifest_digest_msg.as_bytes());
|
||||
assert!(
|
||||
witness.len() > manifest_digest_msg.len(),
|
||||
"domain tag must make witness preimage structurally distinct"
|
||||
);
|
||||
assert!(!witness.starts_with(b"aaaa"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
//! key store (separate concern). Tests use a fixed-bytes seed for
|
||||
//! determinism — never check in real Seed keys here.
|
||||
|
||||
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
|
||||
use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey};
|
||||
|
||||
use crate::witness::{canonical_bytes, WitnessEvent};
|
||||
|
||||
@@ -58,6 +58,16 @@ pub fn sign_event(event: &WitnessEvent, key: &SigningKey) -> Signature {
|
||||
/// Verify an Ed25519 signature against a witness event using the
|
||||
/// Seed's public key. `Ok(())` iff the signature is valid for the
|
||||
/// event's canonical bytes under this key.
|
||||
///
|
||||
/// Uses `verify_strict` (not the permissive `Verifier::verify`) on
|
||||
/// purpose: for a tamper-evident *audit* chain the signature is the
|
||||
/// attestation, so non-canonical encodings and small-order public
|
||||
/// keys must be rejected. `verify_strict` enforces RFC 8032's
|
||||
/// stricter checks, giving the "one canonical signature per event"
|
||||
/// property an auditor relies on when comparing or deduplicating
|
||||
/// signed witness records. The public key is caller-pinned (the
|
||||
/// Seed's known verifying key) — never parsed from the event — so a
|
||||
/// forged event carrying its own key cannot self-verify.
|
||||
pub fn verify_signature(
|
||||
event: &WitnessEvent,
|
||||
signature: &Signature,
|
||||
@@ -71,7 +81,7 @@ pub fn verify_signature(
|
||||
&event.payload,
|
||||
);
|
||||
public_key
|
||||
.verify(&bytes, signature)
|
||||
.verify_strict(&bytes, signature)
|
||||
.map_err(|_| SignatureVerifyError::Invalid)
|
||||
}
|
||||
|
||||
@@ -140,6 +150,58 @@ mod tests {
|
||||
verify_signature(&event, &sig, &public).expect("clean signature verifies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signature_commits_to_domain_tag_not_bare_fields() {
|
||||
// The signature is over the domain-tagged canonical bytes. A
|
||||
// signature produced over the *un-tagged* concatenation of the
|
||||
// same fields must NOT verify — proving cross-protocol
|
||||
// separation reaches the signature layer, not just the hash.
|
||||
// Fails on the old encoding where the signed message began
|
||||
// directly with `prev_hash` (no tag).
|
||||
use ed25519_dalek::Signer;
|
||||
let key = fixed_key();
|
||||
let public = key.verifying_key();
|
||||
let event = fresh_event();
|
||||
|
||||
// Hand-build the OLD (un-tagged) preimage and sign it.
|
||||
let mut untagged = Vec::new();
|
||||
untagged.extend_from_slice(&event.prev_hash.0);
|
||||
untagged.extend_from_slice(&event.seq.to_be_bytes());
|
||||
untagged.extend_from_slice(&event.timestamp_unix_s.to_be_bytes());
|
||||
untagged.extend_from_slice(&(event.kind.len() as u32).to_be_bytes());
|
||||
untagged.extend_from_slice(event.kind.as_bytes());
|
||||
untagged.extend_from_slice(&(event.payload.len() as u32).to_be_bytes());
|
||||
untagged.extend_from_slice(&event.payload);
|
||||
let old_sig = key.sign(&untagged);
|
||||
|
||||
// The current verifier (which uses the domain-tagged message)
|
||||
// must reject a signature made over the un-tagged bytes.
|
||||
let err = verify_signature(&event, &old_sig, &public).unwrap_err();
|
||||
assert_eq!(err, SignatureVerifyError::Invalid);
|
||||
|
||||
// Sanity: the proper signature still verifies.
|
||||
let good = sign_event(&event, &key);
|
||||
verify_signature(&event, &good, &public).expect("tagged signature verifies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_uses_strict_path_and_pins_caller_key() {
|
||||
// Regression guard: verification must run through the strict
|
||||
// path against a CALLER-supplied key. A wrong key fails; the
|
||||
// event never carries its own verifying key, so a forged event
|
||||
// cannot self-attest. (verify_strict additionally rejects
|
||||
// non-canonical / small-order encodings.)
|
||||
let key = fixed_key();
|
||||
let wrong = SigningKey::from_bytes(b"another-wrong-key-another-wrong-");
|
||||
let event = fresh_event();
|
||||
let sig = sign_event(&event, &key);
|
||||
verify_signature(&event, &sig, &key.verifying_key()).expect("right key verifies");
|
||||
assert_eq!(
|
||||
verify_signature(&event, &sig, &wrong.verifying_key()).unwrap_err(),
|
||||
SignatureVerifyError::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_rejects_signature_under_wrong_key() {
|
||||
let key = fixed_key();
|
||||
|
||||
@@ -12,8 +12,20 @@ use crate::state::SharedState;
|
||||
#[derive(Serialize)]
|
||||
pub struct ApiRunning { message: &'static str }
|
||||
|
||||
pub async fn api_root() -> Json<ApiRunning> {
|
||||
Json(ApiRunning { message: "API running." })
|
||||
/// `GET /api/` — the HA `APIStatusView` ("API running." ping).
|
||||
///
|
||||
/// Security (HC-API-AUTH-01): HA's `APIStatusView` inherits
|
||||
/// `requires_auth = True` from `HomeAssistantView`, so an unauthenticated
|
||||
/// (or wrong-token) request to `/api/` returns **401**, not 200. HA
|
||||
/// clients (and the companion app) rely on this status route as a
|
||||
/// *token-validation probe* — a 200 here would tell a client a bad token
|
||||
/// is good, and would let an unauthenticated party confirm a live
|
||||
/// HOMECORE-API endpoint. The P2 handler skipped the bearer gate that
|
||||
/// every sibling route applies; this restores wire-compat by validating
|
||||
/// the bearer like `get_config`/`get_states` before replying.
|
||||
pub async fn api_root(headers: HeaderMap, State(s): State<SharedState>) -> ApiResult<Json<ApiRunning>> {
|
||||
let _ = BearerAuth::from_headers(&headers, s.tokens()).await?;
|
||||
Ok(Json(ApiRunning { message: "API running." }))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
||||
@@ -298,7 +298,17 @@ impl Connection {
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => break,
|
||||
// A slow consumer that falls >4,096 events behind
|
||||
// gets `Lagged(n)`, which is RECOVERABLE: the bus
|
||||
// doc (`bus.rs` §"Lagged receivers must re-sync")
|
||||
// and HA's WS contract both keep the subscription
|
||||
// alive across a lag. The pre-fix `Err(_) => break`
|
||||
// treated `Lagged` as fatal, silently killing the
|
||||
// client's event stream on a burst (HC-WS-LAG-01).
|
||||
// Skip the dropped window and continue; only a
|
||||
// `Closed` sender ends the task.
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
},
|
||||
evt = domain_rx.recv() => match evt {
|
||||
Ok(de) => {
|
||||
@@ -316,7 +326,12 @@ impl Connection {
|
||||
if tx_clone.send(payload.to_string()).is_err() { break; }
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
// Same recoverable-lag handling as the system arm
|
||||
// above (HC-WS-LAG-01): a lagged domain-event
|
||||
// receiver re-syncs and continues; only `Closed`
|
||||
// terminates the subscription.
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,3 +75,72 @@ async fn from_env_path_enforces_whitelist() {
|
||||
assert!(!store.is_valid("not_in_whitelist").await);
|
||||
assert!(!store.is_dev_mode().await, "from_env must NOT be dev mode");
|
||||
}
|
||||
|
||||
// ─── HC-API-AUTH-01: `GET /api/` must be auth-gated like every sibling ───
|
||||
//
|
||||
// HA's `APIStatusView` inherits `requires_auth = True`, so `/api/` returns
|
||||
// 401 for a missing/wrong bearer and 200 only for a valid one. The pre-fix
|
||||
// `api_root` took no headers and unconditionally returned 200 — these two
|
||||
// tests FAIL on that code.
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_rejects_missing_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"GET /api/ with NO bearer must be 401 (HC-API-AUTH-01) — HA's \
|
||||
APIStatusView requires_auth=True; a 200 here lets an \
|
||||
unauthenticated party confirm a live endpoint and tells a \
|
||||
token-validation probe a bad token is good"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_rejects_wrong_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.header("Authorization", "Bearer the_wrong_token")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"GET /api/ with a WRONG bearer must be 401 (HC-API-AUTH-01)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_root_accepts_correct_bearer() {
|
||||
let app = router(provisioned_state("the_real_token").await);
|
||||
let resp = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/")
|
||||
.header("Authorization", "Bearer the_real_token")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::OK,
|
||||
"GET /api/ with the correct bearer must still return 200 (API running.)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -166,3 +166,100 @@ async fn ping_pong_reply_is_received() {
|
||||
assert_eq!(reply["type"], "pong");
|
||||
assert_eq!(reply["id"], 7);
|
||||
}
|
||||
|
||||
/// Variant of [`spawn_server_with_token`] that also returns a `HomeCore`
|
||||
/// handle (cheap `Arc` clone) so the test can fire events into the *same*
|
||||
/// bus the served subscription reads from.
|
||||
async fn spawn_server_returning_homecore(valid_token: &str) -> (SocketAddr, HomeCore) {
|
||||
let hc = HomeCore::new();
|
||||
let tokens = LongLivedTokenStore::empty();
|
||||
tokens.register(valid_token).await;
|
||||
let state = SharedState::with_tokens(hc.clone(), "Test", "test-version", tokens);
|
||||
let app = router(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(addr, hc)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscription_survives_broadcast_lag() {
|
||||
// HC-WS-LAG-01: the per-subscription event task must treat a broadcast
|
||||
// `Lagged(n)` as RECOVERABLE (re-sync + continue), matching the bus
|
||||
// contract ("Lagged receivers must re-sync") and HA's WS semantics.
|
||||
//
|
||||
// The pre-fix `Err(_) => break` killed the whole event-stream task on
|
||||
// the first lag, so after a >4,096-event burst the client's stream
|
||||
// went permanently silent. This test fires far more than the 4,096
|
||||
// channel capacity to force a `Lagged`, then fires ONE more event and
|
||||
// asserts the subscription still delivers it. FAILS (5s timeout) on
|
||||
// the old code because the task is already dead.
|
||||
use homecore::{Context, DomainEvent};
|
||||
|
||||
let (addr, hc) = spawn_server_returning_homecore("good_token_abc").await;
|
||||
let url = format!("ws://{addr}/api/websocket");
|
||||
let (mut ws, _resp) = connect_async(&url).await.unwrap();
|
||||
|
||||
let _ = next_json(&mut ws).await; // auth_required
|
||||
ws.send(Message::Text(
|
||||
serde_json::json!({"type":"auth","access_token":"good_token_abc"}).to_string(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let auth = next_json(&mut ws).await;
|
||||
assert_eq!(auth["type"], "auth_ok");
|
||||
|
||||
// Subscribe to a specific domain event type so unrelated traffic is
|
||||
// filtered out and we can deterministically match the post-lag event.
|
||||
ws.send(Message::Text(
|
||||
serde_json::json!({"id": 1, "type": "subscribe_events", "event_type": "lag_probe"})
|
||||
.to_string(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let ack = next_json(&mut ws).await; // result ok for the subscribe
|
||||
assert_eq!(ack["type"], "result");
|
||||
assert_eq!(ack["success"], true);
|
||||
|
||||
// Flood the bus far past EVENT_CHANNEL_CAPACITY (4,096) with events the
|
||||
// subscription FILTERS OUT (different event_type). Because the client
|
||||
// never reads them off the WS, the server-side broadcast receiver falls
|
||||
// behind and the NEXT `recv()` yields `Lagged`. We fire synchronously
|
||||
// and don't yield to the WS reader, guaranteeing the overflow.
|
||||
for i in 0..6000u32 {
|
||||
hc.bus().fire_domain(DomainEvent::new(
|
||||
"noise",
|
||||
serde_json::json!({ "i": i }),
|
||||
Context::new(),
|
||||
));
|
||||
}
|
||||
|
||||
// Now fire the event the client IS subscribed to. On the fixed code the
|
||||
// task recovered from `Lagged` and continues, so this is delivered. On
|
||||
// the old code the task broke on `Lagged` and this never arrives.
|
||||
hc.bus().fire_domain(DomainEvent::new(
|
||||
"lag_probe",
|
||||
serde_json::json!({ "marker": "post-lag" }),
|
||||
Context::new(),
|
||||
));
|
||||
|
||||
// Drain frames until we see our post-lag event (ignoring any noise the
|
||||
// filter let slip before the lag), bounded by a timeout.
|
||||
let got = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let v = next_json(&mut ws).await;
|
||||
if v["type"] == "event" && v["event"]["event_type"] == "lag_probe" {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect(
|
||||
"subscription went silent after a broadcast lag — Lagged was treated \
|
||||
as fatal (HC-WS-LAG-01)",
|
||||
);
|
||||
assert_eq!(got["event"]["data"]["marker"], "post-lag");
|
||||
}
|
||||
|
||||
@@ -29,8 +29,10 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_yaml = "0.9"
|
||||
serde_json = "1"
|
||||
|
||||
# MiniJinja — HA-compatible Jinja2 template engine in pure Rust (ADR-129 §2.1)
|
||||
minijinja = { version = "2", features = ["json", "loader"] }
|
||||
# MiniJinja — HA-compatible Jinja2 template engine in pure Rust (ADR-129 §2.1).
|
||||
# `fuel` bounds instruction count so a malicious `template:` condition cannot
|
||||
# spin the engine with a nested-loop / huge-repeat DoS (HC-SEC-01).
|
||||
minijinja = { version = "2", features = ["json", "loader", "fuel"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "1"
|
||||
|
||||
@@ -70,6 +70,32 @@ impl ExecutionContext {
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bound for a `delay` / `wait_for_trigger` timeout, in seconds
|
||||
/// (~100 years). Caps absurd values so `Duration::from_secs_f64` cannot
|
||||
/// overflow-panic on e.g. `seconds: 1e308`, while still allowing any
|
||||
/// realistic automation delay (HC-SEC-02).
|
||||
const MAX_DELAY_SECS: f64 = 3.15e9;
|
||||
|
||||
/// Convert a user-supplied seconds value into a `Duration` without
|
||||
/// panicking (HC-SEC-02).
|
||||
///
|
||||
/// `Duration::from_secs_f64` **panics** on negative, NaN, infinite, or
|
||||
/// overflowing inputs. Those values are all reachable from a crafted
|
||||
/// automation YAML (`delay: {seconds: -1}`, `.nan`, `.inf`, `1e308`), so a
|
||||
/// single hostile config would crash the running automation task. We
|
||||
/// instead saturate to a safe range — matching Home Assistant's lenient
|
||||
/// treatment of a non-positive delay as "no delay":
|
||||
///
|
||||
/// - non-finite (NaN / ±inf) → `0`
|
||||
/// - negative → `0`
|
||||
/// - above [`MAX_DELAY_SECS`] → clamped to the cap
|
||||
fn safe_duration_from_secs(seconds: f64) -> Duration {
|
||||
if !seconds.is_finite() || seconds <= 0.0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
Duration::from_secs_f64(seconds.min(MAX_DELAY_SECS))
|
||||
}
|
||||
|
||||
/// Action configuration. Deserialized from YAML `action:` blocks.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "action", rename_all = "snake_case")]
|
||||
@@ -154,7 +180,10 @@ impl Action {
|
||||
Ok(result)
|
||||
}
|
||||
Action::Delay { seconds } => {
|
||||
let dur = Duration::from_secs_f64(*seconds);
|
||||
// `safe_duration_from_secs` guards against negative /
|
||||
// NaN / infinite / overflowing values that would
|
||||
// otherwise panic `Duration::from_secs_f64` (HC-SEC-02).
|
||||
let dur = safe_duration_from_secs(*seconds);
|
||||
sleep(dur).await;
|
||||
Ok(serde_json::Value::Null)
|
||||
}
|
||||
@@ -172,7 +201,8 @@ impl Action {
|
||||
// P1 stub — just sleeps for the timeout duration if specified.
|
||||
// Full trigger subscription lands in P2.
|
||||
if let Some(secs) = timeout_seconds {
|
||||
sleep(Duration::from_secs_f64(*secs)).await;
|
||||
// Same non-panicking guard as `Delay` (HC-SEC-02).
|
||||
sleep(safe_duration_from_secs(*secs)).await;
|
||||
}
|
||||
Ok(serde_json::Value::Null)
|
||||
}
|
||||
@@ -243,6 +273,68 @@ mod tests {
|
||||
assert!(result.is_null());
|
||||
}
|
||||
|
||||
// ── HC-SEC-02: a crafted delay must not panic the run task ─────────
|
||||
//
|
||||
// `Duration::from_secs_f64` panics on negative / NaN / infinite /
|
||||
// overflowing inputs, all reachable from a YAML `delay:` value. On the
|
||||
// pre-fix code each of these aborts the spawned automation task with a
|
||||
// panic; the guard saturates to a safe Duration instead. These tests
|
||||
// fail on old (panic = test failure).
|
||||
#[tokio::test]
|
||||
async fn delay_negative_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: -1.0 }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "negative delay must be treated as 0, not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delay_nan_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: f64::NAN }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "NaN delay must be treated as 0, not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delay_infinite_seconds_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::Delay { seconds: f64::INFINITY }.execute(&mut ctx).await;
|
||||
assert!(result.is_ok(), "infinite delay must saturate to 0, not panic");
|
||||
}
|
||||
|
||||
// Note: the overflow case (1e300) is covered by the synchronous
|
||||
// `safe_duration_saturates_hostile_values` unit test below — executing
|
||||
// `Action::Delay { seconds: 1e300 }` would genuinely sleep for the
|
||||
// clamped (~100-year) duration, so we assert the conversion directly
|
||||
// rather than through `execute`.
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_for_trigger_negative_timeout_does_not_panic() {
|
||||
let hc = HomeCore::new();
|
||||
let mut ctx = ExecutionContext::new(hc, "auto");
|
||||
let result = Action::WaitForTrigger { timeout_seconds: Some(-5.0) }
|
||||
.execute(&mut ctx)
|
||||
.await;
|
||||
assert!(result.is_ok(), "negative wait timeout must not panic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_duration_saturates_hostile_values() {
|
||||
assert_eq!(safe_duration_from_secs(-1.0), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::NAN), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::INFINITY), Duration::ZERO);
|
||||
assert_eq!(safe_duration_from_secs(f64::NEG_INFINITY), Duration::ZERO);
|
||||
// legitimate value preserved
|
||||
assert_eq!(safe_duration_from_secs(2.5), Duration::from_secs_f64(2.5));
|
||||
// huge value clamped to the cap, not overflow-panicked
|
||||
assert_eq!(
|
||||
safe_duration_from_secs(1e300),
|
||||
Duration::from_secs_f64(MAX_DELAY_SECS)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_call_unregistered_returns_error() {
|
||||
let hc = HomeCore::new();
|
||||
|
||||
@@ -13,6 +13,26 @@ use homecore::{EntityId, StateMachine};
|
||||
|
||||
use crate::error::AutomationError;
|
||||
|
||||
/// Instruction budget for a single template render (HC-SEC-01).
|
||||
///
|
||||
/// Templates come from user automation config; without a bound a single
|
||||
/// `template:` condition like
|
||||
/// `{% for i in range(10000) %}{% for j in range(10000) %}x{% endfor %}{% endfor %}`
|
||||
/// renders a multi-gigabyte string and pins a CPU for tens of seconds —
|
||||
/// a memory/CPU denial-of-service (the bfld-class "unbounded expansion").
|
||||
/// MiniJinja's `fuel` feature charges ~1 unit per VM instruction; a
|
||||
/// nested loop burns one unit per iteration, so the budget caps total
|
||||
/// work regardless of how the loops are nested. 1,000,000 instructions is
|
||||
/// far more than any legitimate HA template needs (a typical condition is
|
||||
/// a few dozen) while killing the attack in well under a second.
|
||||
const TEMPLATE_FUEL: u64 = 1_000_000;
|
||||
|
||||
/// Hard cap on the source length of a template (HC-SEC-01, defense in
|
||||
/// depth). A legitimate HA `value_template` is a one-liner; anything past
|
||||
/// 64 KiB is rejected before compilation so a pathological source string
|
||||
/// can neither be compiled nor emitted verbatim.
|
||||
const MAX_TEMPLATE_SOURCE_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// MiniJinja environment pre-loaded with HA-compatible globals.
|
||||
///
|
||||
/// Constructed once per `AutomationEngine` and shared via `Arc`. The
|
||||
@@ -27,6 +47,10 @@ impl TemplateEnvironment {
|
||||
pub fn new(states: Arc<StateMachine>) -> Self {
|
||||
let mut env = Environment::new();
|
||||
|
||||
// Bound per-render work so a hostile `template:` condition cannot
|
||||
// DoS the engine via nested loops / huge repeats (HC-SEC-01).
|
||||
env.set_fuel(Some(TEMPLATE_FUEL));
|
||||
|
||||
// --- states(entity_id) ---
|
||||
// Returns the current state string of an entity, or "unavailable".
|
||||
let states_sm = Arc::clone(&states);
|
||||
@@ -88,7 +112,21 @@ impl TemplateEnvironment {
|
||||
}
|
||||
|
||||
/// Render a template string and return the string output.
|
||||
///
|
||||
/// Renders are bounded by an instruction budget ([`TEMPLATE_FUEL`]) and
|
||||
/// a source-length cap ([`MAX_TEMPLATE_SOURCE_BYTES`]); a malicious
|
||||
/// template that exhausts the budget returns a [`AutomationError::TemplateRender`]
|
||||
/// error rather than running unbounded (HC-SEC-01).
|
||||
pub fn render(&self, template_str: &str) -> Result<String, AutomationError> {
|
||||
// Reject pathologically large sources before compilation (defense
|
||||
// in depth — fuel already bounds runtime work).
|
||||
if template_str.len() > MAX_TEMPLATE_SOURCE_BYTES {
|
||||
return Err(AutomationError::TemplateRender(format!(
|
||||
"template source too large: {} bytes (max {})",
|
||||
template_str.len(),
|
||||
MAX_TEMPLATE_SOURCE_BYTES
|
||||
)));
|
||||
}
|
||||
// Wrap bare expressions like `{{ states('light.kitchen') }}`
|
||||
// in a minimal template wrapper.
|
||||
let tmpl = self
|
||||
@@ -191,4 +229,68 @@ mod tests {
|
||||
assert!(!env.render_bool("0").unwrap());
|
||||
assert!(!env.render_bool("off").unwrap());
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: template DoS is bounded by fuel ─────────────────────
|
||||
//
|
||||
// A `template:` condition is user config. Before the fuel bound a
|
||||
// nested-loop template rendered a multi-GB string over ~11 s (proven
|
||||
// empirically). With fuel enabled it must fail FAST with an error
|
||||
// instead of expanding unboundedly. On the pre-fix code (no `fuel`
|
||||
// feature / `set_fuel`) this render succeeds and burns CPU+RAM, so
|
||||
// this test fails on old (it would `Ok` and exceed the time bound).
|
||||
#[test]
|
||||
fn nested_loop_template_is_bounded_not_unbounded_dos() {
|
||||
use std::time::Instant;
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// 5000 * 5000 = 25M iterations on the old engine (~100 MB, ~11 s).
|
||||
let malicious =
|
||||
"{% for i in range(5000) %}{% for j in range(5000) %}xxxx{% endfor %}{% endfor %}";
|
||||
let start = Instant::now();
|
||||
let result = env.render(malicious);
|
||||
let elapsed = start.elapsed();
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"malicious nested-loop template must be rejected (ran out of fuel), got Ok"
|
||||
);
|
||||
assert!(
|
||||
elapsed.as_secs() < 3,
|
||||
"bounded render must fail fast; took {elapsed:?} (unbounded DoS on old engine)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: a single huge repeat is also bounded ────────────────
|
||||
#[test]
|
||||
fn single_huge_repeat_template_is_bounded() {
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// range() caps at 10k per call, but multiplied bodies still need a
|
||||
// bound; drive enough instructions to exhaust fuel via deep nesting.
|
||||
let malicious = "{% for a in range(9999) %}{% for b in range(9999) %}\
|
||||
{% for c in range(9999) %}z{% endfor %}{% endfor %}{% endfor %}";
|
||||
let result = env.render(malicious);
|
||||
assert!(result.is_err(), "deeply nested loops must exhaust fuel and error");
|
||||
}
|
||||
|
||||
// ── HC-SEC-01: oversized template source is rejected pre-compile ───
|
||||
#[test]
|
||||
fn oversized_template_source_is_rejected() {
|
||||
let sm = Arc::new(StateMachine::new());
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// 128 KiB of literal text — exceeds MAX_TEMPLATE_SOURCE_BYTES.
|
||||
let big = "x".repeat(128 * 1024);
|
||||
let result = env.render(&big);
|
||||
assert!(result.is_err(), "oversized template source must be rejected");
|
||||
}
|
||||
|
||||
// ── A legitimate small template still renders fine within budget ───
|
||||
#[test]
|
||||
fn legitimate_template_still_renders_within_fuel() {
|
||||
let sm = sm_with("light.kitchen", "on", serde_json::json!({}));
|
||||
let env = TemplateEnvironment::new(sm);
|
||||
// A normal HA condition with a modest loop — well under budget.
|
||||
let ok = "{% for i in range(50) %}{{ states('light.kitchen') }}{% endfor %}";
|
||||
let out = env.render(ok).expect("legitimate template must render");
|
||||
assert!(out.contains("on"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,15 @@ use homecore::event::{DomainEvent, StateChangedEvent};
|
||||
use crate::dedup::fnv64a_hash;
|
||||
use crate::schema::ALL_DDL;
|
||||
|
||||
/// Hard upper bound on rows returned by [`Recorder::get_state_history`].
|
||||
///
|
||||
/// Without this cap a wide `[since, until]` window over a high-frequency entity
|
||||
/// would load an unbounded number of rows into memory (a memory-DoS). The value
|
||||
/// is deliberately generous — large enough never to truncate a realistic
|
||||
/// history-graph query, small enough to bound the worst case. Callers needing a
|
||||
/// wider span page by narrowing the window.
|
||||
pub const MAX_HISTORY_ROWS: i64 = 1_000_000;
|
||||
|
||||
/// Errors returned by `Recorder` operations.
|
||||
#[derive(Error, Debug)]
|
||||
pub enum RecorderError {
|
||||
@@ -380,7 +389,17 @@ impl Recorder {
|
||||
}
|
||||
|
||||
/// Query state history for `entity_id` between `since` and `until`.
|
||||
/// Returns state snapshots in ascending `last_updated_ts` order.
|
||||
/// Returns state snapshots in ascending `last_updated_ts` order, capped at
|
||||
/// [`MAX_HISTORY_ROWS`] rows (oldest-first within the window).
|
||||
///
|
||||
/// ## Bounded result set (memory-DoS guard)
|
||||
///
|
||||
/// A high-frequency entity (e.g. a power sensor polled per-second) writes
|
||||
/// ~86k rows/day; a wide `[since, until]` window over months would otherwise
|
||||
/// load millions of rows into a single in-memory `Vec`, an unbounded-memory
|
||||
/// denial-of-service. The query therefore carries a hard `LIMIT` so the
|
||||
/// working set is bounded regardless of the requested time range. Callers
|
||||
/// that genuinely need a wider span must page by narrowing the window.
|
||||
pub async fn get_state_history(
|
||||
&self,
|
||||
entity_id: &EntityId,
|
||||
@@ -398,11 +417,13 @@ impl Recorder {
|
||||
WHERE s.entity_id = ? \
|
||||
AND s.last_updated_ts >= ? \
|
||||
AND s.last_updated_ts <= ? \
|
||||
ORDER BY s.last_updated_ts ASC",
|
||||
ORDER BY s.last_updated_ts ASC \
|
||||
LIMIT ?",
|
||||
)
|
||||
.bind(entity_id.as_str())
|
||||
.bind(since_ts)
|
||||
.bind(until_ts)
|
||||
.bind(MAX_HISTORY_ROWS)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
@@ -426,6 +447,79 @@ impl Recorder {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Purge history older than `older_than`, returning a [`PurgeStats`] summary.
|
||||
///
|
||||
/// Deletes:
|
||||
/// - `states` rows whose `last_updated_ts` is **strictly before** the cutoff,
|
||||
/// - `events` rows whose `time_fired_ts` is strictly before the cutoff,
|
||||
/// - then garbage-collects any `state_attributes` blob no surviving state
|
||||
/// row still references (so dedup-shared blobs are only dropped once their
|
||||
/// last referencing state is gone).
|
||||
///
|
||||
/// ## Retention boundary (data-integrity guard)
|
||||
///
|
||||
/// The cutoff is **exclusive**: a row exactly at `older_than` is retained.
|
||||
/// This makes `purge(t)` idempotent on the boundary and guarantees that a
|
||||
/// row written at the same instant the retention window opens is never lost
|
||||
/// to an off-by-one. Anything *at or after* `older_than` survives.
|
||||
///
|
||||
/// ## Atomicity (no partial-corrupt state)
|
||||
///
|
||||
/// All three deletes run inside a single transaction. A failure mid-purge
|
||||
/// rolls the whole operation back — the store is never left with states
|
||||
/// deleted but their events kept, or attributes orphaned by a half-purge.
|
||||
///
|
||||
/// Note: this reclaims logical rows; it does not `VACUUM` the file. SQLite
|
||||
/// reuses freed pages for subsequent writes, so disk growth stays bounded
|
||||
/// under a periodic purge even without an explicit vacuum.
|
||||
pub async fn purge(&self, older_than: DateTime<Utc>) -> Result<PurgeStats, RecorderError> {
|
||||
let cutoff_ts = older_than.timestamp_micros() as f64 / 1_000_000.0;
|
||||
|
||||
let mut tx = self.pool.begin().await?;
|
||||
|
||||
let states_deleted = sqlx::query("DELETE FROM states WHERE last_updated_ts < ?")
|
||||
.bind(cutoff_ts)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
let events_deleted = sqlx::query("DELETE FROM events WHERE time_fired_ts < ?")
|
||||
.bind(cutoff_ts)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
// GC attribute blobs no surviving state references. A dedup-shared blob
|
||||
// is only removed once its last referencing state row is gone.
|
||||
let attributes_deleted = sqlx::query(
|
||||
"DELETE FROM state_attributes \
|
||||
WHERE attributes_id NOT IN \
|
||||
(SELECT attributes_id FROM states WHERE attributes_id IS NOT NULL)",
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(PurgeStats {
|
||||
states_deleted,
|
||||
events_deleted,
|
||||
attributes_deleted,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary of a [`Recorder::purge`] run.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PurgeStats {
|
||||
/// Number of `states` rows deleted.
|
||||
pub states_deleted: u64,
|
||||
/// Number of `events` rows deleted.
|
||||
pub events_deleted: u64,
|
||||
/// Number of orphaned `state_attributes` blobs garbage-collected.
|
||||
pub attributes_deleted: u64,
|
||||
}
|
||||
|
||||
/// A state row returned from `get_state_history`.
|
||||
@@ -722,6 +816,214 @@ mod tests {
|
||||
assert!(rows.is_empty(), "genuine no-match is empty, not an error");
|
||||
}
|
||||
|
||||
// ── SQL injection (parameterization guarantee) ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn malicious_entity_id_is_stored_literally_not_executed() {
|
||||
// FAILS if any query interpolated entity_id into SQL: the `states` table
|
||||
// would be dropped and the later COUNT would error / mismatch. Bound
|
||||
// parameters store the metacharacter-laden string verbatim instead.
|
||||
let recorder = open_memory().await;
|
||||
|
||||
// A valid domain.name whose `name` part carries SQL metacharacters.
|
||||
// EntityId::parse permits this, so it reaches the bind path as data.
|
||||
let evil = "light.x_drop_table_states_select";
|
||||
recorder
|
||||
.record_state(&make_state_event(evil, "'; DROP TABLE states; --", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// states table still exists and holds exactly the one row we inserted.
|
||||
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM states")
|
||||
.fetch_one(&recorder.pool)
|
||||
.await
|
||||
.expect("states table must still exist — proves no injection");
|
||||
assert_eq!(count.0, 1);
|
||||
|
||||
// The malicious state string round-trips literally.
|
||||
let rows = recorder
|
||||
.search_states_by_text("DROP TABLE", 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 1, "metacharacter payload matched as a literal");
|
||||
assert_eq!(rows[0].state, "'; DROP TABLE states; --");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn like_metacharacters_in_query_are_literal_not_wildcards() {
|
||||
// A `%` in the search text must match a literal percent sign, not act as
|
||||
// a SQL LIKE wildcard. Proves the ESCAPE clause + metacharacter escaping.
|
||||
let recorder = open_memory().await;
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.a", "100%", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.b", "50", serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Literal "%" must match only sensor.a's "100%", NOT every row.
|
||||
let rows = recorder.search_states_by_text("%", 10).await.unwrap();
|
||||
assert_eq!(rows.len(), 1, "'%' is a literal, not a match-all wildcard");
|
||||
assert_eq!(rows[0].entity_id.as_str(), "sensor.a");
|
||||
|
||||
// Underscore is likewise literal: matches nothing here.
|
||||
let none = recorder.search_states_by_text("_", 10).await.unwrap();
|
||||
assert!(none.is_empty(), "'_' is literal, matches no row");
|
||||
}
|
||||
|
||||
// ── get_state_history bound (memory-DoS guard) ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn history_query_carries_a_limit_clause() {
|
||||
// Pin: the history SQL must carry a LIMIT bound (memory-DoS guard).
|
||||
// Inserting a million rows is infeasible in a unit test, so we prove the
|
||||
// clause is wired by bulk-inserting more rows than a deliberately tiny
|
||||
// bound and asserting the executed query honours a LIMIT. We bypass the
|
||||
// public method (whose cap is MAX_HISTORY_ROWS) and run the *same* SQL
|
||||
// shape with a small bind to demonstrate the LIMIT term is effective —
|
||||
// and separately assert the constant is a sane positive bound.
|
||||
assert!(MAX_HISTORY_ROWS > 0, "history cap must be positive");
|
||||
let recorder = open_memory().await;
|
||||
for v in &["1", "2", "3", "4", "5"] {
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.bounded", v, serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
// Same query shape as get_state_history, with a tiny LIMIT bind: if the
|
||||
// SQL lacked a LIMIT term this would return all 5; with it, exactly 2.
|
||||
let capped: Vec<(i64,)> = sqlx::query_as(
|
||||
"SELECT s.state_id FROM states s \
|
||||
WHERE s.entity_id = ? \
|
||||
ORDER BY s.last_updated_ts ASC LIMIT ?",
|
||||
)
|
||||
.bind("sensor.bounded")
|
||||
.bind(2_i64)
|
||||
.fetch_all(&recorder.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(capped.len(), 2, "LIMIT term effectively bounds the result set");
|
||||
|
||||
// And the real method returns all rows when under the cap.
|
||||
let eid = entity("sensor.bounded");
|
||||
let rows = recorder
|
||||
.get_state_history(&eid, Utc::now() - chrono::Duration::seconds(10), Utc::now() + chrono::Duration::seconds(10))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 5, "all rows under the cap return");
|
||||
}
|
||||
|
||||
// ── purge (retention correctness + atomicity) ───────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_keeps_boundary_row_and_drops_older() {
|
||||
// FAILS if purge had an off-by-one (deleting the row exactly at cutoff)
|
||||
// or deleted too much/too little. Cutoff is EXCLUSIVE: a row at the
|
||||
// cutoff instant survives; strictly-older rows are removed.
|
||||
let recorder = open_memory().await;
|
||||
let eid = entity("sensor.r");
|
||||
|
||||
// Three rows at known, increasing timestamps.
|
||||
for v in &["old", "mid", "new"] {
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.r", v, serde_json::json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
// Read back the actual timestamps so the cutoff is exact.
|
||||
let since = Utc::now() - chrono::Duration::seconds(60);
|
||||
let until = Utc::now() + chrono::Duration::seconds(60);
|
||||
let all = recorder.get_state_history(&eid, since, until).await.unwrap();
|
||||
assert_eq!(all.len(), 3);
|
||||
// Cut off exactly at the middle row's timestamp.
|
||||
let mid_ts = all[1].last_updated_ts;
|
||||
let cutoff = DateTime::<Utc>::from_timestamp_micros((mid_ts * 1_000_000.0) as i64).unwrap();
|
||||
|
||||
let stats = recorder.purge(cutoff).await.unwrap();
|
||||
assert_eq!(stats.states_deleted, 1, "only the strictly-older 'old' row");
|
||||
|
||||
let remaining = recorder.get_state_history(&eid, since, until).await.unwrap();
|
||||
assert_eq!(remaining.len(), 2, "boundary 'mid' row is KEPT (exclusive cutoff)");
|
||||
assert_eq!(remaining[0].state, "mid");
|
||||
assert_eq!(remaining[1].state, "new");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_gcs_orphaned_attributes_but_keeps_shared() {
|
||||
// Dedup means two states can share one attribute blob. Purging one of
|
||||
// them must NOT drop the still-referenced blob; purging the last one must.
|
||||
let recorder = open_memory().await;
|
||||
let shared = serde_json::json!({"unit": "C"});
|
||||
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.a", "20", shared.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
recorder
|
||||
.record_state(&make_state_event("sensor.b", "21", shared.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let attr_count = |r: &Recorder| {
|
||||
let pool = r.pool.clone();
|
||||
async move {
|
||||
let c: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM state_attributes")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
c.0
|
||||
}
|
||||
};
|
||||
assert_eq!(attr_count(&recorder).await, 1, "deduped to one blob");
|
||||
|
||||
// Purge before sensor.b's write → removes sensor.a only; blob still
|
||||
// referenced by sensor.b, so it must survive.
|
||||
let eid_b = entity("sensor.b");
|
||||
let rows_b = recorder
|
||||
.get_state_history(&eid_b, Utc::now() - chrono::Duration::seconds(60), Utc::now() + chrono::Duration::seconds(60))
|
||||
.await
|
||||
.unwrap();
|
||||
let b_ts = rows_b[0].last_updated_ts;
|
||||
let cutoff = DateTime::<Utc>::from_timestamp_micros((b_ts * 1_000_000.0) as i64).unwrap();
|
||||
let stats = recorder.purge(cutoff).await.unwrap();
|
||||
assert_eq!(stats.states_deleted, 1, "sensor.a purged");
|
||||
assert_eq!(stats.attributes_deleted, 0, "shared blob still referenced — kept");
|
||||
assert_eq!(attr_count(&recorder).await, 1, "blob survives");
|
||||
|
||||
// Now purge everything → sensor.b gone, blob orphaned → GC'd.
|
||||
let stats2 = recorder.purge(Utc::now() + chrono::Duration::seconds(120)).await.unwrap();
|
||||
assert_eq!(stats2.states_deleted, 1, "sensor.b purged");
|
||||
assert_eq!(stats2.attributes_deleted, 1, "now-orphaned blob GC'd");
|
||||
assert_eq!(attr_count(&recorder).await, 0, "no blobs remain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_also_removes_old_events() {
|
||||
let recorder = open_memory().await;
|
||||
let ctx = Context::new();
|
||||
recorder
|
||||
.record_event(&DomainEvent::new("call_service", serde_json::json!({}), ctx))
|
||||
.await
|
||||
.unwrap();
|
||||
// Purge with a far-future cutoff removes the event.
|
||||
let stats = recorder
|
||||
.purge(Utc::now() + chrono::Duration::seconds(120))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.events_deleted, 1);
|
||||
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM events")
|
||||
.fetch_one(&recorder.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count.0, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_semantic_falls_back_to_text_with_null_index() {
|
||||
// With the default NullSemanticIndex, search_semantic must STILL return
|
||||
|
||||
@@ -30,7 +30,7 @@ pub mod schema;
|
||||
pub mod semantic;
|
||||
|
||||
// Re-export the primary public API surface.
|
||||
pub use db::{Recorder, RecorderError};
|
||||
pub use db::{PurgeStats, Recorder, RecorderError, StateRow, MAX_HISTORY_ROWS};
|
||||
pub use listener::RecorderListener;
|
||||
|
||||
/// Null semantic index used when the `ruvector` feature is off.
|
||||
|
||||
@@ -135,10 +135,13 @@ pub fn render_events(event: &BfldEvent) -> Vec<TopicMessage> {
|
||||
|
||||
if let Some(zone) = &event.zone_id {
|
||||
// Emit a JSON string so consumers can distinguish "no zone" (omitted)
|
||||
// from "single-zone deployment" (always the same zone string).
|
||||
// from "single-zone deployment" (always the same zone string). The zone
|
||||
// name is operator-controlled; escape JSON metacharacters so a name
|
||||
// containing a quote or backslash cannot produce malformed/injected
|
||||
// JSON. Mirrors ha_discovery.rs::push_str_field's escaping.
|
||||
out.push(TopicMessage {
|
||||
topic: TopicMessage::ruview_topic(node, "zone_activity"),
|
||||
payload: format!("\"{zone}\""),
|
||||
payload: json_string_literal(zone),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -155,3 +158,26 @@ pub fn render_events(event: &BfldEvent) -> Vec<TopicMessage> {
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Wrap `value` in JSON double-quote delimiters, escaping the metacharacters
|
||||
/// that would otherwise break out of the string literal (`"`, `\`, control
|
||||
/// chars, and the bare `\n`/`\r`/`\t` whitespace). Kept in lockstep with
|
||||
/// `ha_discovery::push_str_field` so state-topic and discovery payloads escape
|
||||
/// identically.
|
||||
fn json_string_literal(value: &str) -> String {
|
||||
let mut out = String::with_capacity(value.len() + 2);
|
||||
out.push('"');
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
|
||||
c => out.push(c),
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
out
|
||||
}
|
||||
|
||||
@@ -141,6 +141,15 @@ impl BfldPipeline {
|
||||
/// builds the frame via [`BfldFrame::from_payload`] so the CRC covers the
|
||||
/// section-prefixed bytes.
|
||||
///
|
||||
/// The emitted frame's payload is forced into compliance with the active
|
||||
/// privacy class via [`crate::PrivacyGate::demote`]: at `Anonymous` the
|
||||
/// identity-leaky `compressed_angle_matrix` and `csi_delta` sections are
|
||||
/// stripped, and at `Restricted` the amplitude/phase proxies are stripped
|
||||
/// too. This closes the gap (ADR-141) where a frame stamped with a
|
||||
/// restrictive class byte could otherwise carry the full high-information
|
||||
/// BFI payload across a [`crate::NetworkSink`]. Research classes (`Raw`,
|
||||
/// `Derived`) keep the full payload — `demote` is a no-op there.
|
||||
///
|
||||
/// Returns `None` whenever the gate drops the underlying event (Reject or
|
||||
/// Recalibrate), so `process_to_frame` is a strict subset of `process`.
|
||||
pub fn process_to_frame(
|
||||
@@ -151,11 +160,21 @@ impl BfldPipeline {
|
||||
embedding: Option<IdentityEmbedding>,
|
||||
) -> Option<BfldFrame> {
|
||||
let timestamp_ns = inputs.timestamp_ns;
|
||||
let active_class = self.current_privacy_class();
|
||||
let _gate_signal = self.process(inputs, embedding)?;
|
||||
let mut header = header_template;
|
||||
header.timestamp_ns = timestamp_ns;
|
||||
header.privacy_class = self.current_privacy_class().as_u8();
|
||||
Some(BfldFrame::from_payload(header, &payload))
|
||||
header.privacy_class = active_class.as_u8();
|
||||
let frame = BfldFrame::from_payload(header, &payload);
|
||||
// Enforce the payload-content policy for the stamped class. The frame
|
||||
// is already at `active_class`, so this is a same-class demotion: it
|
||||
// performs no class change but strips the sections that class forbids.
|
||||
// demote() only fails on InvalidDemote (target < source), which cannot
|
||||
// happen here because source == target, so the expect is unreachable.
|
||||
Some(
|
||||
crate::PrivacyGate::demote(frame, active_class)
|
||||
.expect("same-class demote is always valid"),
|
||||
)
|
||||
}
|
||||
|
||||
/// `true` if `enable_privacy_mode()` has been called more recently than
|
||||
|
||||
@@ -127,6 +127,38 @@ fn zone_payload_is_json_string_with_quotes() {
|
||||
assert_eq!(zone.payload, "\"living_room\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zone_payload_escapes_json_metacharacters() {
|
||||
// A zone name containing a double-quote or backslash must not break out of
|
||||
// the JSON string literal it is emitted into. ha_discovery.rs already
|
||||
// escapes operator-controlled strings via push_str_field; render_events
|
||||
// must do the same for parity so the state-topic payload is always valid
|
||||
// JSON that Home Assistant can parse.
|
||||
let ev = BfldEvent::with_privacy_gating(
|
||||
"seed-01".into(),
|
||||
0,
|
||||
true,
|
||||
0.1,
|
||||
1,
|
||||
0.9,
|
||||
Some(r#"living"room\back"#.into()),
|
||||
PrivacyClass::Anonymous,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let msgs = render_events(&ev);
|
||||
let zone = msgs
|
||||
.iter()
|
||||
.find(|m| m.topic.contains("zone_activity"))
|
||||
.expect("zone_activity topic");
|
||||
// Expected: the inner quote and backslash are backslash-escaped, wrapped in
|
||||
// one pair of unescaped delimiter quotes -> a single valid JSON string.
|
||||
assert_eq!(zone.payload, r#""living\"room\\back""#);
|
||||
// And it must parse as JSON back to the original zone string.
|
||||
let parsed: String = serde_json::from_str(&zone.payload).expect("valid JSON string");
|
||||
assert_eq!(parsed, r#"living"room\back"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_risk_payload_is_fixed_precision_decimal() {
|
||||
let msgs = render_events(&sample_event(PrivacyClass::Anonymous, false));
|
||||
|
||||
@@ -88,6 +88,11 @@ fn process_to_frame_returns_none_under_sustained_high_risk() {
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_round_trips_through_bytes() {
|
||||
// Default pipeline class is Anonymous(2). The frame must round-trip through
|
||||
// wire bytes with no CRC error; the payload it carries is the privacy-gated
|
||||
// (angle-matrix-stripped) form, not the raw input — see
|
||||
// process_to_frame_at_anonymous_strips_identity_leaky_sections for the
|
||||
// content assertion. This test pins byte/CRC consistency only.
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
@@ -100,7 +105,10 @@ fn process_to_frame_round_trips_through_bytes() {
|
||||
let bytes = frame.to_bytes();
|
||||
let parsed = BfldFrame::from_bytes(&bytes).expect("frame must round-trip");
|
||||
let parsed_payload = parsed.parse_payload().expect("payload must round-trip");
|
||||
assert_eq!(parsed_payload, typed_payload());
|
||||
// Round-trip preserves whatever the privacy gate left in place.
|
||||
assert_eq!(parsed_payload, frame.parse_payload().unwrap());
|
||||
// And the identity surface is gone at Anonymous.
|
||||
assert!(parsed_payload.compressed_angle_matrix.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -141,6 +149,94 @@ fn process_to_frame_preserves_header_template_identity_fields() {
|
||||
assert_eq!({ frame.header.channel }, 36);
|
||||
}
|
||||
|
||||
// --- ADR-141 privacy-gate-correctness regression -------------------------
|
||||
//
|
||||
// `process_to_frame` stamps the frame with the pipeline's privacy_class but
|
||||
// (pre-fix) serialized the caller-supplied payload UNCHANGED. That let a frame
|
||||
// labeled Anonymous(2) / Restricted(3) carry the full identity-leaky
|
||||
// `compressed_angle_matrix` (+ amplitude/phase/csi_delta) that
|
||||
// `PrivacyGate::demote` is documented (privacy_gate_demote.rs) to strip at
|
||||
// exactly those classes. A NetworkSink accepts class >= Derived, so such a
|
||||
// frame would publish the beamforming angle matrix (identity surface) to the
|
||||
// network despite its restrictive class byte. These tests pin that the payload
|
||||
// content matches what the stamped class permits.
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_at_anonymous_strips_identity_leaky_sections() {
|
||||
// Default pipeline class is Anonymous(2): the angle matrix and csi_delta
|
||||
// MUST NOT survive into the emitted frame, matching PrivacyGate::demote.
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
let mut leaky = typed_payload();
|
||||
leaky.csi_delta = Some(vec![0x55; 24]);
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(1_700_000_000_000_000_000, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
leaky,
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("low-risk frame must be emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Anonymous.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert!(
|
||||
payload.compressed_angle_matrix.is_empty(),
|
||||
"Anonymous frame must NOT carry the compressed_angle_matrix (identity surface)",
|
||||
);
|
||||
assert!(
|
||||
payload.csi_delta.is_none(),
|
||||
"Anonymous frame must NOT carry csi_delta",
|
||||
);
|
||||
// Aggregate sensing sections survive.
|
||||
assert_eq!(payload.snr_vector.len(), 8);
|
||||
assert_eq!(payload.amplitude_proxy.len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_in_privacy_mode_strips_amplitude_and_phase() {
|
||||
// privacy_mode -> Restricted(3): amplitude + phase proxies must ALSO drop.
|
||||
let mut p = BfldPipeline::new(
|
||||
BfldConfig::new("seed-01").with_privacy_class(PrivacyClass::Anonymous),
|
||||
);
|
||||
p.enable_privacy_mode();
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(0, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
typed_payload(),
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("frame emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Restricted.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert!(payload.compressed_angle_matrix.is_empty(), "angle matrix stripped at Restricted");
|
||||
assert!(payload.amplitude_proxy.is_empty(), "amplitude stripped at Restricted");
|
||||
assert!(payload.phase_proxy.is_empty(), "phase stripped at Restricted");
|
||||
assert_eq!(payload.snr_vector.len(), 8, "snr_vector survives");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_at_derived_preserves_full_payload() {
|
||||
// Derived(1) is a research mode that legitimately keeps the angle matrix.
|
||||
// The strip must NOT over-fire at classes below Anonymous.
|
||||
let mut p = BfldPipeline::new(
|
||||
BfldConfig::new("seed-01").with_privacy_class(PrivacyClass::Derived),
|
||||
);
|
||||
let frame = p
|
||||
.process_to_frame(
|
||||
inputs(0, [0.1, 0.1, 0.1, 0.1]),
|
||||
header_template(),
|
||||
typed_payload(),
|
||||
Some(embedding()),
|
||||
)
|
||||
.expect("frame emitted");
|
||||
assert_eq!({ frame.header.privacy_class }, PrivacyClass::Derived.as_u8());
|
||||
let payload = frame.parse_payload().expect("payload parses");
|
||||
assert_eq!(
|
||||
payload, typed_payload(),
|
||||
"Derived research frame keeps the full payload unchanged",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_to_frame_uses_input_timestamp_not_template_timestamp() {
|
||||
let mut p = BfldPipeline::new(BfldConfig::new("seed-01"));
|
||||
|
||||
@@ -43,6 +43,20 @@ pub struct Features {
|
||||
pub const EMBED_MIN_SCORE: f32 = 0.25;
|
||||
|
||||
impl Features {
|
||||
/// The all-zero feature vector — the well-defined result of an empty (or
|
||||
/// wholly non-finite) capture. Total by construction: downstream
|
||||
/// specialists read it as "no signal" rather than panicking or poisoning a
|
||||
/// threshold (see [`Features::from_series`]).
|
||||
pub const ZERO: Features = Features {
|
||||
mean: 0.0,
|
||||
variance: 0.0,
|
||||
motion: 0.0,
|
||||
breathing_score: 0.0,
|
||||
breathing_hz: 0.0,
|
||||
heart_score: 0.0,
|
||||
heart_hz: 0.0,
|
||||
};
|
||||
|
||||
/// A fixed-length numeric embedding for nearest-prototype classifiers.
|
||||
///
|
||||
/// The hz components are zeroed unless their periodicity score clears
|
||||
@@ -77,29 +91,33 @@ impl Features {
|
||||
}
|
||||
|
||||
/// Extract features from a per-frame scalar series sampled at `fs` Hz.
|
||||
///
|
||||
/// **Total / fail-closed:** non-finite samples (`NaN`/`±inf`) are dropped
|
||||
/// before any statistic is computed, so a single garbage CSI frame cannot
|
||||
/// poison `mean`/`variance` into `NaN` and silently disable a persisted
|
||||
/// specialist (a `NaN` threshold makes every `>` comparison false). A
|
||||
/// series with no finite samples yields [`Features::ZERO`], exactly like
|
||||
/// the empty series. Same defensive contract as
|
||||
/// [`GeometryEmbedding`](crate::geometry_embedding::GeometryEmbedding):
|
||||
/// adversarial input degrades to "no signal", never to `NaN`.
|
||||
pub fn from_series(series: &[f32], fs: f32) -> Features {
|
||||
let n = series.len();
|
||||
// Drop non-finite samples: a corrupt frame counts as no frame, not as
|
||||
// a NaN that propagates through every downstream statistic.
|
||||
let clean: Vec<f32> = series.iter().copied().filter(|v| v.is_finite()).collect();
|
||||
let n = clean.len();
|
||||
if n == 0 {
|
||||
return Features {
|
||||
mean: 0.0,
|
||||
variance: 0.0,
|
||||
motion: 0.0,
|
||||
breathing_score: 0.0,
|
||||
breathing_hz: 0.0,
|
||||
heart_score: 0.0,
|
||||
heart_hz: 0.0,
|
||||
};
|
||||
return Features::ZERO;
|
||||
}
|
||||
let mean = series.iter().copied().sum::<f32>() / n as f32;
|
||||
let variance = series.iter().map(|v| (v - mean) * (v - mean)).sum::<f32>() / n as f32;
|
||||
let mean = clean.iter().copied().sum::<f32>() / n as f32;
|
||||
let variance = clean.iter().map(|v| (v - mean) * (v - mean)).sum::<f32>() / n as f32;
|
||||
let motion = if n > 1 {
|
||||
series.windows(2).map(|w| (w[1] - w[0]).abs()).sum::<f32>() / (n - 1) as f32
|
||||
clean.windows(2).map(|w| (w[1] - w[0]).abs()).sum::<f32>() / (n - 1) as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// De-mean before periodicity search.
|
||||
let centered: Vec<f32> = series.iter().map(|v| v - mean).collect();
|
||||
let centered: Vec<f32> = clean.iter().map(|v| v - mean).collect();
|
||||
let (breathing_hz, breathing_score) = autocorr_dominant(¢ered, fs, 0.1, 0.6);
|
||||
let (heart_hz, heart_score) = autocorr_dominant(¢ered, fs, 0.8, 3.0);
|
||||
|
||||
@@ -254,6 +272,36 @@ mod tests {
|
||||
assert_eq!(f.breathing_hz, 0.0);
|
||||
}
|
||||
|
||||
/// Fail-closed regression: a NaN/inf in the scalar series (corrupt CSI
|
||||
/// frame) must NOT poison the features into `NaN`/`inf`. Pre-fix, a single
|
||||
/// `NaN` made `mean`/`variance` `NaN`, which — baked into a persisted
|
||||
/// `PresenceSpecialist::threshold` — silently disabled presence detection
|
||||
/// (every `f.variance > NaN` is false). Non-finite samples are dropped.
|
||||
#[test]
|
||||
fn non_finite_samples_do_not_poison_features() {
|
||||
let f = Features::from_series(&[1.0, 2.0, f32::NAN, 4.0, f32::INFINITY, 6.0], 15.0);
|
||||
assert!(f.mean.is_finite(), "mean must stay finite, got {}", f.mean);
|
||||
assert!(f.variance.is_finite(), "variance must stay finite, got {}", f.variance);
|
||||
assert!(f.motion.is_finite(), "motion must stay finite, got {}", f.motion);
|
||||
for x in f.embedding() {
|
||||
assert!(x.is_finite(), "embedding slot non-finite: {x}");
|
||||
}
|
||||
// Mean is over the 4 finite samples {1,2,4,6} only.
|
||||
assert!((f.mean - 3.25).abs() < 1e-5, "mean over finite samples, got {}", f.mean);
|
||||
// Equivalence: dropping the non-finite samples must equal feeding only
|
||||
// the finite ones — proves the filter, not just finiteness.
|
||||
let only_finite = Features::from_series(&[1.0, 2.0, 4.0, 6.0], 15.0);
|
||||
assert_eq!(f, only_finite);
|
||||
}
|
||||
|
||||
/// A series with no finite samples degrades to the all-zero `ZERO`, exactly
|
||||
/// like the empty series — never `NaN`.
|
||||
#[test]
|
||||
fn all_non_finite_series_is_zero() {
|
||||
let f = Features::from_series(&[f32::NAN, f32::INFINITY, f32::NEG_INFINITY], 15.0);
|
||||
assert_eq!(f, Features::ZERO);
|
||||
}
|
||||
|
||||
/// ADR-152 "heart-band leakage" regression: a strong breathing rhythm must
|
||||
/// NOT register as a heart-band periodicity — its in-band autocorr maximum
|
||||
/// sits at the band edge (monotonic leak), not an interior peak.
|
||||
|
||||
@@ -15,6 +15,28 @@ use serde::{Deserialize, Serialize};
|
||||
use crate::anchor::{AnchorLabel, Posture};
|
||||
use crate::extract::{AnchorFeature, Features};
|
||||
|
||||
/// Default minimum breathing-band periodicity score to report a rate, used when
|
||||
/// a [`BreathingSpecialist`] carries no explicit `min_score` (the serde / pre-
|
||||
/// trained-default case). Respiration is a strong, narrowband modulation, so a
|
||||
/// moderate floor rejects noise windows without dropping real breaths.
|
||||
pub const DEFAULT_BREATHING_MIN_SCORE: f32 = 0.25;
|
||||
|
||||
/// Default minimum HR-band periodicity score, used when a [`HeartbeatSpecialist`]
|
||||
/// carries no explicit `min_score`. Higher than breathing's: sub-mm chest
|
||||
/// displacement at HR frequencies sits near the CSI noise floor (ADR-151 §3.2),
|
||||
/// so the heartbeat head demands a cleaner peak before reporting.
|
||||
pub const DEFAULT_HEARTBEAT_MIN_SCORE: f32 = 0.3;
|
||||
|
||||
/// Multiple of the typical inter-anchor spread ([`AnomalySpecialist::scale`])
|
||||
/// beyond which a live window is fully out-of-distribution (anomaly score 1.0):
|
||||
/// a window more than this many spreads from every enrolled prototype is novel.
|
||||
pub const ANOMALY_OUTLIER_SPREADS: f32 = 2.0;
|
||||
|
||||
/// Anomaly score above which the window is *labelled* "anomalous" (vs "normal").
|
||||
/// Distinct from the runtime veto threshold ([`crate::runtime`]); this only
|
||||
/// drives the human-readable label.
|
||||
pub const ANOMALY_LABEL_CUTOFF: f32 = 0.5;
|
||||
|
||||
/// Which biological signal a specialist estimates.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum SpecialistKind {
|
||||
@@ -229,7 +251,7 @@ impl Specialist for BreathingSpecialist {
|
||||
let min = if self.min_score > 0.0 {
|
||||
self.min_score
|
||||
} else {
|
||||
0.25
|
||||
DEFAULT_BREATHING_MIN_SCORE
|
||||
};
|
||||
if f.breathing_score < min || f.breathing_hz <= 0.0 {
|
||||
return None;
|
||||
@@ -258,7 +280,7 @@ impl Specialist for HeartbeatSpecialist {
|
||||
let min = if self.min_score > 0.0 {
|
||||
self.min_score
|
||||
} else {
|
||||
0.3
|
||||
DEFAULT_HEARTBEAT_MIN_SCORE
|
||||
};
|
||||
if f.heart_score < min || f.heart_hz <= 0.0 {
|
||||
return None;
|
||||
@@ -383,13 +405,13 @@ impl Specialist for AnomalySpecialist {
|
||||
.sqrt();
|
||||
best = best.min(d);
|
||||
}
|
||||
// >2× the typical spread → anomalous.
|
||||
let score = (best / (2.0 * self.scale)).clamp(0.0, 1.0);
|
||||
// Beyond ANOMALY_OUTLIER_SPREADS× the typical spread → fully anomalous.
|
||||
let score = (best / (ANOMALY_OUTLIER_SPREADS * self.scale)).clamp(0.0, 1.0);
|
||||
Some(SpecialistReading {
|
||||
kind: SpecialistKind::Anomaly,
|
||||
value: score,
|
||||
confidence: 0.6,
|
||||
label: Some(if score > 0.5 { "anomalous" } else { "normal" }.into()),
|
||||
label: Some(if score > ANOMALY_LABEL_CUTOFF { "anomalous" } else { "normal" }.into()),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -505,6 +527,32 @@ mod tests {
|
||||
assert!(b.infer(&feat(5.0, 0.2, 0.3, 0.1)).is_none()); // low score → none
|
||||
}
|
||||
|
||||
/// De-magic pin: the named default min-scores must equal the historical
|
||||
/// literal values, and the gate boundary must be `score >= min` (a window
|
||||
/// exactly at the default floor reports; a hair below does not).
|
||||
#[test]
|
||||
fn default_min_score_constants_match_prior_literals() {
|
||||
assert_eq!(DEFAULT_BREATHING_MIN_SCORE, 0.25);
|
||||
assert_eq!(DEFAULT_HEARTBEAT_MIN_SCORE, 0.3);
|
||||
let b = BreathingSpecialist::default(); // min_score = 0.0 → uses default
|
||||
assert!(
|
||||
b.infer(&feat(5.0, 0.2, 0.3, DEFAULT_BREATHING_MIN_SCORE)).is_some(),
|
||||
"score exactly at the default floor must report"
|
||||
);
|
||||
assert!(
|
||||
b.infer(&feat(5.0, 0.2, 0.3, DEFAULT_BREATHING_MIN_SCORE - 1e-3)).is_none(),
|
||||
"score below the default floor must not report"
|
||||
);
|
||||
}
|
||||
|
||||
/// De-magic pin for the anomaly score scale + label cutoff (value-identical
|
||||
/// to the prior `2.0 * scale` / `> 0.5` literals).
|
||||
#[test]
|
||||
fn anomaly_constants_match_prior_literals() {
|
||||
assert_eq!(ANOMALY_OUTLIER_SPREADS, 2.0);
|
||||
assert_eq!(ANOMALY_LABEL_CUTOFF, 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restlessness_normalizes() {
|
||||
let anchors = vec![
|
||||
|
||||
@@ -205,7 +205,7 @@ impl StreamingEngine {
|
||||
pub fn new(mode: PrivacyMode, model_version: u16, registration: GeoRegistration) -> Self {
|
||||
Self {
|
||||
fuser: MultistaticFuser::with_config(MultistaticConfig::default()),
|
||||
coherence_accept: 0.85,
|
||||
coherence_accept: Self::DEFAULT_COHERENCE_ACCEPT,
|
||||
privacy: PrivacyModeRegistry::new(mode),
|
||||
world: WorldGraph::new(registration),
|
||||
model_version,
|
||||
@@ -213,7 +213,11 @@ impl StreamingEngine {
|
||||
array: ArrayCoordinator::new(ArrayCoordinatorConfig::default()),
|
||||
node_geom: BTreeMap::new(),
|
||||
evolution: None,
|
||||
slam: RfSlam::with_discovery(0.5, 5, 0.6),
|
||||
slam: RfSlam::with_discovery(
|
||||
Self::SLAM_ASSOC_RADIUS_M,
|
||||
Self::SLAM_MIN_SIGHTINGS,
|
||||
Self::SLAM_MIN_COHERENCE,
|
||||
),
|
||||
person_tracks: BTreeMap::new(),
|
||||
semantic_retention: Self::DEFAULT_SEMANTIC_RETENTION,
|
||||
adapter: None,
|
||||
@@ -257,6 +261,31 @@ impl StreamingEngine {
|
||||
/// durable history belongs to the recorder).
|
||||
pub const DEFAULT_SEMANTIC_RETENTION: usize = 7_200;
|
||||
|
||||
/// Cross-node coherence at or above which fusion records a positive
|
||||
/// `CoherenceGateThreshold` evidence ref (ADR-137). Below it the cycle still
|
||||
/// emits, but without that corroborating evidence — so this gate shapes the
|
||||
/// trust record, not the privacy class. (== prior inline 0.85.)
|
||||
pub const DEFAULT_COHERENCE_ACCEPT: f32 = 0.85;
|
||||
|
||||
/// ADR-143 reflector-discovery parameters used to build the persistent
|
||||
/// `RfSlam`: association radius (m) within which two sightings are the same
|
||||
/// reflector, the minimum number of sightings before a reflector is
|
||||
/// considered stable, and the minimum per-sighting coherence to admit it.
|
||||
/// (== prior inline `with_discovery(0.5, 5, 0.6)`.)
|
||||
pub const SLAM_ASSOC_RADIUS_M: f64 = 0.5;
|
||||
/// Minimum sightings before a discovered reflector is treated as stable.
|
||||
pub const SLAM_MIN_SIGHTINGS: u64 = 5;
|
||||
/// Minimum per-sighting coherence to admit a reflector sighting.
|
||||
pub const SLAM_MIN_COHERENCE: f32 = 0.6;
|
||||
|
||||
/// ADR-143 static-anchor classification thresholds passed to
|
||||
/// `RfSlam::static_anchors`: the wall/ceiling stationarity ceiling and the
|
||||
/// mobile-reflector floor (anchors more mobile than this are dropped, not
|
||||
/// persisted). (== prior inline `static_anchors(0.05, 1.0)`.)
|
||||
pub const ANCHOR_WALL_CEILING: f64 = 0.05;
|
||||
/// Mobility floor above which a reflector is treated as mobile (skipped).
|
||||
pub const ANCHOR_MOBILE_FLOOR: f64 = 1.0;
|
||||
|
||||
/// Override the `SemanticState` retention cap (minimum 1).
|
||||
pub fn set_semantic_retention(&mut self, max_states: usize) {
|
||||
self.semantic_retention = max_states.max(1);
|
||||
@@ -331,7 +360,9 @@ impl StreamingEngine {
|
||||
self.slam.observe(obs);
|
||||
}
|
||||
let mut written = Vec::new();
|
||||
for (pos, class) in self.slam.static_anchors(0.05, 1.0) {
|
||||
for (pos, class) in
|
||||
self.slam.static_anchors(Self::ANCHOR_WALL_CEILING, Self::ANCHOR_MOBILE_FLOOR)
|
||||
{
|
||||
let kind = match class {
|
||||
wifi_densepose_signal::ruvsense::ReflectorClass::Wall => AnchorKind::Reflector,
|
||||
wifi_densepose_signal::ruvsense::ReflectorClass::Furniture => AnchorKind::Furniture,
|
||||
@@ -595,19 +626,46 @@ impl StreamingEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Domain-separation tag for the witness hash. Bumping this string
|
||||
/// intentionally invalidates every previously-recorded witness (a schema break).
|
||||
const WITNESS_DOMAIN: &[u8] = b"ruview.engine.witness.v1";
|
||||
|
||||
/// Length-prefix a variable-length field into the witness hash so adjacent
|
||||
/// fields can never be confused for one another. The 8-byte little-endian
|
||||
/// length makes the field framing unambiguous regardless of the bytes inside
|
||||
/// it (a field can contain the separator, the domain tag, anything).
|
||||
fn witness_field(h: &mut blake3::Hasher, bytes: &[u8]) {
|
||||
h.update(&(bytes.len() as u64).to_le_bytes());
|
||||
h.update(bytes);
|
||||
}
|
||||
|
||||
/// Deterministic BLAKE3 witness over a trust decision: the provenance tuple
|
||||
/// (evidence ‖ model ‖ calibration ‖ privacy decision) plus the effective
|
||||
/// privacy-class byte. Stable across runs for identical decisions — the
|
||||
/// "signed operational belief" fingerprint (ADR-137 §2.7 / ADR-028).
|
||||
///
|
||||
/// # Witness integrity (review finding: domain separation)
|
||||
/// Every privacy-relevant field is **length-prefixed** before hashing, and the
|
||||
/// (variable-length) evidence list is preceded by an explicit count. Without
|
||||
/// this framing the fields were concatenated boundary-to-boundary, so a string
|
||||
/// straddling a field boundary (e.g. an adapter id absorbing the leading bytes
|
||||
/// of the calibration epoch, or a model_version absorbing a trailing evidence
|
||||
/// ref) collided with a *different* trust decision — silently un-distinguishing
|
||||
/// two distinct privacy-relevant inputs and defeating the tamper/drift audit.
|
||||
/// `model_version` is operator-influenceable (per-room adapter id, ADR-150
|
||||
/// §3.4), so the ambiguity was reachable, not merely theoretical.
|
||||
fn witness_of(p: &SemanticProvenance, class: PrivacyClass) -> [u8; 32] {
|
||||
let mut h = blake3::Hasher::new();
|
||||
h.update(WITNESS_DOMAIN);
|
||||
// Explicit evidence count, then each ref length-prefixed: the number of
|
||||
// evidence refs is itself privacy-relevant and must be unambiguous.
|
||||
h.update(&(p.evidence.len() as u64).to_le_bytes());
|
||||
for e in &p.evidence {
|
||||
h.update(e.as_bytes());
|
||||
h.update(b"\x1f");
|
||||
witness_field(&mut h, e.as_bytes());
|
||||
}
|
||||
h.update(p.model_version.as_bytes());
|
||||
h.update(p.calibration_version.as_bytes());
|
||||
h.update(p.privacy_decision.as_bytes());
|
||||
witness_field(&mut h, p.model_version.as_bytes());
|
||||
witness_field(&mut h, p.calibration_version.as_bytes());
|
||||
witness_field(&mut h, p.privacy_decision.as_bytes());
|
||||
h.update(&[class.as_u8()]);
|
||||
*h.finalize().as_bytes()
|
||||
}
|
||||
@@ -1113,4 +1171,179 @@ mod tests {
|
||||
// StrictNoIdentity base = Restricted, even with no contradiction.
|
||||
assert_eq!(out.effective_class, PrivacyClass::Restricted);
|
||||
}
|
||||
|
||||
/// De-magic pin (review finding): the named engine constants must keep
|
||||
/// their prior inline values exactly, so the de-magic is a pure rename with
|
||||
/// no behavior change.
|
||||
#[test]
|
||||
fn engine_constants_match_prior_values() {
|
||||
assert_eq!(StreamingEngine::DEFAULT_COHERENCE_ACCEPT, 0.85);
|
||||
assert_eq!(StreamingEngine::SLAM_ASSOC_RADIUS_M, 0.5);
|
||||
assert_eq!(StreamingEngine::SLAM_MIN_SIGHTINGS, 5);
|
||||
assert_eq!(StreamingEngine::SLAM_MIN_COHERENCE, 0.6);
|
||||
assert_eq!(StreamingEngine::ANCHOR_WALL_CEILING, 0.05);
|
||||
assert_eq!(StreamingEngine::ANCHOR_MOBILE_FLOOR, 1.0);
|
||||
}
|
||||
|
||||
/// Privacy monotonicity (the crux): across EVERY base mode, a forced
|
||||
/// contradiction may only ever make the emitted class *more* restrictive
|
||||
/// (higher byte) and never less. Demotion is single-step and clamps at
|
||||
/// Restricted; a clean cycle emits exactly the base class. This is the
|
||||
/// information-only-removed invariant of ADR-141/120 stated as a property
|
||||
/// over the whole mode set.
|
||||
#[test]
|
||||
fn forced_contradiction_never_relaxes_class() {
|
||||
let cal_mismatch = [Some(CalibrationId(1)), Some(CalibrationId(2))]; // disagree → contradiction
|
||||
let cal_match = [Some(CalibrationId(5)), Some(CalibrationId(5))];
|
||||
let frames = [node_frame(0, 1000, 56), node_frame(1, 1001, 56)];
|
||||
for mode in [
|
||||
PrivacyMode::RawResearch,
|
||||
PrivacyMode::PrivateHome,
|
||||
PrivacyMode::EnterpriseAnonymous,
|
||||
PrivacyMode::CareWithConsent,
|
||||
PrivacyMode::StrictNoIdentity,
|
||||
] {
|
||||
let base_class = mode.target_class();
|
||||
|
||||
// Clean cycle: emits exactly the base class (no relaxation upward).
|
||||
let mut clean = StreamingEngine::new(mode, 1, GeoRegistration::default());
|
||||
let room_c = clean.add_room("r", "R");
|
||||
let oc = clean
|
||||
.process_cycle_calibrated(&frames, &cal_match, room_c, 1)
|
||||
.unwrap();
|
||||
assert_eq!(oc.effective_class, base_class, "clean cycle == base class");
|
||||
assert!(!oc.demoted);
|
||||
|
||||
// Forced contradiction: class byte only ever increases (more
|
||||
// restrictive), never decreases below the base.
|
||||
let mut dirty = StreamingEngine::new(mode, 1, GeoRegistration::default());
|
||||
let room_d = dirty.add_room("r", "R");
|
||||
let od = dirty
|
||||
.process_cycle_calibrated(&frames, &cal_mismatch, room_d, 1)
|
||||
.unwrap();
|
||||
assert!(od.demoted, "calibration mismatch must demote in {mode:?}");
|
||||
assert!(
|
||||
od.effective_class.as_u8() >= base_class.as_u8(),
|
||||
"demotion must never relax: {mode:?} base={:?} got={:?}",
|
||||
base_class,
|
||||
od.effective_class
|
||||
);
|
||||
// And it must be strictly more restrictive unless already clamped
|
||||
// at the most-restrictive class.
|
||||
if base_class != PrivacyClass::Restricted {
|
||||
assert!(
|
||||
od.effective_class.as_u8() > base_class.as_u8(),
|
||||
"unclamped demotion must increase restriction in {mode:?}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(od.effective_class, PrivacyClass::Restricted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fail-closed boundary: an empty cycle (zero frames) must NOT emit a
|
||||
/// trusted output at all — fusion rejects it and the engine surfaces a
|
||||
/// hard error. There is no degenerate output that could carry a stale or
|
||||
/// over-permissive class.
|
||||
#[test]
|
||||
fn empty_cycle_fails_closed() {
|
||||
let (mut e, room) = engine();
|
||||
let err = e.process_cycle(&[], CalibrationId(1), room, 1);
|
||||
assert!(matches!(err, Err(EngineError::Fusion(_))), "empty cycle must error, got {err:?}");
|
||||
// No SemanticState was appended (room + sensor only).
|
||||
assert_eq!(e.world().node_count(), 2);
|
||||
assert_eq!(e.cycle_count(), 0, "a failed cycle must not advance the counter");
|
||||
}
|
||||
|
||||
/// Single-node boundary characterization: a one-node cycle fuses (no
|
||||
/// multistatic cross-check is possible), reports no mesh (n<2), and emits a
|
||||
/// well-formed witness at the base class. Documents that single-node sensing
|
||||
/// is a valid, non-demoting mode — not a silent bypass.
|
||||
#[test]
|
||||
fn single_node_cycle_is_well_formed() {
|
||||
let (mut e, room) = engine();
|
||||
let out = e
|
||||
.process_cycle(&[node_frame(0, 1000, 56)], CalibrationId(1), room, 1)
|
||||
.unwrap();
|
||||
assert!(out.mesh.is_none(), "one node has no mesh cut");
|
||||
assert!(out.directional.is_none(), "no geometry registered");
|
||||
assert_eq!(out.effective_class, PrivacyClass::Anonymous); // PrivateHome base
|
||||
assert_ne!(out.witness, [0u8; 32], "witness still emitted");
|
||||
}
|
||||
|
||||
/// Witness domain-separation (review finding): the witness must change
|
||||
/// whenever ANY privacy-relevant field changes. The model_version,
|
||||
/// calibration_version, and privacy_decision fields are concatenated into
|
||||
/// the hash; without an unambiguous delimiter between them, a string that
|
||||
/// straddles the model/calibration boundary collides with a different
|
||||
/// (model, calibration) tuple.
|
||||
///
|
||||
/// `model_version` is operator-influenceable through the per-room adapter id
|
||||
/// (ADR-150 §3.4), and `calibration_version` is `cal:<hex>` — so the two
|
||||
/// provenances below are *both reachable* and represent genuinely different
|
||||
/// trust decisions (different model identity, different calibration epoch),
|
||||
/// yet the field-boundary ambiguity makes them hash-collide. A colliding
|
||||
/// witness silently un-distinguishes two distinct privacy-relevant inputs,
|
||||
/// defeating the tamper/drift audit guarantee.
|
||||
#[test]
|
||||
fn witness_distinguishes_model_calibration_boundary() {
|
||||
let class = PrivacyClass::Anonymous;
|
||||
// A: model "rfenc-v1+adapter:X", calibration epoch "cal:00ab".
|
||||
let a = SemanticProvenance {
|
||||
evidence: vec!["ev".into()],
|
||||
model_version: "rfenc-v1+adapter:X".into(),
|
||||
calibration_version: "cal:00ab".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
// B: adapter id absorbs the leading "cal:00a" of A's calibration; B's
|
||||
// own calibration is the remaining "b". A.model‖A.cal == B.model‖B.cal,
|
||||
// so the unseparated concatenation hashes identically — yet these are
|
||||
// distinct (model identity, calibration epoch) tuples.
|
||||
let b = SemanticProvenance {
|
||||
evidence: vec!["ev".into()],
|
||||
model_version: "rfenc-v1+adapter:Xcal:00a".into(),
|
||||
calibration_version: "b".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
assert_ne!(a.model_version, b.model_version);
|
||||
assert_ne!(a.calibration_version, b.calibration_version);
|
||||
// Sanity: the two collide under naive concatenation.
|
||||
assert_eq!(
|
||||
format!("{}{}", a.model_version, a.calibration_version),
|
||||
format!("{}{}", b.model_version, b.calibration_version),
|
||||
);
|
||||
assert_ne!(
|
||||
witness_of(&a, class),
|
||||
witness_of(&b, class),
|
||||
"distinct (model, calibration) tuples must not share a witness"
|
||||
);
|
||||
}
|
||||
|
||||
/// Witness domain-separation across the evidence/model boundary: a witness
|
||||
/// must distinguish an extra evidence ref from a model_version that absorbs
|
||||
/// the same bytes. The evidence loop terminates each ref with one separator;
|
||||
/// the model field must itself be unambiguously delimited from the (variable
|
||||
/// number of) evidence refs that precede it.
|
||||
#[test]
|
||||
fn witness_distinguishes_evidence_model_boundary() {
|
||||
let class = PrivacyClass::Anonymous;
|
||||
let a = SemanticProvenance {
|
||||
evidence: vec!["e1".into(), "e2".into()],
|
||||
model_version: "m".into(),
|
||||
calibration_version: "cal:1".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
let b = SemanticProvenance {
|
||||
evidence: vec!["e1".into()],
|
||||
// absorbs "e2" + its 0x1f separator into the model field.
|
||||
model_version: "e2\u{1f}m".into(),
|
||||
calibration_version: "cal:1".into(),
|
||||
privacy_decision: "PrivateHome/Anonymous".into(),
|
||||
};
|
||||
assert_ne!(
|
||||
witness_of(&a, class),
|
||||
witness_of(&b, class),
|
||||
"an extra evidence ref must not collide with a model_version that absorbs it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,11 @@ pub fn haversine(a: &GeoPoint, b: &GeoPoint) -> f64 {
|
||||
let lat1 = a.lat.to_radians();
|
||||
let lat2 = b.lat.to_radians();
|
||||
let h = (dlat / 2.0).sin().powi(2) + lat1.cos() * lat2.cos() * (dlon / 2.0).sin().powi(2);
|
||||
2.0 * WGS84_A * h.sqrt().asin()
|
||||
// `asin` is only defined on [-1, 1]. For (near-)antipodal points floating
|
||||
// rounding can push `h.sqrt()` to 1.0 + epsilon, and `asin(>1)` is NaN —
|
||||
// which would silently poison any distance-based comparison downstream.
|
||||
// Clamp into domain so the result is always a finite distance.
|
||||
2.0 * WGS84_A * h.sqrt().clamp(0.0, 1.0).asin()
|
||||
}
|
||||
|
||||
/// WGS84 to local ENU (East-North-Up) relative to origin, in meters.
|
||||
@@ -83,3 +87,73 @@ pub fn tiles_for_bbox(bbox: &GeoBBox, zoom: u8) -> Vec<TileCoord> {
|
||||
}
|
||||
tiles
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── haversine asin-domain robustness ───────────────────────────────────
|
||||
//
|
||||
// For (near-)antipodal points, floating rounding can push the haversine
|
||||
// term `h` to 1.0 + ~4e-16, and `asin(sqrt(h)) = asin(>1)` is NaN. A NaN
|
||||
// distance silently breaks every downstream comparison (all `<`/`>` become
|
||||
// false), so the result must stay finite. This exact pair produced
|
||||
// h = 1.0000000000000004 pre-fix (verified empirically).
|
||||
|
||||
#[test]
|
||||
fn haversine_near_antipodal_is_finite_not_nan() {
|
||||
let a = GeoPoint {
|
||||
lat: -44.4994,
|
||||
lon: -178.957_22,
|
||||
alt: 0.0,
|
||||
};
|
||||
let b = GeoPoint {
|
||||
lat: 44.499_399_99,
|
||||
lon: 1.042_780_01,
|
||||
alt: 0.0,
|
||||
};
|
||||
let d = haversine(&a, &b);
|
||||
assert!(d.is_finite(), "near-antipodal haversine must be finite, got {d}");
|
||||
// Half-circumference is ~20_037 km; result must be close to that.
|
||||
assert!(
|
||||
(19_000_000.0..21_000_000.0).contains(&d),
|
||||
"antipodal distance should be ~half-circumference, got {d}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn haversine_identical_points_is_zero() {
|
||||
let p = GeoPoint {
|
||||
lat: 43.65,
|
||||
lon: -79.38,
|
||||
alt: 0.0,
|
||||
};
|
||||
let d = haversine(&p, &p);
|
||||
assert!(d.is_finite() && d < 1e-6, "identical points → 0, got {d}");
|
||||
}
|
||||
|
||||
// ── pole-singularity robustness (degenerate geometry) ──────────────────
|
||||
//
|
||||
// The ENU transforms divide by cos(lat); at the poles cos(±90°) = 0, so
|
||||
// the longitude term is non-finite. We do not change the transform (that
|
||||
// would alter near-pole results), but we pin that the call does NOT panic.
|
||||
|
||||
#[test]
|
||||
fn wgs84_to_enu_at_pole_does_not_panic() {
|
||||
let origin = GeoPoint {
|
||||
lat: 90.0,
|
||||
lon: 0.0,
|
||||
alt: 0.0,
|
||||
};
|
||||
let point = GeoPoint {
|
||||
lat: 89.99,
|
||||
lon: 10.0,
|
||||
alt: 0.0,
|
||||
};
|
||||
// Must return without panicking. North/up stay finite; east may be
|
||||
// non-finite at the exact pole — assert the bounded components only.
|
||||
let enu = wgs84_to_enu(&point, &origin);
|
||||
assert!(enu[1].is_finite(), "north component must be finite");
|
||||
assert!(enu[2].is_finite(), "up component must be finite");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,21 @@ pub fn parse_hgt(data: &[u8], origin_lat: f64, origin_lon: f64) -> Result<Elevat
|
||||
let n_samples = data.len() / 2;
|
||||
let side = (n_samples as f64).sqrt() as usize;
|
||||
|
||||
// A valid SRTM grid is at least 2x2 — anything smaller has no cell spacing.
|
||||
// Without this guard, `side - 1` underflows (panic in debug, wraps to a
|
||||
// huge value in release) and `1.0 / (side - 1)` yields a garbage/inf
|
||||
// `cell_size_deg` that then poisons every `ElevationGrid::get` lookup. A
|
||||
// truncated download, a 404 HTML body, or an empty response can all reach
|
||||
// here, so fail loudly instead of corrupting the persisted grid.
|
||||
if side < 2 {
|
||||
anyhow::bail!(
|
||||
"HGT data too small: {} bytes ({} samples, side {}) — need at least a 2x2 grid",
|
||||
data.len(),
|
||||
n_samples,
|
||||
side
|
||||
);
|
||||
}
|
||||
|
||||
let heights: Vec<f32> = data
|
||||
.chunks_exact(2)
|
||||
.map(|c| {
|
||||
@@ -129,3 +144,42 @@ pub fn extract_subgrid(grid: &ElevationGrid, center: &GeoPoint, radius_m: f64) -
|
||||
heights,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── parse_hgt degenerate-input robustness ──────────────────────────────
|
||||
//
|
||||
// Before the `side < 2` guard, an empty or sub-2x2 buffer made
|
||||
// `1.0 / (side - 1)` underflow `side` (panic in debug / huge wrap in
|
||||
// release) and produce a garbage `cell_size_deg`. A truncated download or
|
||||
// a 404 HTML page reaches `parse_hgt`, so these must Err, not panic/poison.
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_empty_data_errors_not_panics() {
|
||||
let res = parse_hgt(&[], 40.0, -75.0);
|
||||
assert!(res.is_err(), "empty HGT must Err, got {res:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_single_sample_errors() {
|
||||
// 2 bytes = 1 sample → side 1 → div-by-zero cell_size (inf) pre-fix.
|
||||
let res = parse_hgt(&[0u8, 0u8], 40.0, -75.0);
|
||||
assert!(res.is_err(), "1-sample HGT must Err, got {res:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_hgt_minimal_2x2_is_finite() {
|
||||
// 4 samples = 8 bytes → side 2 → cell_size = 1.0 (finite, valid).
|
||||
let data = vec![0u8; 8];
|
||||
let grid = parse_hgt(&data, 40.0, -75.0).expect("2x2 HGT should parse");
|
||||
assert_eq!(grid.cols, 2);
|
||||
assert_eq!(grid.rows, 2);
|
||||
assert!(
|
||||
grid.cell_size_deg.is_finite() && grid.cell_size_deg > 0.0,
|
||||
"cell_size must be finite positive, got {}",
|
||||
grid.cell_size_deg
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -700,4 +700,79 @@ mod tests {
|
||||
assert!(conf > 0.7, "self-similarity should exceed match threshold");
|
||||
}
|
||||
}
|
||||
|
||||
// ── NaN-state-poisoning guard (the proven recurring bug class) ──────────
|
||||
//
|
||||
// The calibration/vitals crates were both bitten by a single non-finite
|
||||
// sample latching into persistent state and freezing all outputs forever.
|
||||
// Here the auto-accumulating persistent state is `occupancy` (an EMA:
|
||||
// `*occ = *occ*0.7 + new*0.3`) and `vitals` (motion/breathing/heart).
|
||||
//
|
||||
// The UDP parser can only ever emit finite amplitudes/phases (sqrt and
|
||||
// atan2 of i8 values), so the realistic ingress is already safe. This test
|
||||
// is stronger: it injects an adversarial hand-built `CsiFrame` carrying
|
||||
// NaN/inf amplitudes and phases (possible because the fields are public),
|
||||
// and pins that the persistent state self-heals to finite values rather
|
||||
// than latching NaN and silently freezing — i.e. the bug class is absent.
|
||||
#[test]
|
||||
fn nonfinite_frame_does_not_poison_persistent_state() {
|
||||
let mut s = CsiPipelineState::default();
|
||||
// Warm up with valid frames so vitals/occupancy are populated.
|
||||
seed_state_with_frames(&mut s, 60);
|
||||
|
||||
// A valid baseline must be finite to start.
|
||||
assert!(s.occupancy.iter().all(|d| d.is_finite()));
|
||||
assert!(s.vitals.breathing_rate.is_finite());
|
||||
assert!(s.vitals.motion_score.is_finite());
|
||||
|
||||
// Inject a stream of poisoned frames: NaN/inf amplitudes + phases on a
|
||||
// valid header (node_id 1, finite rssi). Mimics a corrupt sensor.
|
||||
for i in 0..40 {
|
||||
let nan_frame = CsiFrame {
|
||||
node_id: 1,
|
||||
n_antennas: 1,
|
||||
n_subcarriers: 32,
|
||||
channel: 6,
|
||||
rssi: -50,
|
||||
noise_floor: -90,
|
||||
timestamp_us: 10_000 + i,
|
||||
iq_data: vec![0i8; 64],
|
||||
amplitudes: vec![f32::NAN; 32],
|
||||
phases: vec![f32::INFINITY; 32],
|
||||
};
|
||||
s.process_frame(nan_frame);
|
||||
}
|
||||
|
||||
// Persistent auto-accumulating state must remain finite — a single
|
||||
// poisoned frame (or 40) must not permanently corrupt outputs.
|
||||
assert!(
|
||||
s.occupancy.iter().all(|d| d.is_finite()),
|
||||
"occupancy EMA must not latch NaN/inf"
|
||||
);
|
||||
assert!(
|
||||
s.vitals.breathing_rate.is_finite(),
|
||||
"breathing_rate must stay finite, got {}",
|
||||
s.vitals.breathing_rate
|
||||
);
|
||||
assert!(
|
||||
s.vitals.heart_rate.is_finite(),
|
||||
"heart_rate must stay finite, got {}",
|
||||
s.vitals.heart_rate
|
||||
);
|
||||
assert!(
|
||||
s.vitals.motion_score.is_finite(),
|
||||
"motion_score must stay finite, got {}",
|
||||
s.vitals.motion_score
|
||||
);
|
||||
|
||||
// And the pipeline must recover: feeding valid frames again yields a
|
||||
// finite, in-range breathing estimate (not a frozen NaN).
|
||||
seed_state_with_frames(&mut s, 60);
|
||||
assert!(s.vitals.breathing_rate.is_finite());
|
||||
assert!(
|
||||
(0.0..=40.0).contains(&s.vitals.breathing_rate),
|
||||
"breathing must be in clamp range after recovery, got {}",
|
||||
s.vitals.breathing_rate
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,4 +184,43 @@ mod tests {
|
||||
let fused = fuse_clouds(&[&a], 0.5);
|
||||
assert_eq!(fused.points.len(), 1, "three close points → one voxel");
|
||||
}
|
||||
|
||||
// ── degenerate-input robustness (no panic, sensible output) ────────────
|
||||
//
|
||||
// These pin that the voxel accumulators handle empty / single / all-
|
||||
// coincident inputs without dividing by zero or panicking. The per-voxel
|
||||
// count is always >= 1 (the entry is created on first insert), so the
|
||||
// `/n` averaging is safe — but make that contract explicit so a future
|
||||
// refactor cannot silently reintroduce a div-by-zero.
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_empty_input_is_empty() {
|
||||
let fused = fuse_clouds(&[], 0.1);
|
||||
assert!(fused.points.is_empty(), "no clouds → no points");
|
||||
let empty = PointCloud::new("empty");
|
||||
let fused2 = fuse_clouds(&[&empty], 0.1);
|
||||
assert!(fused2.points.is_empty(), "empty cloud → no points");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_single_point_is_finite() {
|
||||
let a = cloud_with("a", &[(1.0, 2.0, 3.0)]);
|
||||
let fused = fuse_clouds(&[&a], 0.1);
|
||||
assert_eq!(fused.points.len(), 1);
|
||||
let p = &fused.points[0];
|
||||
assert!(
|
||||
p.x.is_finite() && p.y.is_finite() && p.z.is_finite() && p.intensity.is_finite(),
|
||||
"single-point voxel must average to a finite point"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuse_clouds_all_coincident_collapses_finite() {
|
||||
// Many identical points → one voxel, finite averaged centroid.
|
||||
let a = cloud_with("a", &[(0.5, 0.5, 0.5); 100]);
|
||||
let fused = fuse_clouds(&[&a], 0.25);
|
||||
assert_eq!(fused.points.len(), 1, "coincident points → one voxel");
|
||||
let p = &fused.points[0];
|
||||
assert!((p.x - 0.5).abs() < 1e-4 && p.x.is_finite());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
edition = "2021"
|
||||
description = "ADR-262 anti-corruption bridge: converts RuView WiFi-CSI sensing output into signed RuField FieldEvents (P0–P5 privacy mapping + ed25519 provenance)"
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
# ADR-262 §5.4: this crate is the single coupling point ("anti-corruption
|
||||
# layer") between RuView and the standalone RuField MFS spec. It depends on the
|
||||
# `vendor/rufield` submodule crates **via path** (the `vendor/rvcsi` pattern) —
|
||||
# RuView does NOT depend on published rufield crates (there are none) and does
|
||||
# NOT make rufield a v2 workspace member. The four crates below are pure-Rust
|
||||
# (serde / serde_json / toml / sha2 / ed25519-dalek only — no tch / openblas /
|
||||
# ndarray / candle), so they build under `--no-default-features`.
|
||||
[dependencies]
|
||||
rufield-core = { path = "../../../vendor/rufield/crates/rufield-core" }
|
||||
rufield-provenance = { path = "../../../vendor/rufield/crates/rufield-provenance" }
|
||||
rufield-privacy = { path = "../../../vendor/rufield/crates/rufield-privacy" }
|
||||
rufield-fusion = { path = "../../../vendor/rufield/crates/rufield-fusion" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,206 @@
|
||||
//! The conversion: `SensingSnapshot` → signed `FieldEvent` (ADR-262 P1).
|
||||
//!
|
||||
//! This is the in-process `SensingServerAdapter` core (ADR-262 §4 P1 / §5.1):
|
||||
//! it consumes a `(SensingUpdate, TrustedOutput)` join — modelled here as a
|
||||
//! [`SensingSnapshot`] of owned primitives — and emits one signed
|
||||
//! [`FieldEvent`] (`Modality::WifiCsi`, axis `[Frequency]`) per cycle.
|
||||
|
||||
use crate::privacy::egress_class;
|
||||
use crate::snapshot::{SensingSnapshot, SignalField};
|
||||
use rufield_core::{
|
||||
FieldAxis, FieldEvent, FieldTensor, Modality, Observation, PrivacyClass, ProvenanceRef,
|
||||
SensorDescriptor,
|
||||
};
|
||||
use rufield_provenance::{sha256_hex, Signer};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Model id stamped on emitted events (ADR-262 — derived features come from
|
||||
/// RuView's `/ws/sensing` pipeline, not a trained encoder).
|
||||
const MODEL_ID: &str = "ruview_sensing_server_v1";
|
||||
|
||||
/// Firmware hash placeholder until the real ESP32 firmware image hash is wired
|
||||
/// through (ADR-262 §8 open question 3 — the BLAKE3 engine witness slot). A
|
||||
/// stable `sha256:` over the model id keeps it a real digest, not a fake.
|
||||
fn firmware_hash() -> String {
|
||||
sha256_hex(MODEL_ID.as_bytes())
|
||||
}
|
||||
|
||||
/// Squash a non-negative power-like scalar into `[0, 1]` deterministically.
|
||||
/// `x / (x + 1)` — monotone, no panics, no calibration claim.
|
||||
fn squash(x: f64) -> f32 {
|
||||
if !x.is_finite() || x <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(x / (x + 1.0)) as f32
|
||||
}
|
||||
|
||||
/// Build the `Observation.features` map the RuField fusion engine reads
|
||||
/// (`rufield-fusion/engine.rs:217-228`: `motion_energy`, `breathing_band`,
|
||||
/// `transient`, `presence`, `range_m`, plus `posture_height`).
|
||||
fn build_features(snap: &SensingSnapshot, range_m: Option<f32>) -> BTreeMap<String, f32> {
|
||||
let f = &snap.features;
|
||||
let mut m = BTreeMap::new();
|
||||
m.insert("motion_energy".to_string(), squash(f.motion_band_power));
|
||||
m.insert("breathing_band".to_string(), squash(f.breathing_band_power));
|
||||
m.insert("transient".to_string(), squash(f.change_points as f64));
|
||||
m.insert(
|
||||
"presence".to_string(),
|
||||
if snap.classification.presence { 1.0 } else { 0.0 },
|
||||
);
|
||||
if let Some(r) = range_m {
|
||||
m.insert("range_m".to_string(), r);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// Derive a real range (metres) and motion vector from the strongest signal
|
||||
/// field peak, if a field is present. Returns `(range_m, motion_vector,
|
||||
/// space_cell)` — all `None` when there is no field (we do NOT fabricate
|
||||
/// coordinates, per ADR-262 §4 P1).
|
||||
fn derive_position(
|
||||
field: Option<&SignalField>,
|
||||
) -> (Option<f32>, Option<[f32; 3]>, Option<[i32; 3]>) {
|
||||
let Some(field) = field else {
|
||||
return (None, None, None);
|
||||
};
|
||||
let Some(cell) = field.peak_cell() else {
|
||||
return (None, None, None);
|
||||
};
|
||||
// Range from origin in grid-cell units (real readout, not calibrated
|
||||
// metres — the honesty caveat from `field_localize.rs:16-27`).
|
||||
let [x, y, z] = cell;
|
||||
let range = ((x * x + y * y + z * z) as f32).sqrt();
|
||||
let mag = if range > 0.0 { range } else { 1.0 };
|
||||
let motion_vector = [x as f32 / mag, y as f32 / mag, z as f32 / mag];
|
||||
(Some(range), Some(motion_vector), Some(cell))
|
||||
}
|
||||
|
||||
/// Stable, deterministic event id from `(node_id, timestamp_ns)`. No RNG, so
|
||||
/// the same snapshot always yields the same id (required for the determinism
|
||||
/// gate).
|
||||
fn event_id(snap: &SensingSnapshot) -> String {
|
||||
format!("ruview-{}-{}", snap.node_id, snap.timestamp_ns)
|
||||
}
|
||||
|
||||
/// Convert a [`SensingSnapshot`] to a **signed** [`FieldEvent`] (ADR-262 P1).
|
||||
///
|
||||
/// 1. Builds a `FieldTensor` (`Modality::WifiCsi`, axis `[Frequency]`) whose
|
||||
/// values are the RuView feature scalars, with the real `timestamp_ns`.
|
||||
/// 2. Builds an `Observation` — `motion_vector`/`range_m`/`space_cell` derived
|
||||
/// from the signal-field peak when present (else `None`; coordinates are
|
||||
/// never fabricated), `confidence` from the classification, labels from
|
||||
/// motion-level/presence.
|
||||
/// 3. Stamps the §3.3 egress privacy class (information-content mapping with
|
||||
/// the demotion floor) on both tensor and observation.
|
||||
/// 4. Builds a real `ProvenanceRef` (sha256 raw hash over the tensor/feature
|
||||
/// bytes, `synthetic = false`) and **signs** it with the supplied ed25519
|
||||
/// [`Signer`] so `rufield_provenance::is_fusable` passes.
|
||||
///
|
||||
/// Determinism: with no RNG anywhere and a deterministic ed25519 signer, the
|
||||
/// same `snap` + same signer seed yields a byte-identical event.
|
||||
#[must_use]
|
||||
pub fn snapshot_to_field_event(snap: &SensingSnapshot, signer: &Signer) -> FieldEvent {
|
||||
let class = egress_class(snap.trust_class, snap.identity_bound, snap.demoted);
|
||||
|
||||
let (range_m, motion_vector, space_cell) = derive_position(snap.signal_field.as_ref());
|
||||
|
||||
// ── 1. Tensor ──────────────────────────────────────────────────────────
|
||||
// The frequency-domain feature scalars, in a stable order.
|
||||
let f = &snap.features;
|
||||
let values: Vec<f32> = vec![
|
||||
f.mean_rssi as f32,
|
||||
f.variance as f32,
|
||||
f.motion_band_power as f32,
|
||||
f.breathing_band_power as f32,
|
||||
f.dominant_freq_hz as f32,
|
||||
f.spectral_power as f32,
|
||||
];
|
||||
let confidence = (snap.classification.confidence as f32).clamp(0.0, 1.0);
|
||||
let noise_floor = f.variance.max(0.0) as f32;
|
||||
let calibration_id = format!("ruview_node_{}", snap.node_id);
|
||||
|
||||
// `FieldTensor::new` only errors on a shape/axis mismatch; our shape
|
||||
// exactly matches `values.len()` and one axis, so this is infallible here.
|
||||
let tensor = FieldTensor::new(
|
||||
snap.timestamp_ns,
|
||||
Modality::WifiCsi,
|
||||
vec![FieldAxis::Frequency],
|
||||
vec![values.len()],
|
||||
values,
|
||||
confidence,
|
||||
noise_floor,
|
||||
Some(calibration_id.clone()),
|
||||
class,
|
||||
)
|
||||
.expect("feature tensor shape is well-formed by construction");
|
||||
|
||||
// ── 2. Observation ─────────────────────────────────────────────────────
|
||||
let observation = Observation {
|
||||
zone_id: Some(snap.node_id.clone()),
|
||||
space_cell,
|
||||
range_m,
|
||||
velocity_mps: None,
|
||||
motion_vector,
|
||||
confidence,
|
||||
features: build_features(snap, range_m),
|
||||
labels: build_labels(snap),
|
||||
privacy_class: class,
|
||||
};
|
||||
|
||||
// ── 3. Provenance (real sha256 over the tensor bytes) ───────────────────
|
||||
let raw_hash = sha256_hex(
|
||||
&serde_json::to_vec(&tensor).expect("tensor serializes to JSON for hashing"),
|
||||
);
|
||||
let provenance = ProvenanceRef {
|
||||
raw_hash,
|
||||
firmware_hash: firmware_hash(),
|
||||
model_id: MODEL_ID.to_string(),
|
||||
calibration_id,
|
||||
synthetic: false, // a real (non-synthetic) live/replay event
|
||||
signature_hex: None,
|
||||
signer_pubkey_hex: None,
|
||||
};
|
||||
|
||||
let sensor = SensorDescriptor {
|
||||
modality: "wifi_csi".to_string(),
|
||||
vendor: "esp32".to_string(),
|
||||
device_id: snap.node_id.clone(),
|
||||
placement: "unknown".to_string(),
|
||||
clock_domain: "local".to_string(),
|
||||
};
|
||||
|
||||
let mut event = FieldEvent::new(
|
||||
event_id(snap),
|
||||
snap.timestamp_ns,
|
||||
sensor,
|
||||
tensor,
|
||||
observation,
|
||||
provenance,
|
||||
);
|
||||
|
||||
// ── 4. Sign (ed25519) so `is_fusable` passes for this real event ────────
|
||||
signer
|
||||
.sign_event(&mut event)
|
||||
.expect("ed25519 signing of a serializable event is infallible");
|
||||
|
||||
event
|
||||
}
|
||||
|
||||
/// Labels from the classification. These are descriptive (`person_present`,
|
||||
/// `motion_<level>`); the RuField fusion engine never reads labels
|
||||
/// (`event.rs:45-48`), so this carries no identity.
|
||||
fn build_labels(snap: &SensingSnapshot) -> Vec<String> {
|
||||
let mut labels = Vec::new();
|
||||
if snap.classification.presence {
|
||||
labels.push("person_present".to_string());
|
||||
}
|
||||
labels.push(format!("motion_{}", snap.classification.motion_level));
|
||||
labels
|
||||
}
|
||||
|
||||
/// Convenience: the privacy class that *would* be stamped for a snapshot,
|
||||
/// without building the whole event. Useful for egress badges (P3) and tests.
|
||||
#[must_use]
|
||||
pub fn snapshot_egress_class(snap: &SensingSnapshot) -> PrivacyClass {
|
||||
egress_class(snap.trust_class, snap.identity_bound, snap.demoted)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//! # wifi-densepose-rufield
|
||||
//!
|
||||
//! ADR-262 **anti-corruption bridge**: converts RuView's live WiFi-CSI sensing
|
||||
//! output into signed RuField [`FieldEvent`](rufield_core::FieldEvent)s.
|
||||
//!
|
||||
//! This crate is the **single coupling point** (ADR-262 §5.4) between RuView and
|
||||
//! the standalone RuField MFS spec (`vendor/rufield`, ADR-260). It depends on
|
||||
//! the four pure-Rust rufield crates **via path** — `rufield-core`,
|
||||
//! `-provenance`, `-privacy`, `-fusion` — and on **no** RuView internal crate.
|
||||
//! Inputs are owned primitives ([`SensingSnapshot`]) that mirror what RuView's
|
||||
//! sensing cycle produces, so the bridge never imports `SensingUpdate` /
|
||||
//! `TrustedOutput` directly.
|
||||
//!
|
||||
//! ## What P1 ships (honesty — ADR-262 §0 / §6)
|
||||
//!
|
||||
//! This is **P1 plumbing**: a tested `SensingSnapshot → FieldEvent` conversion
|
||||
//! plus the **fail-closed privacy mapping** that is the §3.3 correctness item.
|
||||
//! It is **not** wired into the live server (that is P3) and makes **no accuracy
|
||||
//! claim** — RuField v0.1 is synthetic end-to-end and RuView's single-link CSI
|
||||
//! carries its own caveats. The gates here are round-trip / fusability /
|
||||
//! privacy-safety / determinism, not validated F1.
|
||||
//!
|
||||
//! ## The critical correctness item: the privacy mapping (§3.3)
|
||||
//!
|
||||
//! RuView's `Derived` class has byte value `1` (below `Anonymous = 2`) yet
|
||||
//! carries an identity embedding. The bridge maps it to **P4/P5 by information
|
||||
//! content, never P1** — see [`map_privacy`]. Mapping off the byte would leak
|
||||
//! identity as low-privacy; [`map_privacy`] (and its dedicated test
|
||||
//! `derived_identity_never_maps_to_low_privacy`) exist specifically to prevent
|
||||
//! that.
|
||||
//!
|
||||
//! ## Example
|
||||
//!
|
||||
//! ```
|
||||
//! use wifi_densepose_rufield::{
|
||||
//! snapshot_to_field_event, SensingSnapshot, SensingFeatures, SensingClass,
|
||||
//! RuViewPrivacyClass,
|
||||
//! };
|
||||
//! use rufield_provenance::{Signer, is_fusable};
|
||||
//!
|
||||
//! let snap = SensingSnapshot {
|
||||
//! timestamp_ns: 1_791_986_400_000_000_000,
|
||||
//! features: SensingFeatures {
|
||||
//! mean_rssi: -55.0,
|
||||
//! variance: 0.4,
|
||||
//! motion_band_power: 2.0,
|
||||
//! breathing_band_power: 0.3,
|
||||
//! dominant_freq_hz: 0.25,
|
||||
//! change_points: 1,
|
||||
//! spectral_power: 3.0,
|
||||
//! },
|
||||
//! classification: SensingClass {
|
||||
//! motion_level: "low".into(),
|
||||
//! presence: true,
|
||||
//! confidence: 0.82,
|
||||
//! },
|
||||
//! signal_field: None,
|
||||
//! trust_class: RuViewPrivacyClass::Anonymous,
|
||||
//! demoted: false,
|
||||
//! identity_bound: false,
|
||||
//! node_id: "esp32_room_01".into(),
|
||||
//! };
|
||||
//!
|
||||
//! let signer = Signer::from_seed(b"adr-262-bridge-seed-32-bytes-ok!");
|
||||
//! let event = snapshot_to_field_event(&snap, &signer);
|
||||
//! assert!(is_fusable(&event)); // ed25519-signed, non-synthetic ⇒ fusable
|
||||
//! ```
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod bridge;
|
||||
pub mod privacy;
|
||||
pub mod snapshot;
|
||||
|
||||
pub use bridge::{snapshot_egress_class, snapshot_to_field_event};
|
||||
pub use privacy::{apply_demotion_floor, egress_class, map_privacy};
|
||||
pub use snapshot::{
|
||||
RuViewPrivacyClass, SensingClass, SensingFeatures, SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
// Re-export the rufield surface a bridge consumer needs, so callers depend on
|
||||
// one crate.
|
||||
pub use rufield_core::{Destination, FieldEvent, Modality, PrivacyClass, PrivacyDecision};
|
||||
pub use rufield_fusion::RuFieldFusion;
|
||||
pub use rufield_privacy::{DefaultPrivacyGuard, PrivacyPolicy};
|
||||
pub use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
|
||||
/// Whether a mapped [`PrivacyClass`] may be surfaced on a **network** egress
|
||||
/// (ADR-262 §4 P3 — the live `/api/field` / `/ws/field` surface must respect
|
||||
/// the same default §10 network policy `/ws/sensing` honours, never emitting
|
||||
/// above-policy data).
|
||||
///
|
||||
/// **Fail-closed for a live, unattended surface.** The live RuView surface has
|
||||
/// **no per-event consent or identity-binding ceremony** — so this is *stricter*
|
||||
/// than [`DefaultPrivacyGuard::authorize`]: it requires BOTH that the default
|
||||
/// guard would `Allow` the class onto [`Destination::Network`] with **no consent
|
||||
/// granted**, AND that the class is at or below the default network ceiling
|
||||
/// ([`PrivacyClass::P2`]). The second clause deliberately drops P4/P5 even
|
||||
/// though the guard's consent/identity *exceptions* would let an explicitly
|
||||
/// consented/identity-bound P4/P5 through — because the live surface cannot
|
||||
/// honestly assert that consent. Net effect: only **P1/P2** leave the box; P0
|
||||
/// (raw) and P3/P4/P5 are held edge-local.
|
||||
///
|
||||
/// This is the privacy-safety pin for the live surface: a `Derived` cycle maps
|
||||
/// to P4 (or P5 when identity-bound) via [`map_privacy`] and is therefore
|
||||
/// **never** surfaced as a network event — neither as a low-privacy P1 (the
|
||||
/// §3.3 mapping trap) nor at all.
|
||||
#[must_use]
|
||||
pub fn network_egress_allowed(class: PrivacyClass, identity_bound: bool) -> bool {
|
||||
use rufield_core::PrivacyGuard;
|
||||
let guard_allows = matches!(
|
||||
DefaultPrivacyGuard::default().authorize(
|
||||
class,
|
||||
Destination::Network,
|
||||
false, // no per-event consent on the live network surface (fail-closed)
|
||||
identity_bound,
|
||||
),
|
||||
PrivacyDecision::Allow
|
||||
);
|
||||
// Additionally cap at the default network ceiling: an unattended live
|
||||
// surface never asserts the P4-consent / P5-identity exception.
|
||||
guard_allows && class <= PrivacyClass::P2
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//! The ADR-262 §3.3 privacy mapping — the critical correctness item.
|
||||
//!
|
||||
//! RuView's effective `PrivacyClass` (4 byte-level classes) is the source of
|
||||
//! truth; the bridge maps it onto RuField's `PrivacyClass` (P0–P5) **at the
|
||||
//! egress boundary, by information content, NEVER by byte value**.
|
||||
//!
|
||||
//! ## The trap (ADR-262 §3, §6)
|
||||
//!
|
||||
//! RuView's `Derived` has byte value `1`, which sorts *below* `Anonymous`
|
||||
//! (byte `2`). A naive byte-mapping (`Derived = 1 → P1`) would leak
|
||||
//! identity-bearing features (`identity_embedding`, `identity_risk_score`) as a
|
||||
//! **low-privacy P1** event. Because `Derived` carries derived *identity*, it
|
||||
//! must map to the **biometric/identity tier (P4/P5)** — never P1. This is the
|
||||
//! single most dangerous mapping mistake; it gets a dedicated test
|
||||
//! (`derived_identity_never_maps_to_low_privacy`).
|
||||
//!
|
||||
//! ## Fail-closed
|
||||
//!
|
||||
//! [`RuViewPrivacyClass`] is a closed enum, so there is no runtime "unknown"
|
||||
//! value to receive — but the mapping is written `match`-exhaustively with an
|
||||
//! explicit, documented arm per class, and the `demoted`/`identity_bound`
|
||||
//! overlays only ever move the result **toward more privacy**, never less.
|
||||
|
||||
use crate::snapshot::RuViewPrivacyClass;
|
||||
use rufield_core::PrivacyClass;
|
||||
|
||||
/// Map a RuView effective `PrivacyClass` onto a RuField `PrivacyClass`
|
||||
/// (ADR-262 §3.3), by information content.
|
||||
///
|
||||
/// | RuView (byte) | → RuField | Rationale |
|
||||
/// |---|---|---|
|
||||
/// | `Raw` (0) | `P0` | raw CSI waveform |
|
||||
/// | `Derived` (1) | `P4` (or `P5` if `identity_bound`) | derived **identity** features ⇒ biometric/identity tier, **not** P1 |
|
||||
/// | `Anonymous` (2) | `P2` | occupancy / motion only |
|
||||
/// | `Restricted` (3) | `P2` (raw suppressed) | matches `suppress_raw_outputs` |
|
||||
///
|
||||
/// `identity_bound` only promotes `Derived` (already identity-derived) from P4
|
||||
/// to P5; it can never lower the class.
|
||||
#[must_use]
|
||||
pub fn map_privacy(ruview_class: RuViewPrivacyClass, identity_bound: bool) -> PrivacyClass {
|
||||
match ruview_class {
|
||||
// Raw CSI amplitude → raw waveform tier.
|
||||
RuViewPrivacyClass::Raw => PrivacyClass::P0,
|
||||
|
||||
// THE CRITICAL ARM (§3.3 / §6): `Derived` carries identity. Map by
|
||||
// information content to the biometric/identity tier P4, and to P5 when
|
||||
// the surface is bound to a named identity. NEVER P1.
|
||||
RuViewPrivacyClass::Derived => {
|
||||
if identity_bound {
|
||||
PrivacyClass::P5
|
||||
} else {
|
||||
PrivacyClass::P4
|
||||
}
|
||||
}
|
||||
|
||||
// Anonymous occupancy / motion aggregate → P2.
|
||||
RuViewPrivacyClass::Anonymous => PrivacyClass::P2,
|
||||
|
||||
// Restricted: occupancy with risk score / hash stripped and raw
|
||||
// suppressed. Capped at P2 (occupancy tier), matching
|
||||
// `EngineBridge::suppress_raw_outputs` (`engine_bridge.rs:240`).
|
||||
RuViewPrivacyClass::Restricted => PrivacyClass::P2,
|
||||
}
|
||||
}
|
||||
|
||||
/// The §4 P2 gate (b) monotonicity overlay: a governed-engine **demotion**
|
||||
/// (`TrustedOutput.demoted == true`) must never let the emitted class fall
|
||||
/// below P2 (occupancy floor), and raw is suppressed.
|
||||
///
|
||||
/// This is applied *after* [`map_privacy`] and can only raise the class
|
||||
/// (toward more privacy) — it is fail-closed by construction.
|
||||
#[must_use]
|
||||
pub fn apply_demotion_floor(class: PrivacyClass, demoted: bool) -> PrivacyClass {
|
||||
if demoted && class < PrivacyClass::P2 {
|
||||
PrivacyClass::P2
|
||||
} else {
|
||||
class
|
||||
}
|
||||
}
|
||||
|
||||
/// The full egress class for a snapshot: information-content mapping with the
|
||||
/// demotion floor overlaid. This is what the bridge stamps on the emitted
|
||||
/// `FieldEvent`.
|
||||
#[must_use]
|
||||
pub fn egress_class(
|
||||
ruview_class: RuViewPrivacyClass,
|
||||
identity_bound: bool,
|
||||
demoted: bool,
|
||||
) -> PrivacyClass {
|
||||
apply_demotion_floor(map_privacy(ruview_class, identity_bound), demoted)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn derived_maps_to_identity_tier_not_p1() {
|
||||
// The single most dangerous mapping mistake: Derived (byte 1) must NOT
|
||||
// become P1. It carries identity ⇒ P4, or P5 if identity-bound.
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, true), PrivacyClass::P5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_table_matches_adr_262_section_3_3() {
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Raw, false), PrivacyClass::P0);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Anonymous, false), PrivacyClass::P2);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Restricted, false), PrivacyClass::P2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapping_ignores_non_monotonic_byte_value() {
|
||||
// Derived's byte (1) is *below* Anonymous's byte (2), but Derived's
|
||||
// mapped class must be *above* Anonymous's mapped class — proving the
|
||||
// mapping uses information content, not the byte.
|
||||
assert!(RuViewPrivacyClass::Derived.raw_byte() < RuViewPrivacyClass::Anonymous.raw_byte());
|
||||
assert!(
|
||||
map_privacy(RuViewPrivacyClass::Derived, false)
|
||||
> map_privacy(RuViewPrivacyClass::Anonymous, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demotion_floor_only_raises_privacy() {
|
||||
// Raw → P0, but a demoted cycle floors to P2 with raw suppressed.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, true), PrivacyClass::P2);
|
||||
// Already-high classes are never lowered by the floor.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P5, true), PrivacyClass::P5);
|
||||
// No demotion ⇒ unchanged.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, false), PrivacyClass::P0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_bound_only_promotes() {
|
||||
// identity_bound never lowers privacy; it only promotes Derived P4→P5.
|
||||
for c in [
|
||||
RuViewPrivacyClass::Raw,
|
||||
RuViewPrivacyClass::Derived,
|
||||
RuViewPrivacyClass::Anonymous,
|
||||
RuViewPrivacyClass::Restricted,
|
||||
] {
|
||||
assert!(map_privacy(c, true) >= map_privacy(c, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Owned, primitive input types for the ADR-262 bridge.
|
||||
//!
|
||||
//! These deliberately **mirror** the shapes RuView's sensing cycle produces
|
||||
//! (the `/ws/sensing` `SensingUpdate` build site at
|
||||
//! `wifi-densepose-sensing-server/src/main.rs:~5938` and the `TrustedOutput`
|
||||
//! trust state surfaced via `EngineBridge` at `main.rs:~5886`) **without
|
||||
//! importing** RuView's internal crates. Keeping the bridge an anti-corruption
|
||||
//! layer (ADR-262 §5.4) means it takes owned primitives, not `SensingUpdate`
|
||||
//! or `TrustedOutput` directly — so this crate never depends on
|
||||
//! `wifi-densepose-sensing-server`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The CSI feature scalars RuView publishes on every `/ws/sensing` cycle.
|
||||
///
|
||||
/// Mirrors `FeatureInfo` (`main.rs:368-377`). All values are in RuView's own
|
||||
/// units; the bridge normalizes them into `Observation.features` for fusion.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingFeatures {
|
||||
/// Mean RSSI across the CSI window (dBm).
|
||||
pub mean_rssi: f64,
|
||||
/// CSI amplitude variance.
|
||||
pub variance: f64,
|
||||
/// Motion-band spectral power (drives `motion_energy`).
|
||||
pub motion_band_power: f64,
|
||||
/// Breathing-band spectral power (drives `breathing_band`).
|
||||
pub breathing_band_power: f64,
|
||||
/// Dominant frequency of the CSI window (Hz).
|
||||
pub dominant_freq_hz: f64,
|
||||
/// Number of change points detected in the window (drives `transient`).
|
||||
pub change_points: usize,
|
||||
/// Total spectral power of the window.
|
||||
pub spectral_power: f64,
|
||||
}
|
||||
|
||||
/// The RuView classification block. Mirrors `ClassificationInfo`
|
||||
/// (`main.rs:379-384`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingClass {
|
||||
/// Coarse motion level label (e.g. `"none"`, `"low"`, `"high"`).
|
||||
pub motion_level: String,
|
||||
/// Whether a person is present.
|
||||
pub presence: bool,
|
||||
/// Classification confidence `0.0..=1.0`.
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
/// A RuView signal field — a floor-plane grid of field values. Mirrors
|
||||
/// `SignalField` (`main.rs:386-390`). The bridge derives a real position from
|
||||
/// the strongest field peak (like `field_localize`) and **never fabricates**
|
||||
/// coordinates when this is absent.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SignalField {
|
||||
/// Grid dimensions `[x, y, z]`.
|
||||
pub grid_size: [usize; 3],
|
||||
/// Row-major flattened field values; `len() == grid_size.product()`.
|
||||
pub values: Vec<f64>,
|
||||
}
|
||||
|
||||
impl SignalField {
|
||||
/// Index `[x, y, z]` of the strongest field cell, or `None` if the grid is
|
||||
/// empty / all-NaN. This is the honest "strongest field peak" readout that
|
||||
/// `field_localize` (`field_localize.rs:16-27`) exposes — **not** calibrated
|
||||
/// triangulation.
|
||||
#[must_use]
|
||||
pub fn peak_cell(&self) -> Option<[i32; 3]> {
|
||||
let [nx, ny, nz] = self.grid_size;
|
||||
if nx == 0 || ny == 0 || nz == 0 || self.values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut best_idx: Option<usize> = None;
|
||||
let mut best_val = f64::NEG_INFINITY;
|
||||
for (i, &v) in self.values.iter().enumerate() {
|
||||
if v.is_finite() && v > best_val {
|
||||
best_val = v;
|
||||
best_idx = Some(i);
|
||||
}
|
||||
}
|
||||
let idx = best_idx?;
|
||||
// Row-major: idx = ((x * ny) + y) * nz + z.
|
||||
let z = idx % nz;
|
||||
let y = (idx / nz) % ny;
|
||||
let x = idx / (nz * ny);
|
||||
Some([x as i32, y as i32, z as i32])
|
||||
}
|
||||
}
|
||||
|
||||
/// RuView's effective privacy class (the `effective_class` / privacy byte on
|
||||
/// `TrustedOutput`).
|
||||
///
|
||||
/// This **mirrors** `wifi_densepose_bfld::PrivacyClass` (`bfld/lib.rs:103-116`,
|
||||
/// `#[repr(u8)]`) — the four byte-level classes. The byte values are
|
||||
/// **deliberately non-monotonic in information content**: `Derived = 1` carries
|
||||
/// an identity embedding yet sorts *below* `Anonymous = 2`. The bridge's
|
||||
/// `map_privacy` must therefore map by information content, NEVER by byte value
|
||||
/// (ADR-262 §3.3 — the central correctness item).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RuViewPrivacyClass {
|
||||
/// Byte `0` — raw CSI amplitude, local-only.
|
||||
Raw,
|
||||
/// Byte `1` — derived **identity** features (identity_embedding +
|
||||
/// identity_risk_score), LAN-only. The dangerous one (§3.3).
|
||||
Derived,
|
||||
/// Byte `2` — aggregate occupancy / motion, no identity.
|
||||
Anonymous,
|
||||
/// Byte `3` — care/regulated: occupancy minus risk score and hash;
|
||||
/// raw suppressed.
|
||||
Restricted,
|
||||
}
|
||||
|
||||
impl RuViewPrivacyClass {
|
||||
/// The raw byte value used by RuView's `#[repr(u8)]` enum
|
||||
/// (`bfld/lib.rs:103`). Exposed only so callers can demonstrate the
|
||||
/// non-monotonicity trap in tests; the bridge never maps off this byte.
|
||||
#[must_use]
|
||||
pub fn raw_byte(self) -> u8 {
|
||||
match self {
|
||||
RuViewPrivacyClass::Raw => 0,
|
||||
RuViewPrivacyClass::Derived => 1,
|
||||
RuViewPrivacyClass::Anonymous => 2,
|
||||
RuViewPrivacyClass::Restricted => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One sensing cycle, as a bridge input. Mirrors the join of `SensingUpdate`
|
||||
/// (features + classification + signal_field) and the `TrustedOutput` trust
|
||||
/// state (`trust_class`) that ADR-262 §1.2 / P1 say must be done at the bridge.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingSnapshot {
|
||||
/// Capture time, nanoseconds since Unix epoch (the real `SensingUpdate`
|
||||
/// timestamp, ns).
|
||||
pub timestamp_ns: u64,
|
||||
/// CSI feature scalars (`/ws/sensing` feature set).
|
||||
pub features: SensingFeatures,
|
||||
/// Classification (motion level / presence / confidence).
|
||||
pub classification: SensingClass,
|
||||
/// Optional signal field for a real position readout.
|
||||
pub signal_field: Option<SignalField>,
|
||||
/// RuView's effective privacy class (the source-of-truth, §3.3).
|
||||
pub trust_class: RuViewPrivacyClass,
|
||||
/// Whether the governed engine demoted this cycle (`TrustedOutput.demoted`).
|
||||
/// When `true` the emitted event must be `>= P2` and raw suppressed
|
||||
/// (§3.3 / §4 P2 gate (b)).
|
||||
pub demoted: bool,
|
||||
/// Whether this cycle's identity surface is bound to an enrolled identity
|
||||
/// (RuView's `identity_bound`). Promotes `Derived` to P5 when set.
|
||||
pub identity_bound: bool,
|
||||
/// Stable node id (e.g. `"esp32_room_01"`).
|
||||
pub node_id: String,
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! ADR-262 P1 acceptance gates. Each test below IS an acceptance criterion.
|
||||
//!
|
||||
//! - round-trip: snapshot → FieldEvent → serde → equal
|
||||
//! - is_fusable: emitted event passes the §11 fusability invariant
|
||||
//! - fusion ingest accept: `RuFieldFusion::ingest` accepts it + `infer` runs
|
||||
//! - privacy safety: `Derived` never maps to a low-privacy class (the §3.3 trap)
|
||||
//! - determinism: same snapshot + same signer seed → identical event
|
||||
|
||||
use rufield_core::{FusionEngine, InferenceQuery, PrivacyClass};
|
||||
use rufield_fusion::RuFieldFusion;
|
||||
use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
use wifi_densepose_rufield::{
|
||||
map_privacy, snapshot_to_field_event, RuViewPrivacyClass, SensingClass, SensingFeatures,
|
||||
SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
const SEED: &[u8; 32] = b"adr-262-bridge-seed-32-bytes-ok!";
|
||||
|
||||
fn signer() -> Signer {
|
||||
Signer::from_seed(SEED)
|
||||
}
|
||||
|
||||
/// A representative snapshot with a real signal field (so a position is derived).
|
||||
fn sample_snapshot() -> SensingSnapshot {
|
||||
SensingSnapshot {
|
||||
timestamp_ns: 1_791_986_400_123_456_789,
|
||||
features: SensingFeatures {
|
||||
mean_rssi: -52.5,
|
||||
variance: 0.73,
|
||||
motion_band_power: 2.4,
|
||||
breathing_band_power: 0.6,
|
||||
dominant_freq_hz: 0.27,
|
||||
change_points: 2,
|
||||
spectral_power: 4.1,
|
||||
},
|
||||
classification: SensingClass {
|
||||
motion_level: "high".into(),
|
||||
presence: true,
|
||||
confidence: 0.88,
|
||||
},
|
||||
signal_field: Some(SignalField {
|
||||
grid_size: [2, 1, 2],
|
||||
// peak at flat index 2 → cell [1,0,0]
|
||||
values: vec![0.1, 0.2, 0.9, 0.3],
|
||||
}),
|
||||
trust_class: RuViewPrivacyClass::Anonymous,
|
||||
demoted: false,
|
||||
identity_bound: false,
|
||||
node_id: "esp32_room_01".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_round_trip_serde_equal() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let json = serde_json::to_string(&ev).expect("serialize");
|
||||
let back: rufield_core::FieldEvent = serde_json::from_str(&json).expect("deserialize");
|
||||
assert_eq!(ev, back, "FieldEvent must round-trip through serde unchanged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_is_fusable_verified_receipt() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// Real (non-synthetic) event must carry a verifying ed25519 signature.
|
||||
assert!(!ev.provenance.synthetic, "live event must NOT be marked synthetic");
|
||||
assert!(ev.provenance.signature_hex.is_some(), "must be signed");
|
||||
assert!(verify_event(&ev).is_ok(), "signature must verify");
|
||||
assert!(is_fusable(&ev), "verified receipt ⇒ fusable (§11 invariant)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_fusion_ingest_accepts_and_infers() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let mut engine = RuFieldFusion::new();
|
||||
engine.ingest(ev).expect("fusion engine must accept the signed event");
|
||||
// infer() must run without error (may or may not produce inferences).
|
||||
let inferences = engine
|
||||
.infer(&InferenceQuery::all())
|
||||
.expect("infer() must run");
|
||||
// The graph recorded the event/sensor provenance nodes.
|
||||
assert!(
|
||||
engine.graph().node_count() >= 2,
|
||||
"ingest should record sensor + event nodes"
|
||||
);
|
||||
let _ = inferences; // count is not an accuracy claim
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_privacy_safety_derived_never_maps_to_low_privacy() {
|
||||
// THE critical §3.3 gate. Derived carries identity ⇒ P4/P5, NEVER P1.
|
||||
let p4 = map_privacy(RuViewPrivacyClass::Derived, false);
|
||||
let p5 = map_privacy(RuViewPrivacyClass::Derived, true);
|
||||
assert_eq!(p4, PrivacyClass::P4);
|
||||
assert_eq!(p5, PrivacyClass::P5);
|
||||
assert!(p4 >= PrivacyClass::P4, "Derived must be in the identity tier");
|
||||
assert_ne!(p4, PrivacyClass::P1, "Derived must NEVER be P1");
|
||||
|
||||
// And end-to-end: an emitted event from a Derived snapshot must be P4/P5.
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Derived;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P4,
|
||||
"emitted Derived event must be P4 or P5, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
assert_eq!(ev.observation.privacy_class, ev.tensor.privacy_class);
|
||||
}
|
||||
|
||||
/// Full §3.3 table over every RuView class → expected RuField class.
|
||||
#[test]
|
||||
fn gate_privacy_table_over_every_ruview_class() {
|
||||
let cases = [
|
||||
(RuViewPrivacyClass::Raw, false, PrivacyClass::P0),
|
||||
(RuViewPrivacyClass::Derived, false, PrivacyClass::P4),
|
||||
(RuViewPrivacyClass::Derived, true, PrivacyClass::P5),
|
||||
(RuViewPrivacyClass::Anonymous, false, PrivacyClass::P2),
|
||||
(RuViewPrivacyClass::Restricted, false, PrivacyClass::P2),
|
||||
];
|
||||
for (ruview, id_bound, expected) in cases {
|
||||
assert_eq!(
|
||||
map_privacy(ruview, id_bound),
|
||||
expected,
|
||||
"{ruview:?} (identity_bound={id_bound}) must map to {expected:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fail-closed: a demoted Raw snapshot must NOT emit P0 (raw) — it floors to P2.
|
||||
#[test]
|
||||
fn gate_demotion_is_fail_closed() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Raw; // would be P0
|
||||
snap.demoted = true; // governed engine demotion
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P2,
|
||||
"demoted cycle must floor to >= P2, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_determinism_same_seed_identical_event() {
|
||||
let snap = sample_snapshot();
|
||||
let a = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
let b = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
assert_eq!(a, b, "same snapshot + same signer seed ⇒ identical event");
|
||||
// Including the signature (ed25519 is deterministic).
|
||||
assert_eq!(a.provenance.signature_hex, b.provenance.signature_hex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fabricated_position_when_field_absent() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.signal_field = None;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(ev.observation.range_m.is_none(), "no field ⇒ no fabricated range");
|
||||
assert!(ev.observation.space_cell.is_none(), "no field ⇒ no fabricated cell");
|
||||
assert!(
|
||||
ev.observation.motion_vector.is_none(),
|
||||
"no field ⇒ no fabricated motion vector"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_real_position_from_field_peak() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// peak at flat index 2, grid [2,1,2] (row-major) → cell [1,0,0]
|
||||
assert_eq!(ev.observation.space_cell, Some([1, 0, 0]));
|
||||
assert_eq!(ev.observation.range_m, Some(1.0));
|
||||
}
|
||||
@@ -47,3 +47,7 @@ harness = false
|
||||
[[bench]]
|
||||
name = "fusion_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "ann_bench"
|
||||
harness = false
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Criterion bench for the ADR-261 graph-ANN index: linear scan vs float HNSW
|
||||
//! vs quantized HNSW, on the shared `ann_measure` fixture.
|
||||
//!
|
||||
//! The authoritative recall/QPS numbers in ADR-261 come from the
|
||||
//! `--no-default-features --release` test report
|
||||
//! (`ann_bench_report` in `src/ann_measure.rs`), which is deterministic and
|
||||
//! gate-runnable. This criterion bench times the same operations through the
|
||||
//! criterion harness for stable per-op medians:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
//! ```
|
||||
//!
|
||||
//! Build is excluded from the timed region (done once in setup); only the query
|
||||
//! path is measured. The fixture and both indices are identical to the report's,
|
||||
//! so the bench and the report can never measure different graphs.
|
||||
|
||||
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||
use wifi_densepose_ruvector::ann_measure::{
|
||||
build_indices, build_quant_bits, queries, AnnBenchParams,
|
||||
};
|
||||
|
||||
fn bench_ann(c: &mut Criterion) {
|
||||
// Modest N so the bench builds quickly; the report covers the larger N.
|
||||
let p = AnnBenchParams::default_fixture(10_000);
|
||||
let (float_idx, quant_idx, vectors) = build_indices(p);
|
||||
// Multi-bit quant variants over the SAME graph/fixture (ADR-261 §11).
|
||||
let quant_2bit = build_quant_bits(p, &vectors, 2);
|
||||
let quant_4bit = build_quant_bits(p, &vectors, 4);
|
||||
let qs = queries(p);
|
||||
let k = p.k;
|
||||
|
||||
let mut group = c.benchmark_group("ann_query");
|
||||
group.sample_size(20);
|
||||
|
||||
// Linear scan (brute force) — the no-index baseline.
|
||||
group.bench_function("linear_scan", |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.brute_force(black_box(q), k).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
|
||||
// Float HNSW at a mid beam width.
|
||||
for &ef in &[64usize, 128] {
|
||||
group.bench_function(format!("float_hnsw_ef{ef}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.search(black_box(q), k, ef).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Quantized HNSW (1-bit) at matched beam widths + rerank.
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_1bit_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(quant_idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Multi-bit quant HNSW (ADR-261 §11): 2-bit and 4-bit traversal codes at a
|
||||
// mid beam width, so the criterion medians show the per-bit QPS cost the
|
||||
// scaling study reports against recall.
|
||||
for (label, idx) in [("2bit", &quant_2bit), ("4bit", &quant_4bit)] {
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_{label}_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_ann);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,684 @@
|
||||
//! Deterministic, `--no-default-features`-runnable **ANN benchmark measurement**
|
||||
//! for ADR-261 — the single source of truth for the QPS/recall numbers the ADR
|
||||
//! quotes for **linear scan**, **float HNSW**, and **quantized HNSW**.
|
||||
//!
|
||||
//! Both the criterion bench (`benches/ann_bench.rs`) and the in-crate report test
|
||||
//! ([`tests::ann_bench_report`]) call into here, so they can never silently
|
||||
//! measure different things. The numbers in ADR-261 §6 come from running:
|
||||
//!
|
||||
//! ```text
|
||||
//! cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
//! ann_bench_report -- --nocapture
|
||||
//! ```
|
||||
//!
|
||||
//! # What is measured, and the honesty contract
|
||||
//!
|
||||
//! On one fixed planted-cluster fixture (documented dim/N/K/seed), for each
|
||||
//! method we measure:
|
||||
//! - **recall@10** vs the brute-force exact top-10 (the ground truth),
|
||||
//! - **QPS** = queries / total wall-clock query time (warm; build excluded),
|
||||
//! at matched recall operating points found by sweeping `ef` (HNSW) and
|
||||
//! `(ef, rerank)` (quantized).
|
||||
//!
|
||||
//! The reported **ratio** is the claim, not the absolute QPS (which is
|
||||
//! machine-specific). We do **not** tune the quantized path to manufacture a
|
||||
//! win: if at our scale quantized does not beat float HNSW, the report says so
|
||||
//! and the ADR records the honest negative + the expected larger-N crossover.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::hnsw::{HnswIndex, HnswParams, Metric};
|
||||
use crate::hnsw_quantized::QuantizedHnswIndex;
|
||||
|
||||
/// SplitMix64 — the crate-wide deterministic PRNG (mirrors `coverage.rs`).
|
||||
#[inline]
|
||||
fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
#[inline]
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
((split_mix64(state) >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
#[inline]
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// ANN benchmark fixture parameters, documented in the ADR-261 report.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AnnBenchParams {
|
||||
/// Embedding dimension.
|
||||
pub dim: usize,
|
||||
/// Number of indexed vectors (N).
|
||||
pub n: usize,
|
||||
/// Number of planted clusters (near-neighbour structure).
|
||||
pub clusters: usize,
|
||||
/// Number of queries timed.
|
||||
pub n_queries: usize,
|
||||
/// Top-K.
|
||||
pub k: usize,
|
||||
/// Intra-cluster Gaussian jitter.
|
||||
pub noise: f32,
|
||||
/// Master fixture seed.
|
||||
pub seed: u64,
|
||||
/// Graph construction/level seed.
|
||||
pub graph_seed: u64,
|
||||
/// Rotation seed for the quantized 1-bit codes.
|
||||
pub rot_seed: u64,
|
||||
}
|
||||
|
||||
impl AnnBenchParams {
|
||||
/// The default ADR-261 fixture: AETHER-shape 128-d, planted clusters.
|
||||
pub fn default_fixture(n: usize) -> Self {
|
||||
Self {
|
||||
dim: 128,
|
||||
n,
|
||||
clusters: 64,
|
||||
n_queries: 200,
|
||||
k: 10,
|
||||
noise: 0.35,
|
||||
seed: 0xADADADAD_0000_0261,
|
||||
graph_seed: 0x6261_5247_4148_4E53,
|
||||
rot_seed: 0x5EED_C0DE_1234_5678,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The fixture vectors for `p` (deterministic planted clusters).
|
||||
pub fn fixture(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n)
|
||||
.map(|i| {
|
||||
let c = i % p.clusters;
|
||||
let mut s = p.seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The timed query set for `p` (drawn from the same clusters, disjoint seed).
|
||||
pub fn queries(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n_queries)
|
||||
.map(|q| {
|
||||
let c = q % p.clusters;
|
||||
let mut s = p.seed ^ 0xDEAD_0000_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-method measurement: recall@K and QPS.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct MethodResult {
|
||||
/// Mean recall@K vs brute-force ground truth.
|
||||
pub recall: f64,
|
||||
/// Queries per second (warm wall-clock).
|
||||
pub qps: f64,
|
||||
/// Mean query latency in microseconds.
|
||||
pub latency_us: f64,
|
||||
}
|
||||
|
||||
/// Ground-truth brute-force top-K id sets for every query (computed once).
|
||||
/// Public so the criterion bench and the report test share one definition.
|
||||
pub fn ground_truth(idx: &HnswIndex, queries: &[Vec<f32>], k: usize) -> Vec<HashSet<u32>> {
|
||||
queries
|
||||
.iter()
|
||||
.map(|q| idx.brute_force(q, k).into_iter().map(|(id, _)| id).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Measure **linear scan** (brute force): recall is 1.0 by definition; QPS is the
|
||||
/// timed exact scan. This is the no-index baseline.
|
||||
pub fn measure_linear(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.brute_force(q, k);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **float HNSW** at a given beam width `ef`.
|
||||
pub fn measure_float_hnsw(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.search(q, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **quantized HNSW** at a given `(ef, rerank)`.
|
||||
pub fn measure_quantized_hnsw(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
rerank: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = qidx.search_quantized(q, k, ef, rerank);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build both indices for `p` (shared insertion order + graph seed so the float
|
||||
/// and quantized graphs are identical — the only variable is scoring). The
|
||||
/// quantized index uses the legacy **1-bit** code (ADR-261 §6); use
|
||||
/// [`build_indices_bits`] for the multi-bit scaling study (§11).
|
||||
pub fn build_indices(p: AnnBenchParams) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
build_indices_bits(p, 1)
|
||||
}
|
||||
|
||||
/// Build the float HNSW + a `bits`-bit quantized HNSW over the same fixture,
|
||||
/// sharing the graph seed and insertion order so the *only* variable between the
|
||||
/// float and quantized search is the traversal score. `bits ∈ {1, 2, 4}` (clamped
|
||||
/// in [`QuantizedHnswIndex::build_bits`]). The float index is **independent of
|
||||
/// `bits`** — callers sweeping `bits` should build the float index once and reuse
|
||||
/// it (the quantized graph is identical across `bits`; only the per-node code
|
||||
/// changes).
|
||||
pub fn build_indices_bits(
|
||||
p: AnnBenchParams,
|
||||
bits: u32,
|
||||
) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
let vectors = fixture(p);
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
let mut float_idx = HnswIndex::new(p.dim, Metric::L2, params);
|
||||
for v in &vectors {
|
||||
float_idx.insert(v);
|
||||
}
|
||||
let quant_idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
p.dim,
|
||||
Metric::L2,
|
||||
params,
|
||||
p.rot_seed,
|
||||
bits,
|
||||
p.k * 4,
|
||||
);
|
||||
(float_idx, quant_idx, vectors)
|
||||
}
|
||||
|
||||
/// Build only the `bits`-bit quantized index for `p`, reusing a fixture the
|
||||
/// caller already has (avoids regenerating `N×dim` floats per bit-depth in the
|
||||
/// scaling sweep). The graph seed/insertion order match [`build_indices_bits`],
|
||||
/// so this quantized graph is identical to that one's at the same `p`.
|
||||
pub fn build_quant_bits(p: AnnBenchParams, vectors: &[Vec<f32>], bits: u32) -> QuantizedHnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
QuantizedHnswIndex::build_bits(vectors, p.dim, Metric::L2, params, p.rot_seed, bits, p.k * 4)
|
||||
}
|
||||
|
||||
/// The fastest operating point of a method that meets `target` recall, as
|
||||
/// `(qps, recall, label)`; `None` if no swept op met it.
|
||||
type BestOp = Option<(f64, f64, String)>;
|
||||
|
||||
/// Sweep float HNSW over a fixed `ef` ladder; return the fastest op meeting
|
||||
/// `target` recall.
|
||||
pub fn best_float_op(
|
||||
idx: &HnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> BestOp {
|
||||
let mut best: BestOp = None;
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(idx, qs, truth, k, ef);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef}")));
|
||||
}
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Sweep quant HNSW over a fixed `(ef, rerank)` ladder; return the fastest op
|
||||
/// meeting `target` recall, plus the best recall reached anywhere on the ladder
|
||||
/// (so a not-found verdict can report how close it got).
|
||||
pub fn best_quant_op(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> (BestOp, f64) {
|
||||
let mut best: BestOp = None;
|
||||
let mut best_recall_seen = 0.0f64;
|
||||
for &ef in &[32usize, 64, 128, 256, 512] {
|
||||
for &rr in &[k * 2, k * 5, k * 10, k * 20] {
|
||||
let r = measure_quantized_hnsw(qidx, qs, truth, k, ef, rr);
|
||||
best_recall_seen = best_recall_seen.max(r.recall);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef} rr={rr}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
(best, best_recall_seen)
|
||||
}
|
||||
|
||||
/// One row of the ADR-261 §11 scaling study: at a fixed `(N, b)`, the equal-recall
|
||||
/// (≥ `target`) operating points for float vs quant HNSW and their QPS ratio.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScalingRow {
|
||||
/// Indexed vector count.
|
||||
pub n: usize,
|
||||
/// Traversal-code bit-depth (1, 2, or 4).
|
||||
pub bits: u32,
|
||||
/// Packed bytes per node of the quant code at this `b`.
|
||||
pub bytes_per_node: usize,
|
||||
/// Fastest float-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub float_op: BestOp,
|
||||
/// Fastest quant-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub quant_op: BestOp,
|
||||
/// Best recall the quant ladder reached at this `(N, b)` (≤ `target` ⇒ no op).
|
||||
pub quant_best_recall: f64,
|
||||
/// quant/float QPS ratio at equal recall, if both met `target`.
|
||||
pub ratio: Option<f64>,
|
||||
}
|
||||
|
||||
/// Run the ADR-261 §11 multi-bit scaling study: for each `N ∈ ns` and each
|
||||
/// `b ∈ bits_set`, measure the equal-recall (≥ `target`) QPS ratio of quant-HNSW
|
||||
/// vs float-HNSW on the shared fixture. Deterministic and `--no-default-features`
|
||||
/// runnable. Returns one [`ScalingRow`] per `(N, b)`; the caller prints the table
|
||||
/// and decides the crossover verdict. The float index is built once per `N` and
|
||||
/// reused across `b` (the quant graph is identical across `b`).
|
||||
pub fn run_scaling_study(
|
||||
base: AnnBenchParams,
|
||||
ns: &[usize],
|
||||
bits_set: &[u32],
|
||||
target: f64,
|
||||
) -> Vec<ScalingRow> {
|
||||
let mut rows = Vec::new();
|
||||
for &n in ns {
|
||||
let p = AnnBenchParams { n, ..base };
|
||||
let (float_idx, _q1, vectors) = build_indices_bits(p, 1);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let float_op = best_float_op(&float_idx, &qs, &truth, p.k, target);
|
||||
for &b in bits_set {
|
||||
let qidx = build_quant_bits(p, &vectors, b);
|
||||
let (quant_op, quant_best_recall) =
|
||||
best_quant_op(&qidx, &qs, &truth, p.k, target);
|
||||
let ratio = match (&float_op, &quant_op) {
|
||||
(Some((fqps, _, _)), Some((qqps, _, _))) => Some(qqps / fqps),
|
||||
_ => None,
|
||||
};
|
||||
rows.push(ScalingRow {
|
||||
n,
|
||||
bits: qidx.bits(),
|
||||
bytes_per_node: qidx.bytes_per_node(),
|
||||
float_op: float_op.clone(),
|
||||
quant_op,
|
||||
quant_best_recall,
|
||||
ratio,
|
||||
});
|
||||
}
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fixture_and_queries_are_deterministic() {
|
||||
let p = AnnBenchParams::default_fixture(500);
|
||||
assert_eq!(fixture(p), fixture(p));
|
||||
assert_eq!(queries(p), queries(p));
|
||||
let p2 = AnnBenchParams {
|
||||
seed: p.seed ^ 1,
|
||||
..p
|
||||
};
|
||||
assert_ne!(fixture(p)[0], fixture(p2)[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linear_recall_is_one() {
|
||||
// Linear scan IS the ground truth, so recall must be exactly 1.0.
|
||||
let p = AnnBenchParams::default_fixture(800);
|
||||
let (float_idx, _q, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let r = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
assert!((r.recall - 1.0).abs() < 1e-9, "linear recall {} != 1.0", r.recall);
|
||||
assert!(r.qps > 0.0);
|
||||
}
|
||||
|
||||
/// The ADR-261 measurement report. Prints the linear / float-HNSW /
|
||||
/// quantized-HNSW recall@10 + QPS table and the QPS ratios at matched recall.
|
||||
/// Run with `--release --nocapture` for the numbers the ADR quotes.
|
||||
#[test]
|
||||
fn ann_bench_report() {
|
||||
// N here is the small/CI-friendly default so the standard (debug) test
|
||||
// gate stays fast; the ADR's headline numbers are taken at the larger N
|
||||
// under --release (documented in the ADR with the exact command). This
|
||||
// test asserts only structural invariants so it is gate-safe at any N.
|
||||
let n: usize = std::env::var("ANN_BENCH_N")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(10_000);
|
||||
let p = AnnBenchParams::default_fixture(n);
|
||||
let (float_idx, quant_idx, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
|
||||
println!("\n=== ADR-261 ANN benchmark (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} N={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
p.dim, p.n, p.clusters, p.n_queries, p.k, p.noise, p.graph_seed, p.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 (debug build unless --release)");
|
||||
println!(
|
||||
"{:<28} {:>9} {:>12} {:>12}",
|
||||
"method", "recall@10", "QPS", "lat(us)"
|
||||
);
|
||||
|
||||
let lin = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
"linear scan (brute)", lin.recall, lin.qps, lin.latency_us
|
||||
);
|
||||
|
||||
// Float HNSW across an ef sweep.
|
||||
let mut float_ops: Vec<(usize, MethodResult)> = Vec::new();
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(&float_idx, &qs, &truth, p.k, ef);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("float-HNSW ef={ef}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
float_ops.push((ef, r));
|
||||
}
|
||||
|
||||
// Quantized HNSW across (ef, rerank) sweep.
|
||||
let mut quant_ops: Vec<((usize, usize), MethodResult)> = Vec::new();
|
||||
for &ef in &[32usize, 64, 128, 256] {
|
||||
for &rr in &[p.k * 2, p.k * 5, p.k * 10] {
|
||||
let r = measure_quantized_hnsw(&quant_idx, &qs, &truth, p.k, ef, rr);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("quant-HNSW ef={ef} rr={rr}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
quant_ops.push(((ef, rr), r));
|
||||
}
|
||||
}
|
||||
|
||||
// Equal-recall comparison: pick, for a target recall, the FASTEST op of
|
||||
// each method that meets it, then report the QPS ratios.
|
||||
println!("\n--- equal-recall QPS ratios ---");
|
||||
for &target in &[0.90f64, 0.95, 0.99] {
|
||||
let best_float = float_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
let best_quant = quant_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
match (best_float, best_quant) {
|
||||
(Some((fef, fr)), Some(((qef, qrr), qr))) => {
|
||||
let ratio = qr.qps / fr.qps;
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant ef={} rr={} {:.0} QPS | quant/float={:.2}x | float/linear={:.2}x",
|
||||
target, fef, fr.qps, qef, qrr, qr.qps, ratio, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
(Some((fef, fr)), None) => {
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant: NO op met this recall | float/linear={:.2}x",
|
||||
target, fef, fr.qps, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
println!("recall>={:.2}: neither method met this recall at the swept ops", target);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("=========================================================\n");
|
||||
|
||||
// Structural assertions (gate-safe, any N):
|
||||
// - linear scan is exact,
|
||||
// - the best float-HNSW op clears the correctness gate,
|
||||
// - quantized's best op is at least useful (recall well above random).
|
||||
assert!((lin.recall - 1.0).abs() < 1e-9);
|
||||
let best_float_recall = float_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
assert!(
|
||||
best_float_recall >= 0.95,
|
||||
"best float-HNSW recall {best_float_recall:.4} below 0.95 gate"
|
||||
);
|
||||
let best_quant_recall = quant_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
// Honest floor: the 1-bit Hamming traversal is a COARSE angle proxy, so
|
||||
// at large N its best recall lands well below the float gate (MEASURED
|
||||
// ~0.74 at N=10k — see ADR-261 §6). We assert only that it is clearly
|
||||
// useful (>> random: random top-10 of N=10k is ~0.001), which catches a
|
||||
// fully-broken traversal/rerank without pretending the quantized variant
|
||||
// matches float HNSW. The honest negative IS the result.
|
||||
assert!(
|
||||
best_quant_recall >= 0.30,
|
||||
"best quant-HNSW recall {best_quant_recall:.4} below the 0.30 not-broken floor"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ADR-261 §11 **multi-bit scaling study**. Sweeps `N` and `b ∈ {1,2,4}`,
|
||||
/// printing the `(N, b) → recall / QPS / quant-vs-float ratio at equal recall`
|
||||
/// surface and the crossover verdict. This is the source of truth for the §11
|
||||
/// table. Run for the published numbers with:
|
||||
///
|
||||
/// ```text
|
||||
/// cd v2 && ANN_SCALE_NS=10000,100000,250000 \
|
||||
/// cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
/// scaling_report -- --nocapture --ignored
|
||||
/// ```
|
||||
///
|
||||
/// Marked `#[ignore]` so the default (debug) gate stays fast: it builds and
|
||||
/// queries several indices up to large `N`, which is minutes under `--release`
|
||||
/// and far too slow in debug. The CI-safe structural invariants are checked by
|
||||
/// `scaling_study_small_is_consistent` below at tiny `N`.
|
||||
#[test]
|
||||
#[ignore = "scaling study — run explicitly with --release --ignored; minutes at large N"]
|
||||
fn scaling_report() {
|
||||
// N ladder: default 10k→100k→250k (a clean 25× span that builds+queries in
|
||||
// a few minutes under --release on the test box). Override with
|
||||
// ANN_SCALE_NS=a,b,c. The largest feasible N is documented in the ADR with
|
||||
// the measured build/query time at the cap.
|
||||
let ns: Vec<usize> = std::env::var("ANN_SCALE_NS")
|
||||
.ok()
|
||||
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
|
||||
.unwrap_or_else(|| vec![10_000, 100_000, 250_000]);
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let target = 0.90f64;
|
||||
let base = AnnBenchParams::default_fixture(ns[0]);
|
||||
|
||||
println!("\n=== ADR-261 §11 multi-bit scaling study (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
base.dim, base.clusters, base.n_queries, base.k, base.noise, base.graph_seed, base.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 target recall >= {target:.2} (use --release for QPS)");
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10} {:>22} {:>22} {:>12}",
|
||||
"N", "bits", "B/node", "q_best_rec", "float@target", "quant@target", "quant/float"
|
||||
);
|
||||
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, target);
|
||||
for row in &rows {
|
||||
let float_s = row
|
||||
.float_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let quant_s = row
|
||||
.quant_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let ratio_s = row
|
||||
.ratio
|
||||
.map(|x| format!("{x:.2}x"))
|
||||
.unwrap_or_else(|| "—".to_string());
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10.3} {:>22} {:>22} {:>12}",
|
||||
row.n, row.bits, row.bytes_per_node, row.quant_best_recall, float_s, quant_s, ratio_s
|
||||
);
|
||||
}
|
||||
|
||||
// Crossover verdict: report whether the quant/float ratio EVER exceeds 1.0
|
||||
// at equal recall, and the per-bit trend of the best-quant-recall as N grows
|
||||
// (is quant getting closer to the equal-recall regime, or not).
|
||||
println!("\n--- crossover verdict (quant-HNSW > float-HNSW at equal recall?) ---");
|
||||
let crossover: Vec<&ScalingRow> = rows
|
||||
.iter()
|
||||
.filter(|r| r.ratio.map(|x| x > 1.0).unwrap_or(false))
|
||||
.collect();
|
||||
if crossover.is_empty() {
|
||||
println!("NO crossover at any measured (N, b): quant never met target recall AND beat float QPS.");
|
||||
} else {
|
||||
for r in &crossover {
|
||||
println!(
|
||||
"CROSSOVER at N={} b={}: quant/float = {:.2}x at recall >= {target:.2}",
|
||||
r.n, r.bits, r.ratio.unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
for &b in &bits_set {
|
||||
let trend: Vec<(usize, f64)> = rows
|
||||
.iter()
|
||||
.filter(|r| r.bits == b)
|
||||
.map(|r| (r.n, r.quant_best_recall))
|
||||
.collect();
|
||||
let trend_s: Vec<String> = trend
|
||||
.iter()
|
||||
.map(|(n, r)| format!("N={n}:{r:.3}"))
|
||||
.collect();
|
||||
println!("b={b} best-quant-recall trend: {}", trend_s.join(" "));
|
||||
}
|
||||
println!("======================================================================\n");
|
||||
|
||||
// Structural invariants (gate-safe at any N): at least one float op met
|
||||
// target at every N (the baseline must work), and quant recall is in range.
|
||||
for &n in &ns {
|
||||
let any_float = rows.iter().any(|r| r.n == n && r.float_op.is_some());
|
||||
assert!(any_float, "no float-HNSW op met target recall at N={n} — baseline broken");
|
||||
}
|
||||
for r in &rows {
|
||||
assert!(
|
||||
(0.0..=1.0).contains(&r.quant_best_recall),
|
||||
"quant recall out of range at N={} b={}: {}",
|
||||
r.n,
|
||||
r.bits,
|
||||
r.quant_best_recall
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// CI-safe structural check for the scaling study at tiny `N` (debug-fast):
|
||||
/// the study runs end-to-end, bytes/node scales with `b`, and the float
|
||||
/// baseline meets target at the smallest N. Does **not** assert any crossover
|
||||
/// (that is the §11 measured question, answered by `scaling_report`).
|
||||
#[test]
|
||||
fn scaling_study_small_is_consistent() {
|
||||
let base = AnnBenchParams::default_fixture(1500);
|
||||
let ns = [1500usize, 3000];
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, 0.90);
|
||||
assert_eq!(rows.len(), ns.len() * bits_set.len());
|
||||
// Bytes/node scales with b at dim=128 (D=128): 16 / 32 / 64.
|
||||
for r in rows.iter().filter(|r| r.n == 1500) {
|
||||
let expect = match r.bits {
|
||||
1 => 16,
|
||||
2 => 32,
|
||||
_ => 64,
|
||||
};
|
||||
assert_eq!(r.bytes_per_node, expect, "B/node wrong for b={}", r.bits);
|
||||
}
|
||||
// Float baseline must meet target at the smallest N.
|
||||
assert!(
|
||||
rows.iter().any(|r| r.n == 1500 && r.float_op.is_some()),
|
||||
"float baseline failed target at small N"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,826 @@
|
||||
//! A correct, dependency-free **float HNSW** graph-ANN index — ADR-261.
|
||||
//!
|
||||
//! # Why this exists
|
||||
//!
|
||||
//! The ruvector crate's retrieval path (AETHER re-ID hot-cache, the `sketch.rs`
|
||||
//! 1-bit prefilter, room fingerprinting) is, at its core, an **approximate
|
||||
//! nearest-neighbour** problem: dense float embedding in, top-K similar ids out.
|
||||
//! Until now the crate had **no graph index** — every `topk` was a linear scan
|
||||
//! (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is
|
||||
//! fine at the small N the unit fixtures use, but it is `O(N)` per query and does
|
||||
//! not scale.
|
||||
//!
|
||||
//! [ADR-156 §5 #1](../../../../../docs/adr/ADR-156-ruvector-fusion-beyond-sota.md)
|
||||
//! lists **SymphonyQG** (SIGMOD 2025) as the lead beyond-SOTA ANN candidate,
|
||||
//! claiming **3.5–17× QPS over HNSW at equal recall** — but graded that claim
|
||||
//! **CLAIMED**, *"not reproduced on our hardware (no HNSW baseline exists to
|
||||
//! compare against)."* You cannot measure a ratio against a baseline you do not
|
||||
//! have. This module **builds that missing HNSW baseline**; [`crate::hnsw_quantized`]
|
||||
//! builds the quantized-rerank variant that tests the *direction* of the
|
||||
//! SymphonyQG bet. ADR-261 reports the **measured** ratio.
|
||||
//!
|
||||
//! # The algorithm (Malkov & Yashunin, TPAMI 2018)
|
||||
//!
|
||||
//! HNSW = a multi-layer navigable small-world graph. Each inserted point gets a
|
||||
//! random **level** `ℓ` (geometrically distributed, mean `1/ln(M)`); it appears
|
||||
//! in all layers `0..=ℓ`. Layer 0 holds every point; higher layers are
|
||||
//! exponentially sparser "express lanes". A search:
|
||||
//!
|
||||
//! 1. Enters at the top layer's single entry point.
|
||||
//! 2. **Greedy-descends** each layer above 0: repeatedly hop to the neighbour
|
||||
//! closest to the query until no neighbour is closer, then drop a layer.
|
||||
//! 3. At layer 0, runs a **best-first beam search** with beam width `ef`,
|
||||
//! keeping the `ef` closest candidates seen, and returns the closest `k`.
|
||||
//!
|
||||
//! Construction inserts each point by searching for its `ef_construction`
|
||||
//! nearest existing neighbours at each of its layers, then connecting it to a
|
||||
//! pruned subset chosen by the **neighbour-selection heuristic** (Algorithm 4 in
|
||||
//! the paper): prefer neighbours that are closer to the new point than to any
|
||||
//! already-selected neighbour, which keeps the graph navigable (diverse edges)
|
||||
//! instead of clumping all edges toward one cluster.
|
||||
//!
|
||||
//! # Determinism (the proof contract)
|
||||
//!
|
||||
//! Level assignment is the only randomness, and it is driven by a **seeded
|
||||
//! SplitMix64** PRNG (the exact pattern from [`crate::rotation`]) — never
|
||||
//! `Date::now`, an OS RNG, or `rand` without a seed. Two indices built from the
|
||||
//! same `(seed, params, insertion order)` are bit-identical, pinned by
|
||||
//! [`tests::hnsw_is_deterministic_for_seed`]. This matters for reproducible
|
||||
//! benchmarks: the recall/QPS numbers in ADR-261 must be regenerable.
|
||||
//!
|
||||
//! # Robustness (no panic on degenerate input)
|
||||
//!
|
||||
//! Empty index, `k > n`, `k == 0`, a single node, zero-dimension vectors,
|
||||
//! ragged-length queries, and `ef < k` are all handled without panicking —
|
||||
//! pinned by the `*_no_panic` / degenerate tests. Graph traversal is bounded by
|
||||
//! the visited-set and the candidate beam, so there is no unbounded recursion
|
||||
//! (the search is iterative, using explicit heaps).
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{BinaryHeap, HashSet};
|
||||
|
||||
/// Distance metric for the index. Both are computed over `Vec<f32>` with an
|
||||
/// `f64` accumulator for numerical stability on long vectors.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Metric {
|
||||
/// Squared euclidean distance `Σ (a_i − b_i)²`. Monotone in euclidean
|
||||
/// distance, so top-K ranking is identical; we skip the sqrt.
|
||||
L2,
|
||||
/// Cosine **distance** `1 − cos(a, b)`. Smaller = more similar. This is
|
||||
/// AETHER's actual angular metric and what the `sketch.rs` sign code
|
||||
/// approximates, so it is the default for ruvector re-ID.
|
||||
Cosine,
|
||||
}
|
||||
|
||||
impl Metric {
|
||||
/// Distance between two equal-length slices under this metric.
|
||||
///
|
||||
/// Ragged lengths are handled charitably (compared over the shorter prefix);
|
||||
/// a degenerate (zero-norm) cosine input yields the maximum cosine distance
|
||||
/// `1.0` rather than a NaN. Never panics.
|
||||
#[inline]
|
||||
pub fn distance(self, a: &[f32], b: &[f32]) -> f32 {
|
||||
let n = a.len().min(b.len());
|
||||
match self {
|
||||
Metric::L2 => {
|
||||
let mut acc = 0.0f64;
|
||||
for i in 0..n {
|
||||
let d = a[i] as f64 - b[i] as f64;
|
||||
acc += d * d;
|
||||
}
|
||||
acc as f32
|
||||
}
|
||||
Metric::Cosine => {
|
||||
let mut dot = 0.0f64;
|
||||
let mut na = 0.0f64;
|
||||
let mut nb = 0.0f64;
|
||||
for i in 0..n {
|
||||
let (x, y) = (a[i] as f64, b[i] as f64);
|
||||
dot += x * y;
|
||||
na += x * x;
|
||||
nb += y * y;
|
||||
}
|
||||
let denom = (na * nb).sqrt();
|
||||
if denom < 1e-12 {
|
||||
1.0
|
||||
} else {
|
||||
(1.0 - dot / denom) as f32
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construction / search hyper-parameters for an [`HnswIndex`].
|
||||
///
|
||||
/// Defaults follow the paper's recommended starting points (`M = 16`,
|
||||
/// `ef_construction = 200`). `ef_search` is the query-time beam width; larger
|
||||
/// `ef_search` trades QPS for recall — the knob the ADR-261 benchmark sweeps to
|
||||
/// find the equal-recall operating point.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct HnswParams {
|
||||
/// Max neighbours per node on layers ≥ 1. Layer 0 uses `2·M` (`m_max0`),
|
||||
/// the paper's standard asymmetry (the base layer needs higher degree).
|
||||
pub m: usize,
|
||||
/// Candidate list size during construction (`efConstruction`). Larger =
|
||||
/// better-connected graph, slower build.
|
||||
pub ef_construction: usize,
|
||||
/// Default beam width at query time (`ef`). Overridable per-query in
|
||||
/// [`HnswIndex::search`].
|
||||
pub ef_search: usize,
|
||||
/// Seed for the level-assignment PRNG. Fixed ⇒ reproducible graph.
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
impl Default for HnswParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: 0x1157_0000_0000_0001u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A min-distance ordering wrapper: a `BinaryHeap<Candidate>` is a **max-heap**,
|
||||
/// so we negate the comparison to make `peek()` the *closest* candidate when we
|
||||
/// want a min-heap, or use it directly for a max-heap of the *farthest*. We keep
|
||||
/// two explicit newtypes to make the intent unmistakable at each call site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Scored {
|
||||
dist: f32,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl PartialEq for Scored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.dist == other.dist && self.id == other.id
|
||||
}
|
||||
}
|
||||
impl Eq for Scored {}
|
||||
|
||||
/// Max-heap ordering: larger `dist` is "greater" ⇒ at the top. Ties broken by
|
||||
/// id so the order is total and deterministic.
|
||||
impl Ord for Scored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.dist
|
||||
.partial_cmp(&other.dist)
|
||||
.unwrap_or(Ordering::Equal)
|
||||
.then(self.id.cmp(&other.id))
|
||||
}
|
||||
}
|
||||
impl PartialOrd for Scored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// `Reverse`-equivalent for a min-heap (closest at top) without pulling in
|
||||
/// `std::cmp::Reverse` boilerplate at every site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct MinScored(Scored);
|
||||
impl PartialEq for MinScored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.0 == other.0
|
||||
}
|
||||
}
|
||||
impl Eq for MinScored {}
|
||||
impl Ord for MinScored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
other.0.cmp(&self.0) // reversed
|
||||
}
|
||||
}
|
||||
impl PartialOrd for MinScored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// A multi-layer HNSW graph index over dense `Vec<f32>` embeddings.
|
||||
///
|
||||
/// IDs are the **insertion index** (`0..len`), returned by [`HnswIndex::search`]
|
||||
/// alongside the distance. The original vectors are retained (the graph needs
|
||||
/// them for distance computation at query time), so memory is
|
||||
/// `O(N·d) + O(N·M)` — the float vectors plus the adjacency lists.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HnswIndex {
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
dim: usize,
|
||||
/// Stored vectors, indexed by id.
|
||||
vectors: Vec<Vec<f32>>,
|
||||
/// `links[id][layer]` = neighbour ids of `id` on `layer`. A node of level
|
||||
/// `ℓ` has `ℓ+1` layers (`0..=ℓ`).
|
||||
links: Vec<Vec<Vec<u32>>>,
|
||||
/// Per-node top level.
|
||||
levels: Vec<usize>,
|
||||
/// Current entry point id (the highest-level node), or `None` if empty.
|
||||
entry: Option<u32>,
|
||||
/// Highest level currently present in the graph.
|
||||
top_level: usize,
|
||||
/// PRNG state for level assignment (advances per insert).
|
||||
rng_state: u64,
|
||||
}
|
||||
|
||||
impl HnswIndex {
|
||||
/// Create an empty index with the given metric and parameters.
|
||||
///
|
||||
/// `dim` is the expected embedding dimension. Inserts of a different length
|
||||
/// are accepted charitably (the metric compares over the shorter prefix), so
|
||||
/// a wrong-length vector degrades recall rather than panicking — but callers
|
||||
/// should keep dimension uniform.
|
||||
pub fn new(dim: usize, metric: Metric, params: HnswParams) -> Self {
|
||||
Self {
|
||||
metric,
|
||||
params,
|
||||
dim,
|
||||
vectors: Vec::new(),
|
||||
links: Vec::new(),
|
||||
levels: Vec::new(),
|
||||
entry: None,
|
||||
top_level: 0,
|
||||
rng_state: params.seed.wrapping_add(0x9E37_79B9_7F4A_7C15),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of indexed points.
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.vectors.len()
|
||||
}
|
||||
|
||||
/// True iff the index holds no points.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.vectors.is_empty()
|
||||
}
|
||||
|
||||
/// The metric this index ranks by.
|
||||
#[inline]
|
||||
pub fn metric(&self) -> Metric {
|
||||
self.metric
|
||||
}
|
||||
|
||||
/// The expected embedding dimension.
|
||||
#[inline]
|
||||
pub fn dim(&self) -> usize {
|
||||
self.dim
|
||||
}
|
||||
|
||||
/// The current entry-point id (highest-level node), or `None` if empty.
|
||||
/// Exposed so the quantized variant ([`crate::hnsw_quantized`]) can traverse
|
||||
/// the **same** graph with a different (quantized) score.
|
||||
#[inline]
|
||||
pub fn entry_point(&self) -> Option<u32> {
|
||||
self.entry
|
||||
}
|
||||
|
||||
/// The highest level currently present in the graph.
|
||||
#[inline]
|
||||
pub fn top_level(&self) -> usize {
|
||||
self.top_level
|
||||
}
|
||||
|
||||
/// The default query-time beam width (`ef_search`) from this index's params.
|
||||
#[inline]
|
||||
pub fn params_ef_search(&self) -> usize {
|
||||
self.params.ef_search
|
||||
}
|
||||
|
||||
/// Borrow the neighbour ids of `id` on `layer`. Returns an empty slice if the
|
||||
/// id is unknown or the node does not reach that layer — never panics. Used
|
||||
/// by the quantized variant to walk the shared graph.
|
||||
#[inline]
|
||||
pub fn neighbours(&self, id: u32, layer: usize) -> &[u32] {
|
||||
match self.links.get(id as usize).and_then(|l| l.get(layer)) {
|
||||
Some(v) => v.as_slice(),
|
||||
None => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// `m_max` for a layer: `2·M` on layer 0, `M` above. The base layer carries
|
||||
/// every node and needs higher degree to stay connected (the paper's
|
||||
/// asymmetric degree cap).
|
||||
#[inline]
|
||||
fn m_max(&self, layer: usize) -> usize {
|
||||
if layer == 0 {
|
||||
self.params.m * 2
|
||||
} else {
|
||||
self.params.m
|
||||
}
|
||||
}
|
||||
|
||||
/// Draw the next node's level from a geometric distribution with parameter
|
||||
/// `m_l = 1/ln(M)` — the paper's level generator — using the **seeded**
|
||||
/// SplitMix64 stream. `floor(−ln(U) · m_l)` with `U ∈ (0, 1]`.
|
||||
fn assign_level(&mut self) -> usize {
|
||||
let m = self.params.m.max(2) as f64;
|
||||
let m_l = 1.0 / m.ln();
|
||||
// Uniform in (0, 1] from the top 53 bits of a SplitMix64 word.
|
||||
let r = split_mix64(&mut self.rng_state);
|
||||
let u = (((r >> 11) as f64) + 1.0) / ((1u64 << 53) as f64 + 1.0);
|
||||
let level = (-(u.ln()) * m_l).floor();
|
||||
if level.is_finite() && level >= 0.0 {
|
||||
level as usize
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert `embedding` with the next sequential id. Returns the assigned id.
|
||||
///
|
||||
/// Builds the node's adjacency by searching the existing graph for its
|
||||
/// nearest neighbours at each of its layers and connecting via the
|
||||
/// neighbour-selection heuristic. The first insert becomes the entry point.
|
||||
pub fn insert(&mut self, embedding: &[f32]) -> u32 {
|
||||
let id = self.vectors.len() as u32;
|
||||
let vec = embedding.to_vec();
|
||||
let node_level = self.assign_level();
|
||||
|
||||
// Push the node into the arrays UP FRONT with empty per-layer link lists.
|
||||
// This is load-bearing: the bidirectional wiring below does
|
||||
// `self.links[nbr][l].push(id)`, after which a neighbour points at `id`;
|
||||
// a subsequent traversal step in the SAME insert can hop to that
|
||||
// neighbour and read `self.links[id]`. If `id`'s links did not exist yet
|
||||
// that read panics (the bug the recall gate caught). The new node has no
|
||||
// *incoming* edges until we add them, and empty outgoing lists, so it is
|
||||
// unreachable by the searches that run before its edges are wired —
|
||||
// pushing it early is safe and keeps every `self.links[*]` index valid.
|
||||
self.vectors.push(vec.clone());
|
||||
self.links.push(vec![Vec::new(); node_level + 1]);
|
||||
self.levels.push(node_level);
|
||||
|
||||
// First node: it is the entry point, no neighbours to connect.
|
||||
if self.entry.is_none() {
|
||||
self.entry = Some(id);
|
||||
self.top_level = node_level;
|
||||
return id;
|
||||
}
|
||||
|
||||
let entry = self.entry.unwrap();
|
||||
let mut ep = entry;
|
||||
|
||||
// Phase 1: greedy-descend from the top of the graph down to the layer
|
||||
// just above the node's own top level, refining the single entry point.
|
||||
let mut layer = self.top_level;
|
||||
while layer > node_level {
|
||||
ep = self.greedy_closest(&vec, ep, layer);
|
||||
if layer == 0 {
|
||||
break;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
// Phase 2: from min(node_level, top_level) down to 0, search for
|
||||
// ef_construction candidates, select neighbours, and wire bidirectional
|
||||
// edges (pruning the neighbour's list if it overflows m_max).
|
||||
let start = node_level.min(self.top_level);
|
||||
let mut layer = start as isize;
|
||||
while layer >= 0 {
|
||||
let l = layer as usize;
|
||||
let candidates =
|
||||
self.search_layer(&vec, &[ep], self.params.ef_construction.max(1), l);
|
||||
let selected = self.select_neighbours(&vec, &candidates, self.m_max(l));
|
||||
|
||||
// Connect node -> selected (write straight into the node's slot).
|
||||
self.links[id as usize][l] = selected.iter().map(|s| s.id).collect();
|
||||
|
||||
// Connect selected -> node (bidirectional), pruning if needed.
|
||||
for s in &selected {
|
||||
let nbr = s.id as usize;
|
||||
self.links[nbr][l].push(id);
|
||||
if self.links[nbr][l].len() > self.m_max(l) {
|
||||
self.prune_neighbours(nbr as u32, l);
|
||||
}
|
||||
}
|
||||
|
||||
// Move the entry for the next-lower layer to the closest candidate.
|
||||
if let Some(best) = candidates
|
||||
.iter()
|
||||
.min_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal))
|
||||
{
|
||||
ep = best.id;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
if node_level > self.top_level {
|
||||
self.top_level = node_level;
|
||||
self.entry = Some(id);
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Greedy single-best descent on one layer: hop to the neighbour closest to
|
||||
/// `query` until no neighbour improves. Iterative (bounded by the graph) —
|
||||
/// no recursion.
|
||||
fn greedy_closest(&self, query: &[f32], start: u32, layer: usize) -> u32 {
|
||||
let mut best = start;
|
||||
let mut best_d = self.metric.distance(query, &self.vectors[best as usize]);
|
||||
loop {
|
||||
let mut improved = false;
|
||||
for &nbr in &self.links[best as usize][layer] {
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
if d < best_d {
|
||||
best_d = d;
|
||||
best = nbr;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
if !improved {
|
||||
return best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Beam search on one layer (paper Algorithm 2): best-first expansion from
|
||||
/// `entry_points`, keeping the `ef` closest results. Returns the result set
|
||||
/// (unsorted; callers sort/truncate). Bounded by a visited set + the `ef`
|
||||
/// result heap — no recursion, no unbounded growth.
|
||||
fn search_layer(
|
||||
&self,
|
||||
query: &[f32],
|
||||
entry_points: &[u32],
|
||||
ef: usize,
|
||||
layer: usize,
|
||||
) -> Vec<Scored> {
|
||||
let mut visited: HashSet<u32> = HashSet::new();
|
||||
// `candidates`: min-heap (closest first) of nodes to expand.
|
||||
let mut candidates: BinaryHeap<MinScored> = BinaryHeap::new();
|
||||
// `results`: max-heap (farthest first) of the best-ef found so far, so
|
||||
// the top is the current worst and is cheap to evict.
|
||||
let mut results: BinaryHeap<Scored> = BinaryHeap::new();
|
||||
|
||||
for &ep in entry_points {
|
||||
if ep as usize >= self.vectors.len() {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[ep as usize]);
|
||||
let s = Scored { dist: d, id: ep };
|
||||
visited.insert(ep);
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
}
|
||||
// Cap results at ef from the start.
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
|
||||
while let Some(MinScored(cur)) = candidates.pop() {
|
||||
// Stop when the closest unexpanded candidate is farther than the
|
||||
// current worst result and the result set is already full.
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if cur.dist > worst && results.len() >= ef {
|
||||
break;
|
||||
}
|
||||
for &nbr in &self.links[cur.id as usize][layer] {
|
||||
if !visited.insert(nbr) {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if results.len() < ef || d < worst {
|
||||
let s = Scored { dist: d, id: nbr };
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_vec()
|
||||
}
|
||||
|
||||
/// Neighbour-selection heuristic (paper Algorithm 4): from `candidates`,
|
||||
/// greedily pick up to `m` that are **closer to the new point than to any
|
||||
/// already-picked neighbour**, giving diverse, navigable edges instead of a
|
||||
/// clump. Candidates are considered nearest-first.
|
||||
fn select_neighbours(&self, _base: &[f32], candidates: &[Scored], m: usize) -> Vec<Scored> {
|
||||
let mut sorted = candidates.to_vec();
|
||||
sorted.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
let mut selected: Vec<Scored> = Vec::with_capacity(m);
|
||||
for cand in sorted {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
// Keep `cand` only if it is closer to `base` than to every already
|
||||
// selected neighbour — the diversity condition.
|
||||
let cand_vec = &self.vectors[cand.id as usize];
|
||||
let mut keep = true;
|
||||
for sel in &selected {
|
||||
let d_cand_sel = self.metric.distance(cand_vec, &self.vectors[sel.id as usize]);
|
||||
if d_cand_sel < cand.dist {
|
||||
keep = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
selected.push(cand);
|
||||
}
|
||||
}
|
||||
// If the diversity filter left us short (sparse graph), backfill with the
|
||||
// remaining nearest candidates so the node is not under-connected.
|
||||
if selected.len() < m {
|
||||
let chosen: HashSet<u32> = selected.iter().map(|s| s.id).collect();
|
||||
let mut rest: Vec<Scored> = candidates
|
||||
.iter()
|
||||
.filter(|c| !chosen.contains(&c.id))
|
||||
.copied()
|
||||
.collect();
|
||||
rest.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
for c in rest {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
selected.push(c);
|
||||
}
|
||||
}
|
||||
selected
|
||||
}
|
||||
|
||||
/// Re-prune a node's neighbour list on `layer` back down to `m_max` using
|
||||
/// the selection heuristic, after a bidirectional edge pushed it over cap.
|
||||
fn prune_neighbours(&mut self, id: u32, layer: usize) {
|
||||
let base = self.vectors[id as usize].clone();
|
||||
let current: Vec<Scored> = self.links[id as usize][layer]
|
||||
.iter()
|
||||
.map(|&nbr| Scored {
|
||||
dist: self.metric.distance(&base, &self.vectors[nbr as usize]),
|
||||
id: nbr,
|
||||
})
|
||||
.collect();
|
||||
let kept = self.select_neighbours(&base, ¤t, self.m_max(layer));
|
||||
self.links[id as usize][layer] = kept.iter().map(|s| s.id).collect();
|
||||
}
|
||||
|
||||
/// Search for the `k` nearest neighbours of `query`, using beam width `ef`
|
||||
/// (clamped to at least `k`). Returns up to `k` `(id, distance)` pairs sorted
|
||||
/// ascending by distance.
|
||||
///
|
||||
/// Degenerate cases return cleanly: empty index ⇒ empty vec; `k == 0` ⇒ empty
|
||||
/// vec; `k > len` ⇒ all points; a single node ⇒ that node. Never panics.
|
||||
pub fn search(&self, query: &[f32], k: usize, ef: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let entry = match self.entry {
|
||||
Some(e) => e,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
let ef = ef.max(k).max(1);
|
||||
|
||||
// Greedy-descend the upper layers to a good layer-0 entry point.
|
||||
let mut ep = entry;
|
||||
let mut layer = self.top_level;
|
||||
while layer > 0 {
|
||||
ep = self.greedy_closest(query, ep, layer);
|
||||
layer -= 1;
|
||||
}
|
||||
// Beam search on layer 0.
|
||||
let mut results = self.search_layer(query, &[ep], ef, 0);
|
||||
results.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
results.truncate(k);
|
||||
results.into_iter().map(|s| (s.id, s.dist)).collect()
|
||||
}
|
||||
|
||||
/// Search using the index's configured default `ef_search`.
|
||||
#[inline]
|
||||
pub fn search_default(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
self.search(query, k, self.params.ef_search)
|
||||
}
|
||||
|
||||
/// Borrow a stored vector by id (for the quantized variant / reranking).
|
||||
#[inline]
|
||||
pub fn vector(&self, id: u32) -> Option<&[f32]> {
|
||||
self.vectors.get(id as usize).map(|v| v.as_slice())
|
||||
}
|
||||
|
||||
/// Brute-force exact top-K linear scan over the stored vectors — the ANN
|
||||
/// **ground truth** and the linear-scan baseline the benchmark measures
|
||||
/// against. `O(N·d)` per query. Returns up to `k` `(id, distance)` ascending.
|
||||
pub fn brute_force(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut scored: Vec<(u32, f32)> = self
|
||||
.vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (i as u32, self.metric.distance(query, v)))
|
||||
.collect();
|
||||
scored.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
scored.truncate(k);
|
||||
scored
|
||||
}
|
||||
}
|
||||
|
||||
/// SplitMix64 step — the same deterministic PRNG used by [`crate::rotation`].
|
||||
/// Public-domain (Sebastiano Vigna). Dependency-free and reproducible.
|
||||
#[inline]
|
||||
pub(crate) fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// SplitMix64-driven uniform in [0,1) for building fixtures (mirrors
|
||||
/// `coverage.rs`'s style so the planted-cluster geometry matches).
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
let r = split_mix64(state);
|
||||
((r >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// Build a planted-cluster fixture: `n` vectors of `dim`, in `clusters`
|
||||
/// Gaussian clusters. Returns the vectors. Deterministic from `seed`.
|
||||
fn planted(dim: usize, n: usize, clusters: usize, seed: u64) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..clusters)
|
||||
.map(|c| {
|
||||
let mut s = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let c = i % clusters;
|
||||
let mut s = seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..dim).map(|d| centres[c][d] + gauss(&mut s) * 0.35).collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build(vectors: &[Vec<f32>], metric: Metric, seed: u64) -> HnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed,
|
||||
};
|
||||
let mut idx = HnswIndex::new(vectors[0].len(), metric, params);
|
||||
for v in vectors {
|
||||
idx.insert(v);
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
/// Recall@k of HNSW search vs brute-force ground truth, averaged over queries
|
||||
/// drawn from the same planted clusters.
|
||||
fn recall_at_k(
|
||||
idx: &HnswIndex,
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
clusters: usize,
|
||||
k: usize,
|
||||
ef: usize,
|
||||
n_queries: usize,
|
||||
seed: u64,
|
||||
) -> f64 {
|
||||
let centres_seed = seed; // reuse fixture seed for matching cluster geometry
|
||||
let mut total = 0.0f64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut s = centres_seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
// A query near cluster centre c: regenerate the centre then jitter.
|
||||
let mut cs = centres_seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
|
||||
let truth: HashSet<u32> = idx.brute_force(&qv, k).into_iter().map(|(id, _)| id).collect();
|
||||
let got = idx.search(&qv, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / k as f64;
|
||||
let _ = vectors;
|
||||
}
|
||||
total / n_queries as f64
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_index_search_is_empty_no_panic() {
|
||||
let idx = HnswIndex::new(8, Metric::L2, HnswParams::default());
|
||||
assert!(idx.is_empty());
|
||||
assert!(idx.search(&[0.0; 8], 5, 16).is_empty());
|
||||
assert!(idx.brute_force(&[0.0; 8], 5).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_returns_itself() {
|
||||
let mut idx = HnswIndex::new(4, Metric::L2, HnswParams::default());
|
||||
let id = idx.insert(&[1.0, 2.0, 3.0, 4.0]);
|
||||
assert_eq!(id, 0);
|
||||
let r = idx.search(&[1.0, 2.0, 3.0, 4.0], 5, 16);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, 0);
|
||||
assert!(r[0].1 < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn k_zero_and_k_gt_n_no_panic() {
|
||||
let vectors = planted(16, 40, 4, 0xABCD);
|
||||
let idx = build(&vectors, Metric::L2, 0x1234);
|
||||
assert!(idx.search(&vectors[0], 0, 16).is_empty());
|
||||
// k > n returns all n.
|
||||
let r = idx.search(&vectors[0], 1000, 64);
|
||||
assert_eq!(r.len(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_query_no_panic() {
|
||||
let vectors = planted(16, 30, 3, 0x55);
|
||||
let idx = build(&vectors, Metric::Cosine, 0x66);
|
||||
// Short and long queries must not panic.
|
||||
assert!(!idx.search(&[1.0, 2.0, 3.0], 3, 16).is_empty());
|
||||
let long: Vec<f32> = (0..100).map(|i| i as f32).collect();
|
||||
assert!(!idx.search(&long, 3, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn self_query_ranks_self_first() {
|
||||
let vectors = planted(32, 200, 8, 0x77);
|
||||
let idx = build(&vectors, Metric::L2, 0x88);
|
||||
for &probe in &[0usize, 50, 137, 199] {
|
||||
let r = idx.search(&vectors[probe], 1, 64);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, probe as u32, "self-query should return the stored self");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hnsw_is_deterministic_for_seed() {
|
||||
// Same (seed, params, insertion order) ⇒ identical level assignment and
|
||||
// identical search output.
|
||||
let vectors = planted(24, 150, 6, 0x2222);
|
||||
let a = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
let b = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
assert_eq!(a.levels, b.levels, "level assignment must be deterministic");
|
||||
let q = &vectors[42];
|
||||
assert_eq!(a.search(q, 10, 64), b.search(q, 10, 64));
|
||||
// A different seed (almost surely) changes the level structure.
|
||||
let c = build(&vectors, Metric::Cosine, 0x1357);
|
||||
assert_ne!(a.levels, c.levels, "different seed should change levels");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_l2() {
|
||||
// THE CORRECTNESS GATE (ADR-261): HNSW recall@10 vs brute-force must be
|
||||
// >= 0.95 at a reasonable ef. Low recall ⇒ a bug in the graph.
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0x9999;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::L2, 0xAAAA);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (L2) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_cosine() {
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0xBBBB;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::Cosine, 0xCCCC);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (cosine) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn higher_ef_does_not_reduce_recall() {
|
||||
// Monotonicity sanity: more beam width should not hurt recall.
|
||||
let dim = 48;
|
||||
let vectors = planted(dim, 1000, 16, 0xD00D);
|
||||
let idx = build(&vectors, Metric::L2, 0xE00E);
|
||||
let lo = recall_at_k(&idx, &vectors, dim, 16, 10, 16, 48, 0xD00D);
|
||||
let hi = recall_at_k(&idx, &vectors, dim, 16, 10, 128, 48, 0xD00D);
|
||||
assert!(hi + 1e-9 >= lo, "recall dropped with larger ef: {lo:.3} -> {hi:.3}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_dim_no_panic() {
|
||||
// Degenerate zero-dimension index: inserts and searches must not panic.
|
||||
let mut idx = HnswIndex::new(0, Metric::Cosine, HnswParams::default());
|
||||
idx.insert(&[]);
|
||||
idx.insert(&[]);
|
||||
let r = idx.search(&[], 2, 16);
|
||||
assert_eq!(r.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
//! A **SymphonyQG-style quantized-traversal HNSW** — ADR-261 (multi-bit, §11).
|
||||
//!
|
||||
//! # The SymphonyQG bet (what we are testing)
|
||||
//!
|
||||
//! [SymphonyQG (SIGMOD 2025)](../../../../../docs/adr/ADR-261-ruvector-graph-ann-index.md)
|
||||
//! unifies **quantization with graph traversal**: instead of computing the full
|
||||
//! float distance at every node the beam search visits (the cost that dominates
|
||||
//! float HNSW — one `O(d)` float dot/diff per visited node), it scores traversal
|
||||
//! candidates with a **cheap quantized distance** and only computes the exact
|
||||
//! float distance for the *final* candidate set, which it **reranks**. The bet:
|
||||
//! the quantized score is cheap enough — and accurate enough to keep the beam on
|
||||
//! the right path — that you visit roughly as many nodes but pay far less per
|
||||
//! node, and recover the small recall loss with a final exact rerank. Source
|
||||
//! reports **3.5–17× QPS over HNSW at equal recall**.
|
||||
//!
|
||||
//! # Our implementation (honest scope)
|
||||
//!
|
||||
//! We are **not** reproducing SymphonyQG's exact system (their RaBitQ-fused codes,
|
||||
//! their SIMD layout, their refined graph). We build the **direction** of the
|
||||
//! claim from the pieces this crate already has, so the comparison is
|
||||
//! apples-to-apples on *our* hardware:
|
||||
//!
|
||||
//! - **Same graph** as the float [`crate::HnswIndex`] — identical structure,
|
||||
//! identical seed, identical level assignment. The *only* variable between the
|
||||
//! float and quantized search is **how a candidate is scored during traversal**,
|
||||
//! so any QPS/recall difference is attributable to the quantization, not to a
|
||||
//! different graph.
|
||||
//! - **Quantized score = `b`-bit code over the RaBitQ Pass-2 rotated coordinates**
|
||||
//! ([`crate::rotation`] + the multi-bit scalar quantizer mirrored from
|
||||
//! [ADR-156 §10](../../../../../docs/adr/ADR-156-ruvector-fusion-beyond-sota.md)'s
|
||||
//! `coverage::measure_multibit`). Each node stores a `b`-bit-per-dimension code
|
||||
//! over the padded rotation length `D = next_pow2(dim)`. During traversal we
|
||||
//! compare query-code vs node-code by the **L1 distance over the per-dim
|
||||
//! codes** — a few machine words of integer work, no per-dimension float work.
|
||||
//! For `b == 1` the codes are `{0, 1}` and the L1 distance is **exactly the
|
||||
//! 1-bit Hamming distance** of the original ADR-261 construction, so `b == 1`
|
||||
//! is fully backward-compatible.
|
||||
//! - **Exact float rerank** of the final beam: the top `rerank` candidates by
|
||||
//! code-L1 are re-scored with the true float metric and the best `k` returned.
|
||||
//!
|
||||
//! Higher `b` keeps the traversal beam on-path better than 1-bit (ADR-156 §10
|
||||
//! measured 1/2/3/4-bit strict-K coverage at ~46/54/67/74%), at a memory cost
|
||||
//! that scales linearly with `b` (bytes/node = `ceil(D·b/8)`). **Whether the
|
||||
//! extra bits net a QPS win at equal recall — and at what N a crossover with
|
||||
//! float HNSW appears, if any — is the measured question ADR-261 §11 answers.**
|
||||
//! We report the real number, win or lose, and do not tune to manufacture a
|
||||
//! speedup.
|
||||
//!
|
||||
//! # Determinism & robustness
|
||||
//!
|
||||
//! The graph seed drives everything (level assignment), so the quantized index
|
||||
//! is as reproducible as the float one. Empty/degenerate inputs are guarded
|
||||
//! exactly as in [`crate::hnsw`] — no panic on empty index, `k > n`, `k == 0`,
|
||||
//! single node, ragged query, or zero dim.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{BinaryHeap, HashSet};
|
||||
|
||||
use crate::hnsw::{HnswIndex, HnswParams, Metric};
|
||||
use crate::rotation::Rotation;
|
||||
|
||||
/// Symmetric clamp range for the uniform mid-rise scalar quantizer, in rotated-
|
||||
/// coordinate units. The normalized FHT (`1/√D`) puts AETHER-shape rotated
|
||||
/// coordinates roughly in `[-3, 3]`; out-of-range coords clamp to the end codes.
|
||||
/// This is the **same `RANGE = 3.0`** as ADR-156 §10's `coverage::measure_multibit`,
|
||||
/// so the multi-bit code here is the same scheme that module measured.
|
||||
const RANGE: f32 = 3.0;
|
||||
|
||||
/// A `b`-bit-per-dimension scalar code of a rotated embedding over the padded
|
||||
/// length `D`, compared by per-dim L1.
|
||||
///
|
||||
/// For `bits == 1` the per-dim code is `{0, 1}` (sign), and L1 over those codes
|
||||
/// is exactly POPCNT Hamming — so the 1-bit case is bit-for-bit the original
|
||||
/// ADR-261 construction. For `bits ∈ {2, 4}` the code is a uniform mid-rise
|
||||
/// quantizer with `2^bits` levels over `[-RANGE, RANGE]`.
|
||||
#[derive(Debug, Clone)]
|
||||
struct Code {
|
||||
/// Per-dimension codes (`0..2^bits`), one entry per padded dimension `D`.
|
||||
/// Kept unpacked as `u8` for branch-free L1; the *reported* memory cost is
|
||||
/// the packed footprint (`ceil(D·bits/8)`), since a production node would
|
||||
/// store the packed form. (We measure the packed bytes/node explicitly in
|
||||
/// [`QuantizedHnswIndex::bytes_per_node`].)
|
||||
codes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Code {
|
||||
/// L1 distance over the per-dimension codes — the multi-bit generalization
|
||||
/// of Hamming. At `bits == 1` (codes in `{0,1}`) this equals the popcount of
|
||||
/// the XOR, i.e. the 1-bit Hamming distance.
|
||||
#[inline]
|
||||
fn l1(&self, other: &Code) -> u32 {
|
||||
let n = self.codes.len().min(other.codes.len());
|
||||
let mut acc = 0u32;
|
||||
for i in 0..n {
|
||||
acc += (self.codes[i] as i32 - other.codes[i] as i32).unsigned_abs();
|
||||
}
|
||||
acc
|
||||
}
|
||||
}
|
||||
|
||||
/// Quantize the rotated coordinates of `embedding` to a `bits`-bit-per-dimension
|
||||
/// [`Code`] over the padded rotation length `D = rotation.padded_dim()`.
|
||||
///
|
||||
/// `bits == 1` reduces to sign-quantization (code `1` iff the rotated coord ≥ 0),
|
||||
/// preserving the original 1-bit construction; `bits ∈ {2, 4}` uses a uniform
|
||||
/// mid-rise quantizer with `2^bits` levels over `[-RANGE, RANGE]`, identical to
|
||||
/// ADR-156 §10's `measure_multibit`.
|
||||
fn encode(embedding: &[f32], rotation: &Rotation, bits: u32) -> Code {
|
||||
let rotated = rotation.apply_padded(embedding);
|
||||
let levels = 1u32 << bits; // 2^bits codes per dim
|
||||
let codes: Vec<u8> = rotated
|
||||
.iter()
|
||||
.map(|&x| {
|
||||
if bits == 1 {
|
||||
// Sign code: identical to the original 1-bit construction.
|
||||
u8::from(x >= 0.0)
|
||||
} else {
|
||||
let t = ((x + RANGE) / (2.0 * RANGE)).clamp(0.0, 1.0); // → [0,1]
|
||||
let code = (t * (levels - 1) as f32).round() as u32;
|
||||
code.min(levels - 1) as u8
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Code { codes }
|
||||
}
|
||||
|
||||
/// Packed bytes a node's `bits`-bit code occupies over padded length `D`:
|
||||
/// `ceil(D·bits/8)`. The memory cost reported by ADR-261 §11 (1-bit → `D/8`,
|
||||
/// 2-bit → `D/4`, 4-bit → `D/2`).
|
||||
#[inline]
|
||||
fn packed_bytes(padded_dim: usize, bits: u32) -> usize {
|
||||
(padded_dim * bits as usize).div_ceil(8)
|
||||
}
|
||||
|
||||
/// Min-heap node for the quantized beam (closest code-L1 at the top).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct HScored {
|
||||
/// Code-L1 distance (quantized score) — the traversal key.
|
||||
dist: u32,
|
||||
id: u32,
|
||||
}
|
||||
impl PartialEq for HScored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.dist == other.dist && self.id == other.id
|
||||
}
|
||||
}
|
||||
impl Eq for HScored {}
|
||||
impl Ord for HScored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.dist.cmp(&other.dist).then(self.id.cmp(&other.id))
|
||||
}
|
||||
}
|
||||
impl PartialOrd for HScored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
/// Reversed wrapper for a min-heap (smallest code-L1 at the top).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct MinH(HScored);
|
||||
impl PartialEq for MinH {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.0 == other.0
|
||||
}
|
||||
}
|
||||
impl Eq for MinH {}
|
||||
impl Ord for MinH {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
other.0.cmp(&self.0)
|
||||
}
|
||||
}
|
||||
impl PartialOrd for MinH {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// A SymphonyQG-style HNSW: the same graph as [`HnswIndex`], traversed by a
|
||||
/// **cheap `b`-bit code-L1 score**, with a final **exact-float rerank**.
|
||||
///
|
||||
/// Built by inserting the same vectors in the same order with the same seed as
|
||||
/// a float [`HnswIndex`], so the two indices share identical graph structure and
|
||||
/// only differ in how the beam is scored. The shared [`Rotation`] (seed + dim)
|
||||
/// is the index/query frame for the `b`-bit codes. `bits ∈ {1, 2, 4}` selects
|
||||
/// the traversal-code resolution; `bits == 1` is the original 1-bit Hamming
|
||||
/// construction.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct QuantizedHnswIndex {
|
||||
/// The underlying graph (built with the float metric for exact rerank).
|
||||
graph: HnswIndex,
|
||||
/// Per-node `b`-bit codes, indexed by id (parallel to graph vectors).
|
||||
codes: Vec<Code>,
|
||||
/// The rotation frame shared by index and query codes.
|
||||
rotation: Rotation,
|
||||
/// Bits per dimension of the traversal code (`1`, `2`, or `4`).
|
||||
bits: u32,
|
||||
/// Number of final candidates to exact-float rerank (≥ k at query time).
|
||||
default_rerank: usize,
|
||||
}
|
||||
|
||||
impl QuantizedHnswIndex {
|
||||
/// Build a 1-bit quantized index (the original ADR-261 construction).
|
||||
///
|
||||
/// Equivalent to [`QuantizedHnswIndex::build_bits`] with `bits = 1`; kept as
|
||||
/// the backward-compatible entry point so existing callers and tests are
|
||||
/// unchanged.
|
||||
pub fn build(
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
rotation_seed: u64,
|
||||
default_rerank: usize,
|
||||
) -> Self {
|
||||
Self::build_bits(vectors, dim, metric, params, rotation_seed, 1, default_rerank)
|
||||
}
|
||||
|
||||
/// Build a `bits`-bit quantized index over `vectors`, mirroring a float
|
||||
/// [`HnswIndex`] built with the same `(dim, metric, params)` and insertion
|
||||
/// order. The `rotation_seed` fixes the code frame (index and query share it).
|
||||
///
|
||||
/// `bits` is clamped to `{1, 2, 4}` (the resolutions ADR-261 §11 sweeps): any
|
||||
/// other value is rounded up to the nearest of these so the constructor is
|
||||
/// total. `default_rerank` is how many top-code-L1 candidates get an exact
|
||||
/// float re-score before returning the best `k`; it is clamped to `≥ k` at
|
||||
/// query time. A larger rerank recovers more recall at more float cost — the
|
||||
/// knob that, alongside `ef`, sets the equal-recall operating point.
|
||||
pub fn build_bits(
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
rotation_seed: u64,
|
||||
bits: u32,
|
||||
default_rerank: usize,
|
||||
) -> Self {
|
||||
let bits = clamp_bits(bits);
|
||||
let rotation = Rotation::new(rotation_seed, dim);
|
||||
let mut graph = HnswIndex::new(dim, metric, params);
|
||||
let mut codes = Vec::with_capacity(vectors.len());
|
||||
for v in vectors {
|
||||
graph.insert(v);
|
||||
codes.push(encode(v, &rotation, bits));
|
||||
}
|
||||
Self {
|
||||
graph,
|
||||
codes,
|
||||
rotation,
|
||||
bits,
|
||||
default_rerank: default_rerank.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of indexed points.
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.graph.len()
|
||||
}
|
||||
|
||||
/// True iff empty.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.graph.is_empty()
|
||||
}
|
||||
|
||||
/// Borrow the underlying float graph (for shared-graph benchmark parity:
|
||||
/// the float-HNSW baseline runs on *this* graph so the only variable is
|
||||
/// scoring).
|
||||
#[inline]
|
||||
pub fn graph(&self) -> &HnswIndex {
|
||||
&self.graph
|
||||
}
|
||||
|
||||
/// The rerank width this index defaults to.
|
||||
#[inline]
|
||||
pub fn default_rerank(&self) -> usize {
|
||||
self.default_rerank
|
||||
}
|
||||
|
||||
/// Bits per dimension of the traversal code.
|
||||
#[inline]
|
||||
pub fn bits(&self) -> u32 {
|
||||
self.bits
|
||||
}
|
||||
|
||||
/// Packed memory footprint of one node's traversal code, in bytes:
|
||||
/// `ceil(D·bits/8)` where `D = next_pow2(dim)` is the padded rotation length.
|
||||
/// This is the per-node cost ADR-261 §11 reports for each `b`.
|
||||
#[inline]
|
||||
pub fn bytes_per_node(&self) -> usize {
|
||||
packed_bytes(self.rotation.padded_dim(), self.bits)
|
||||
}
|
||||
|
||||
/// SymphonyQG-style search: traverse the graph scoring candidates by the
|
||||
/// **`b`-bit code-L1**, collect a beam of `ef`, then **exact-float rerank**
|
||||
/// the top `rerank` (clamped ≥ k) and return the best `k` as `(id, float_dist)`.
|
||||
///
|
||||
/// Degenerate cases mirror [`HnswIndex::search`]: empty ⇒ empty; `k == 0` ⇒
|
||||
/// empty; `k > n` ⇒ all; never panics.
|
||||
pub fn search_quantized(
|
||||
&self,
|
||||
query: &[f32],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
rerank: usize,
|
||||
) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let ef = ef.max(k).max(1);
|
||||
let rerank = rerank.max(k);
|
||||
let q_code = encode(query, &self.rotation, self.bits);
|
||||
|
||||
// Entry point: the graph's entry (highest-level node).
|
||||
let entry = match self.graph.entry_point() {
|
||||
Some(e) => e,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
// Greedy-descend upper layers by code-L1, then beam-search layer 0.
|
||||
let mut ep = entry;
|
||||
let mut layer = self.graph.top_level();
|
||||
while layer > 0 {
|
||||
ep = self.greedy_code(&q_code, ep, layer);
|
||||
layer -= 1;
|
||||
}
|
||||
let beam = self.beam_code(&q_code, ep, ef);
|
||||
|
||||
// Exact-float rerank of the top `rerank` code-L1 candidates.
|
||||
let mut cand: Vec<HScored> = beam;
|
||||
cand.sort_by_key(|c| c.dist);
|
||||
cand.truncate(rerank);
|
||||
let mut reranked: Vec<(u32, f32)> = cand
|
||||
.iter()
|
||||
.filter_map(|c| {
|
||||
self.graph
|
||||
.vector(c.id)
|
||||
.map(|v| (c.id, self.graph.metric().distance(query, v)))
|
||||
})
|
||||
.collect();
|
||||
reranked.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
reranked.truncate(k);
|
||||
reranked
|
||||
}
|
||||
|
||||
/// Search using the index's default `ef` (from graph params) and rerank.
|
||||
#[inline]
|
||||
pub fn search_default(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
self.search_quantized(query, k, self.graph.params_ef_search(), self.default_rerank)
|
||||
}
|
||||
|
||||
/// Greedy single-best descent on a layer scored by code-L1.
|
||||
fn greedy_code(&self, q_code: &Code, start: u32, layer: usize) -> u32 {
|
||||
let mut best = start;
|
||||
let mut best_d = self.codes[best as usize].l1(q_code);
|
||||
loop {
|
||||
let mut improved = false;
|
||||
for &nbr in self.graph.neighbours(best, layer) {
|
||||
let d = self.codes[nbr as usize].l1(q_code);
|
||||
if d < best_d {
|
||||
best_d = d;
|
||||
best = nbr;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
if !improved {
|
||||
return best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Beam search on layer 0 scored by code-L1. Returns the `ef` best-code nodes
|
||||
/// (unsorted). Iterative — bounded by the visited set + the ef beam.
|
||||
fn beam_code(&self, q_code: &Code, ep: u32, ef: usize) -> Vec<HScored> {
|
||||
let mut visited: HashSet<u32> = HashSet::new();
|
||||
let mut candidates: BinaryHeap<MinH> = BinaryHeap::new();
|
||||
let mut results: BinaryHeap<HScored> = BinaryHeap::new(); // max-heap: worst at top
|
||||
|
||||
let d0 = self.codes[ep as usize].l1(q_code);
|
||||
let s0 = HScored { dist: d0, id: ep };
|
||||
visited.insert(ep);
|
||||
candidates.push(MinH(s0));
|
||||
results.push(s0);
|
||||
|
||||
while let Some(MinH(cur)) = candidates.pop() {
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(u32::MAX);
|
||||
if cur.dist > worst && results.len() >= ef {
|
||||
break;
|
||||
}
|
||||
for &nbr in self.graph.neighbours(cur.id, 0) {
|
||||
if !visited.insert(nbr) {
|
||||
continue;
|
||||
}
|
||||
let d = self.codes[nbr as usize].l1(q_code);
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(u32::MAX);
|
||||
if results.len() < ef || d < worst {
|
||||
let s = HScored { dist: d, id: nbr };
|
||||
candidates.push(MinH(s));
|
||||
results.push(s);
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_vec()
|
||||
}
|
||||
}
|
||||
|
||||
/// Clamp a requested bit-depth to the supported `{1, 2, 4}` set (round up to the
|
||||
/// nearest supported value; `0` → `1`, `3` → `4`, `> 4` → `4`).
|
||||
#[inline]
|
||||
fn clamp_bits(bits: u32) -> u32 {
|
||||
match bits {
|
||||
0 | 1 => 1,
|
||||
2 => 2,
|
||||
_ => 4,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
((split_mix64(state) >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
fn planted(dim: usize, n: usize, clusters: usize, seed: u64) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..clusters)
|
||||
.map(|c| {
|
||||
let mut s = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let c = i % clusters;
|
||||
let mut s = seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..dim).map(|d| centres[c][d] + gauss(&mut s) * 0.35).collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
fn params(seed: u64) -> HnswParams {
|
||||
HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_quantized_search_is_empty_no_panic() {
|
||||
let idx = QuantizedHnswIndex::build(&[], 8, Metric::Cosine, params(1), 0x42, 16);
|
||||
assert!(idx.is_empty());
|
||||
assert!(idx.search_quantized(&[0.0; 8], 5, 16, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_quantized_returns_itself() {
|
||||
let v = vec![vec![1.0, 2.0, 3.0, 4.0]];
|
||||
let idx = QuantizedHnswIndex::build(&v, 4, Metric::L2, params(2), 0x7, 8);
|
||||
let r = idx.search_quantized(&v[0], 3, 16, 8);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn k_zero_and_k_gt_n_no_panic() {
|
||||
let vectors = planted(16, 40, 4, 0xABCD);
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 16, Metric::L2, params(3), 0x9, 32);
|
||||
assert!(idx.search_quantized(&vectors[0], 0, 16, 16).is_empty());
|
||||
let r = idx.search_quantized(&vectors[0], 1000, 64, 64);
|
||||
assert_eq!(r.len(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_query_no_panic() {
|
||||
let vectors = planted(16, 30, 3, 0x55);
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 16, Metric::Cosine, params(4), 0xB, 16);
|
||||
assert!(!idx.search_quantized(&[1.0, 2.0, 3.0], 3, 16, 16).is_empty());
|
||||
let long: Vec<f32> = (0..100).map(|i| i as f32).collect();
|
||||
assert!(!idx.search_quantized(&long, 3, 16, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quantized_is_deterministic() {
|
||||
let vectors = planted(32, 300, 8, 0x2468);
|
||||
let a = QuantizedHnswIndex::build(&vectors, 32, Metric::Cosine, params(0xFEED), 0xC0DE, 32);
|
||||
let b = QuantizedHnswIndex::build(&vectors, 32, Metric::Cosine, params(0xFEED), 0xC0DE, 32);
|
||||
let q = &vectors[100];
|
||||
assert_eq!(
|
||||
a.search_quantized(q, 10, 64, 32),
|
||||
b.search_quantized(q, 10, 64, 32),
|
||||
"quantized search must be deterministic"
|
||||
);
|
||||
}
|
||||
|
||||
/// Recall@10 of quantized-HNSW vs brute-force ground truth, averaged over
|
||||
/// queries. With an exact-float rerank, recall should be high (the rerank
|
||||
/// repairs most of the 1-bit traversal's coarseness). This is the quantized
|
||||
/// variant's correctness gate.
|
||||
#[test]
|
||||
fn quantized_recall_at_10_is_high_with_rerank() {
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0x9999;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
// Generous rerank so the exact float repairs the coarse Hamming beam.
|
||||
let idx = QuantizedHnswIndex::build(&vectors, dim, Metric::L2, params(0xAAAA), 0x5EED, 64);
|
||||
|
||||
let mut total = 0.0f64;
|
||||
let n_queries = 64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut cs = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let mut s = seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
let truth: HashSet<u32> = idx
|
||||
.graph()
|
||||
.brute_force(&qv, 10)
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
let got = idx.search_quantized(&qv, 10, 128, 64);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / 10.0;
|
||||
}
|
||||
let recall = total / n_queries as f64;
|
||||
// The 1-bit code is coarse, so we do not demand the float 0.95 gate here;
|
||||
// but with a 64-wide rerank over an ef=128 beam it must be clearly useful
|
||||
// (well above random). ADR-261 reports the exact number; this gate just
|
||||
// catches a broken traversal/rerank.
|
||||
assert!(
|
||||
recall >= 0.80,
|
||||
"quantized recall@10 = {recall:.4} too low — traversal or rerank bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_dim_no_panic() {
|
||||
let vectors = vec![vec![], vec![]];
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 0, Metric::Cosine, params(5), 0x1, 4);
|
||||
let r = idx.search_quantized(&[], 2, 16, 4);
|
||||
assert_eq!(r.len(), 2);
|
||||
}
|
||||
|
||||
// ----- multi-bit (ADR-261 §11) -----
|
||||
|
||||
/// `bits == 1` via `build_bits` is byte-for-byte the legacy `build` 1-bit
|
||||
/// construction: same codes, same search output. Backward-compatibility pin.
|
||||
#[test]
|
||||
fn one_bit_build_bits_matches_legacy_build() {
|
||||
let vectors = planted(32, 400, 8, 0x1B17);
|
||||
let legacy = QuantizedHnswIndex::build(&vectors, 32, Metric::L2, params(0x5151), 0xC0DE, 40);
|
||||
let viabits =
|
||||
QuantizedHnswIndex::build_bits(&vectors, 32, Metric::L2, params(0x5151), 0xC0DE, 1, 40);
|
||||
assert_eq!(legacy.bits(), 1);
|
||||
assert_eq!(viabits.bits(), 1);
|
||||
let q = &vectors[123];
|
||||
assert_eq!(
|
||||
legacy.search_quantized(q, 10, 64, 40),
|
||||
viabits.search_quantized(q, 10, 64, 40),
|
||||
"build_bits(…,1,…) must equal legacy build(…)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Unsupported bit-depths round up to the supported `{1,2,4}` set so the
|
||||
/// constructor is total (no panic, predictable resolution).
|
||||
#[test]
|
||||
fn bits_are_clamped_to_supported_set() {
|
||||
let vectors = planted(16, 50, 4, 0xB175);
|
||||
for (req, exp) in [(0u32, 1u32), (1, 1), (2, 2), (3, 4), (4, 4), (7, 4)] {
|
||||
let idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
16,
|
||||
Metric::L2,
|
||||
params(0x9),
|
||||
0xB,
|
||||
req,
|
||||
16,
|
||||
);
|
||||
assert_eq!(idx.bits(), exp, "bits {req} should clamp to {exp}");
|
||||
// and it must still search without panic
|
||||
assert!(!idx.search_quantized(&vectors[0], 5, 32, 20).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes/node scales linearly with `bits`: for a power-of-two dim `D`,
|
||||
/// 1-bit → D/8, 2-bit → D/4, 4-bit → D/2.
|
||||
#[test]
|
||||
fn bytes_per_node_scales_with_bits() {
|
||||
let vectors = planted(128, 20, 4, 0xBEEF);
|
||||
let b1 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 1, 16);
|
||||
let b2 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 2, 16);
|
||||
let b4 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 4, 16);
|
||||
assert_eq!(b1.bytes_per_node(), 16, "128-d 1-bit = 16 B/node");
|
||||
assert_eq!(b2.bytes_per_node(), 32, "128-d 2-bit = 32 B/node");
|
||||
assert_eq!(b4.bytes_per_node(), 64, "128-d 4-bit = 64 B/node");
|
||||
}
|
||||
|
||||
/// More bits must not *reduce* recall at a fixed (ef, rerank): the multi-bit
|
||||
/// code is a strictly finer angle proxy than 1-bit, so the traversal beam can
|
||||
/// only land on equal-or-better candidates for the rerank to repair. This is
|
||||
/// the core ADR-261 §11 hypothesis (multi-bit keeps the beam on-path better),
|
||||
/// pinned as a regression gate. We assert a small tolerance for ties.
|
||||
#[test]
|
||||
fn more_bits_does_not_reduce_recall() {
|
||||
let dim = 64;
|
||||
let n = 3000;
|
||||
let clusters = 32;
|
||||
let seed = 0x7A11;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let recall_for = |bits: u32| -> f64 {
|
||||
let idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
dim,
|
||||
Metric::L2,
|
||||
params(0xA11A),
|
||||
0x5EED,
|
||||
bits,
|
||||
// Modest rerank so traversal quality — not a huge rerank pool —
|
||||
// is what drives the recall difference between bit depths.
|
||||
20,
|
||||
);
|
||||
let mut total = 0.0f64;
|
||||
let n_queries = 64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut cs = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let mut s = seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
let truth: HashSet<u32> = idx
|
||||
.graph()
|
||||
.brute_force(&qv, 10)
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
let got = idx.search_quantized(&qv, 10, 64, 20);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / 10.0;
|
||||
}
|
||||
total / n_queries as f64
|
||||
};
|
||||
let r1 = recall_for(1);
|
||||
let r2 = recall_for(2);
|
||||
let r4 = recall_for(4);
|
||||
// 2-bit and 4-bit must be at least as good as 1-bit (small tie tolerance).
|
||||
assert!(
|
||||
r2 + 0.02 >= r1,
|
||||
"2-bit recall {r2:.4} regressed vs 1-bit {r1:.4}"
|
||||
);
|
||||
assert!(
|
||||
r4 + 0.02 >= r1,
|
||||
"4-bit recall {r4:.4} regressed vs 1-bit {r1:.4}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -28,9 +28,12 @@
|
||||
|
||||
#[cfg(feature = "crv")]
|
||||
pub mod crv;
|
||||
pub mod ann_measure;
|
||||
pub mod coverage;
|
||||
pub mod estimator;
|
||||
pub mod event_log;
|
||||
pub mod hnsw;
|
||||
pub mod hnsw_quantized;
|
||||
pub mod mat;
|
||||
pub mod rotation;
|
||||
pub mod signal;
|
||||
@@ -41,6 +44,8 @@ pub use estimator::{
|
||||
DistanceEstimator, EstimatorBank, EstimatorQuery, EstimatorSketch, SideInfo,
|
||||
};
|
||||
pub use event_log::{NoveltyEvent, PrivacyEventLog};
|
||||
pub use hnsw::{HnswIndex, HnswParams, Metric};
|
||||
pub use hnsw_quantized::QuantizedHnswIndex;
|
||||
pub use rotation::Rotation;
|
||||
pub use sketch::{
|
||||
Sketch, SketchBank, SketchError, WireSketch, WireSketchError, WIRE_SKETCH_FORMAT_VERSION,
|
||||
|
||||
@@ -63,6 +63,13 @@ wifi-densepose-worldgraph = { version = "0.3.0", path = "../wifi-densepose-world
|
||||
wifi-densepose-bfld = { version = "0.3.1", path = "../wifi-densepose-bfld", default-features = false }
|
||||
wifi-densepose-geo = { version = "0.1.0", path = "../wifi-densepose-geo" }
|
||||
|
||||
# ADR-262 P3: live RuField surface. The thin anti-corruption bridge that turns
|
||||
# this server's governed sensing cycle into signed RuField `FieldEvent`s on
|
||||
# `/api/field` + `/ws/field`. It path-deps the standalone `vendor/rufield`
|
||||
# submodule (it is the single coupling point — ADR-262 §5.4) and pulls in no
|
||||
# RuView internal crate, so the dep surface added here is just the bridge.
|
||||
wifi-densepose-rufield = { version = "0.3.0", path = "../wifi-densepose-rufield" }
|
||||
|
||||
# midstream — real-time introspection / low-latency tap (ADR-099 D1).
|
||||
# Two crates only, on purpose: scheduler / neural-solver / strange-loop are
|
||||
# explicitly out of scope of ADR-099 (D5).
|
||||
|
||||
@@ -23,6 +23,10 @@ pub mod model_format;
|
||||
pub mod mqtt;
|
||||
pub mod path_safety;
|
||||
pub mod semantic;
|
||||
/// ADR-262 P3: the live RuField surface — turns the governed sensing cycle into
|
||||
/// signed RuField `FieldEvent`s on the additive `/api/field` + `/ws/field`
|
||||
/// endpoints, via the `wifi-densepose-rufield` anti-corruption bridge.
|
||||
pub mod rufield_surface;
|
||||
pub mod rvf_container;
|
||||
pub mod rvf_pipeline;
|
||||
pub mod sona;
|
||||
|
||||
@@ -26,7 +26,7 @@ mod vital_signs;
|
||||
|
||||
// Training pipeline modules (exposed via lib.rs)
|
||||
use wifi_densepose_sensing_server::{
|
||||
dataset, embedding, error_response, graph_transformer, trainer,
|
||||
dataset, embedding, error_response, graph_transformer, rufield_surface, trainer,
|
||||
};
|
||||
|
||||
use ruvector_mincut::{DynamicMinCut, MinCutBuilder};
|
||||
@@ -1093,6 +1093,14 @@ struct AppStateInner {
|
||||
pub(crate) dedup_factor: f64,
|
||||
/// Data directory for persisting runtime config (parent of `firmware_dir`).
|
||||
pub(crate) data_dir: std::path::PathBuf,
|
||||
/// ADR-262 P3: the live RuField surface. Holds the dedicated ed25519 signer
|
||||
/// + a bounded ring of recent signed `FieldEvent`s + the `/ws/field`
|
||||
/// broadcast topic. The governed sensing cycle calls `emit()` on it once per
|
||||
/// cycle (joining `SensingUpdate` features/classification/signal_field with
|
||||
/// the `TrustedOutput` trust class); `/api/field` + `/ws/field` read it.
|
||||
/// Held behind its own `Arc<RwLock<_>>` so the additive field router can
|
||||
/// take it as state without re-locking `AppStateInner`.
|
||||
field_surface: rufield_surface::FieldState,
|
||||
}
|
||||
|
||||
/// If no ESP32 frame arrives within this duration, source reverts to offline.
|
||||
@@ -4000,6 +4008,80 @@ fn derive_single_person_pose(
|
||||
/// the strongest peak so they remain co-located with real energy rather than at
|
||||
/// a fake origin; if the field has no peak above threshold the position stays at
|
||||
/// `[0,0,0]` and `motion_score` still reflects real motion power.
|
||||
/// ADR-262 P3: emit one signed RuField `FieldEvent` for this sensing cycle.
|
||||
///
|
||||
/// Joins the cycle's [`SensingUpdate`] (features / classification /
|
||||
/// signal_field) with the governed engine's trust state (`effective_class` /
|
||||
/// `demoted`, recorded on `engine_bridge` by `observe_cycle`) into a
|
||||
/// `SensingSnapshot`, then surfaces it via the P1 bridge on `/api/field` +
|
||||
/// `/ws/field`. The bridge maps privacy by information content and the surface
|
||||
/// applies the §10 network egress gate, so above-policy cycles never reach the
|
||||
/// wire.
|
||||
///
|
||||
/// **No phantom events:** an empty/no-presence cycle (`presence == false`)
|
||||
/// emits nothing — there is no person to describe, so no event is fabricated
|
||||
/// (ADR-262 §4 P3 / §6). Cycles before the governed engine has produced a trust
|
||||
/// class are likewise skipped (no class ⇒ nothing honest to stamp).
|
||||
///
|
||||
/// `identity_bound` is `false` on the live path: RuView's live cycle does not
|
||||
/// bind an enrolled identity to the surface yet (that is a per-room-calibration
|
||||
/// / AETHER concern, ADR-262 §8 Q4). This is conservative for egress — it only
|
||||
/// ever *lowers* a Derived cycle from P5 to P4, both of which are already held
|
||||
/// edge-local, so it cannot leak.
|
||||
fn emit_rufield_event(s: &AppStateInner, update: &SensingUpdate, node_id: u8) {
|
||||
// No-presence ⇒ no phantom event.
|
||||
if !update.classification.presence {
|
||||
return;
|
||||
}
|
||||
// Need a governed trust class before we can honestly stamp privacy.
|
||||
let Some(effective_class) = s.engine_bridge.effective_class() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let timestamp_ns = if update.timestamp.is_finite() && update.timestamp > 0.0 {
|
||||
(update.timestamp * 1_000_000_000.0) as u64
|
||||
} else {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as u64)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let snap = rufield_surface::build_snapshot(
|
||||
timestamp_ns,
|
||||
format!("esp32_node_{node_id}"),
|
||||
rufield_surface::SensingFeatures {
|
||||
mean_rssi: update.features.mean_rssi,
|
||||
variance: update.features.variance,
|
||||
motion_band_power: update.features.motion_band_power,
|
||||
breathing_band_power: update.features.breathing_band_power,
|
||||
dominant_freq_hz: update.features.dominant_freq_hz,
|
||||
change_points: update.features.change_points,
|
||||
spectral_power: update.features.spectral_power,
|
||||
},
|
||||
rufield_surface::SensingClass {
|
||||
motion_level: update.classification.motion_level.clone(),
|
||||
presence: update.classification.presence,
|
||||
confidence: update.classification.confidence,
|
||||
},
|
||||
Some(rufield_surface::SignalField {
|
||||
grid_size: update.signal_field.grid_size,
|
||||
values: update.signal_field.values.clone(),
|
||||
}),
|
||||
rufield_surface::ruview_class_from_bfld(effective_class),
|
||||
s.engine_bridge.demoted(),
|
||||
false, // identity_bound — see fn-doc (conservative, cannot leak).
|
||||
);
|
||||
|
||||
// `field_surface` is its own Arc<RwLock<_>>; `try_write` is non-blocking and
|
||||
// never deadlocks against the `s` guard (a different lock). The only other
|
||||
// touchers are the read-only `/api/field` / `/ws/field` handlers, so
|
||||
// contention is negligible; a rare miss just drops one cycle's event.
|
||||
if let Ok(mut fs) = s.field_surface.try_write() {
|
||||
fs.emit(&snap);
|
||||
}
|
||||
}
|
||||
|
||||
fn attach_field_positions(update: &mut SensingUpdate) {
|
||||
let Some(persons) = update.persons.as_mut() else {
|
||||
return;
|
||||
@@ -5990,6 +6072,18 @@ async fn udp_receiver_task(state: SharedState, udp_port: u16) {
|
||||
if let Ok(json) = serde_json::to_string(&update) {
|
||||
let _ = s.tx.send(json);
|
||||
}
|
||||
|
||||
// ── ADR-262 P3: emit a signed RuField FieldEvent ────────
|
||||
// Join this cycle's SensingUpdate (features / classification
|
||||
// / signal_field) with the governed engine's trust state
|
||||
// (effective_class / demoted, recorded by `observe_cycle`
|
||||
// above) into a `SensingSnapshot`, and surface it on
|
||||
// `/api/field` + `/ws/field` via the P1 bridge. Only cycles
|
||||
// whose mapped privacy class clears the §10 network egress
|
||||
// gate are surfaced (P1/P2); a `Derived → P4/P5` cycle is
|
||||
// held edge-local. `presence == false` ⇒ no phantom event.
|
||||
emit_rufield_event(&s, &update, node_id);
|
||||
|
||||
s.latest_update = Some(update);
|
||||
|
||||
// Evict stale nodes every 100 ticks to prevent memory leak.
|
||||
@@ -7322,6 +7416,13 @@ async fn main() {
|
||||
);
|
||||
}
|
||||
|
||||
// ADR-262 P3: build the live RuField surface (dedicated ed25519 signer from
|
||||
// WDP_RUFIELD_SIGNING_SEED, else a logged dev default). The same Arc is
|
||||
// stored in AppStateInner (so the sensing loop can `emit()` per cycle) and
|
||||
// cloned into the additive `/api/field` + `/ws/field` router below.
|
||||
let field_surface: rufield_surface::FieldState =
|
||||
Arc::new(RwLock::new(rufield_surface::FieldSurface::from_env()));
|
||||
|
||||
let state: SharedState = Arc::new(RwLock::new(AppStateInner {
|
||||
latest_update: None,
|
||||
rssi_history: VecDeque::new(),
|
||||
@@ -7424,6 +7525,7 @@ async fn main() {
|
||||
// ADR-044 §5.3: runtime-configurable dedup factor (persisted).
|
||||
dedup_factor: runtime_config.dedup_factor,
|
||||
data_dir: data_dir.clone(),
|
||||
field_surface: field_surface.clone(),
|
||||
}));
|
||||
|
||||
// Start background tasks from the resolved plan (issue #1004).
|
||||
@@ -7497,11 +7599,15 @@ async fn main() {
|
||||
let ws_app = Router::new()
|
||||
.route("/ws/sensing", get(ws_sensing_handler))
|
||||
.route("/health", get(health))
|
||||
.with_state(ws_state)
|
||||
// ADR-262 P3: additive `/ws/field` (+ `/api/field`) on the WS port too,
|
||||
// so a client on :8765 can stream signed RuField FieldEvents alongside
|
||||
// `/ws/sensing`. Merged with its own FieldState (different state type).
|
||||
.merge(rufield_surface::router(field_surface.clone()))
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
host_allowlist.clone(),
|
||||
wifi_densepose_sensing_server::host_validation::require_allowed_host,
|
||||
))
|
||||
.with_state(ws_state);
|
||||
));
|
||||
|
||||
let ws_addr = SocketAddr::from((bind_ip, args.ws_port));
|
||||
let ws_listener = tokio::net::TcpListener::bind(ws_addr)
|
||||
@@ -7615,15 +7721,24 @@ async fn main() {
|
||||
bearer_auth_state.clone(),
|
||||
wifi_densepose_sensing_server::bearer_auth::require_bearer,
|
||||
))
|
||||
.with_state(state.clone())
|
||||
// ADR-262 P3: additive RuField surface (`/api/field` + `/ws/field`).
|
||||
// Merged AFTER `.with_state` (so http_app is already `Router<()>` and
|
||||
// can absorb the field router's own `FieldState`). These routes sit
|
||||
// OUTSIDE `/api/v1/*` so they are not bearer-gated, but the
|
||||
// host-validation layer below still applies (it is added last, so it
|
||||
// runs first, over the whole merged router). The surface's own §10
|
||||
// egress gate is what keeps above-policy classes off the wire.
|
||||
.merge(rufield_surface::router(field_surface.clone()))
|
||||
// DNS-rebinding defense: applied last so it runs first on the request
|
||||
// path (axum layers run outermost-in). Rejects requests whose `Host`
|
||||
// header is not in the allowlist before any handler — including
|
||||
// `/health` and `/ws/*` — observes the body.
|
||||
// `/health`, `/ws/*`, and the merged `/api/field` + `/ws/field` —
|
||||
// observes the body.
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
host_allowlist.clone(),
|
||||
wifi_densepose_sensing_server::host_validation::require_allowed_host,
|
||||
))
|
||||
.with_state(state.clone());
|
||||
));
|
||||
|
||||
let http_addr = SocketAddr::from((bind_ip, args.http_port));
|
||||
let http_listener = tokio::net::TcpListener::bind(http_addr)
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
//! ADR-262 **P3** — the live RuField surface.
|
||||
//!
|
||||
//! This is the data-path wiring that turns RuView's governed sensing cycle into
|
||||
//! signed RuField [`FieldEvent`]s on two **additive** network endpoints:
|
||||
//!
|
||||
//! - `GET /api/field` — the most recent surfaced `FieldEvent`(s) as JSON;
|
||||
//! - `GET /ws/field` — a WebSocket that streams each cycle's `FieldEvent`
|
||||
//! (mirrors the `/ws/sensing` broadcast-subscribe pattern).
|
||||
//!
|
||||
//! It is purely additive: `/ws/sensing` and every existing endpoint are
|
||||
//! unchanged. The conversion itself lives entirely in the P1
|
||||
//! [`wifi_densepose_rufield`] anti-corruption bridge (ADR-262 §5.4 — the single
|
||||
//! coupling point); this module only (a) holds the dedicated signer + a bounded
|
||||
//! ring buffer of recent events in server state, (b) builds a
|
||||
//! [`SensingSnapshot`] from the **same real data** the cycle already produced
|
||||
//! (`SensingUpdate` features/classification/signal_field joined with the
|
||||
//! governed-engine [`TrustedOutput`] trust state at `main.rs:~5886`/`:~5938`),
|
||||
//! and (c) applies the §10 network egress gate so above-policy classes never
|
||||
//! reach the wire.
|
||||
//!
|
||||
//! ## Honesty (ADR-262 §0 / §6)
|
||||
//!
|
||||
//! This wires **real** RuView sensing into RuField events on a live endpoint,
|
||||
//! but: (a) it is the **single-link CSI** sensing with its existing caveats —
|
||||
//! there is **no validated room-coordinate accuracy** (`field_localize` says so;
|
||||
//! positions are "strongest field peak", not triangulation); (b) the signing
|
||||
//! key is a **dedicated dev/sensing key** pending the ADR-262 §8 Q1 ownership
|
||||
//! decision (reusing the `cog-ha-matter` Ed25519 key is the **deferred P2**
|
||||
//! call — P3 deliberately uses a standalone key so it does not pre-empt that);
|
||||
//! (c) **no accuracy is claimed.** The win is narrowly: "RuView's live sensing
|
||||
//! now speaks RuField on `/ws/field`."
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
State,
|
||||
},
|
||||
response::{IntoResponse, Json},
|
||||
};
|
||||
use tokio::sync::{broadcast, RwLock};
|
||||
|
||||
// Re-export the bridge input types `main.rs` needs to build a snapshot, so the
|
||||
// server-side call site depends only on `rufield_surface` (the server seam).
|
||||
pub use wifi_densepose_rufield::{
|
||||
network_egress_allowed, snapshot_to_field_event, FieldEvent, RuViewPrivacyClass,
|
||||
SensingClass, SensingFeatures, SensingSnapshot, Signer, SignalField,
|
||||
};
|
||||
|
||||
/// How many recent surfaced `FieldEvent`s the ring buffer retains. Small and
|
||||
/// bounded — this is a live tap, not a store (ADR-262 §4 P3 "small bounded ring
|
||||
/// buffer of recent events").
|
||||
pub const FIELD_RING_CAPACITY: usize = 64;
|
||||
|
||||
/// Broadcast channel depth for `/ws/field`. Matches the `/ws/sensing` `tx`
|
||||
/// channel size (256) so a slow field client drops messages rather than
|
||||
/// stalling the sensing loop.
|
||||
pub const FIELD_BROADCAST_CAPACITY: usize = 256;
|
||||
|
||||
/// Environment variable carrying the 32-byte hex/raw signing seed for the
|
||||
/// dedicated RuField sensing signer. When unset, a deterministic dev default is
|
||||
/// used (with a logged warning). See [`FieldSurface::from_env`].
|
||||
pub const SIGNING_SEED_ENV: &str = "WDP_RUFIELD_SIGNING_SEED";
|
||||
|
||||
/// Deterministic dev signing seed used when [`SIGNING_SEED_ENV`] is unset. This
|
||||
/// is a **dev/sensing key**, intentionally standalone (ADR-262 §8 Q1 — the
|
||||
/// `cog-ha-matter` key reuse is the deferred P2 decision, not pre-empted here).
|
||||
const DEV_SIGNING_SEED: &[u8; 32] = b"adr262-ruview-rufield-dev-seed!!";
|
||||
|
||||
/// The live RuField surface state held in `AppStateInner` (ADR-262 P3).
|
||||
///
|
||||
/// Owns the **dedicated** ed25519 [`Signer`], a bounded ring buffer of the most
|
||||
/// recent network-surfaced events, and the `/ws/field` broadcast sender.
|
||||
pub struct FieldSurface {
|
||||
signer: Signer,
|
||||
/// Bounded ring of recent **network-surfaced** events (most recent last).
|
||||
recent: VecDeque<FieldEvent>,
|
||||
/// Broadcast topic for `/ws/field` (JSON-serialized `FieldEvent`s).
|
||||
tx: broadcast::Sender<String>,
|
||||
/// True when the dev default seed is in use (drives a one-time warning and
|
||||
/// is surfaced in `/api/field` metadata so operators can see they are on a
|
||||
/// dev key).
|
||||
using_dev_key: bool,
|
||||
}
|
||||
|
||||
impl FieldSurface {
|
||||
/// Build a surface with an explicit 32-byte seed (deterministic signer).
|
||||
#[must_use]
|
||||
pub fn from_seed(seed: &[u8; 32], using_dev_key: bool) -> Self {
|
||||
let (tx, _rx) = broadcast::channel(FIELD_BROADCAST_CAPACITY);
|
||||
Self {
|
||||
signer: Signer::from_seed(seed),
|
||||
recent: VecDeque::with_capacity(FIELD_RING_CAPACITY),
|
||||
tx,
|
||||
using_dev_key,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a surface from the environment (ADR-262 §4 P3 / open-question 1).
|
||||
///
|
||||
/// Reads [`SIGNING_SEED_ENV`] as either a 64-char hex string or a raw 32+
|
||||
/// byte UTF-8 value (first 32 bytes used). When unset/invalid it falls back
|
||||
/// to the deterministic [`DEV_SIGNING_SEED`] and logs a `WARN` — the key is
|
||||
/// a standalone **dev/sensing** key, NOT the deferred-P2 `cog-ha-matter`
|
||||
/// key.
|
||||
#[must_use]
|
||||
pub fn from_env() -> Self {
|
||||
match std::env::var(SIGNING_SEED_ENV).ok().and_then(|v| parse_seed(&v)) {
|
||||
Some(seed) => {
|
||||
tracing::info!(
|
||||
"ADR-262 P3: RuField surface using signing seed from {SIGNING_SEED_ENV} \
|
||||
(dedicated sensing key)"
|
||||
);
|
||||
Self::from_seed(&seed, false)
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"ADR-262 P3: {SIGNING_SEED_ENV} unset/invalid — RuField surface using the \
|
||||
DETERMINISTIC DEV signing key. This is a dev/sensing key pending the \
|
||||
ADR-262 §8 Q1 (P2) key-ownership decision; set {SIGNING_SEED_ENV} (64-hex \
|
||||
or 32-byte value) for a real deployment."
|
||||
);
|
||||
Self::from_seed(DEV_SIGNING_SEED, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The public key of the dedicated signer (hex), so consumers can verify
|
||||
/// receipts without the private seed.
|
||||
#[must_use]
|
||||
pub fn signer_pubkey_hex(&self) -> String {
|
||||
self.signer.public_hex()
|
||||
}
|
||||
|
||||
/// Whether the dev default key is in use.
|
||||
#[must_use]
|
||||
pub fn using_dev_key(&self) -> bool {
|
||||
self.using_dev_key
|
||||
}
|
||||
|
||||
/// A `/ws/field` subscription.
|
||||
#[must_use]
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<String> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
|
||||
/// The most recent surfaced events, oldest→newest.
|
||||
#[must_use]
|
||||
pub fn recent(&self) -> Vec<FieldEvent> {
|
||||
self.recent.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Convert one cycle's [`SensingSnapshot`] into a signed [`FieldEvent`],
|
||||
/// apply the §10 network egress gate, and — **iff** the event may leave the
|
||||
/// box — push it into the ring + broadcast it on `/ws/field`.
|
||||
///
|
||||
/// Returns `Some(event)` when an event was surfaced, `None` when the cycle
|
||||
/// was held edge-local (above network policy — e.g. a `Derived → P4/P5`
|
||||
/// cycle) or carried no presence. Two structural guarantees live here, so
|
||||
/// they hold regardless of caller:
|
||||
///
|
||||
/// - **no phantom events** — a no-presence cycle (`presence == false`)
|
||||
/// surfaces nothing (ADR-262 §4 P3 / §6); there is no person to describe.
|
||||
/// - **privacy-safety pin** — above-policy classes (P0, P3–P5) are never
|
||||
/// placed on the network surface; only egress-safe P1/P2 events leave.
|
||||
pub fn emit(&mut self, snap: &SensingSnapshot) -> Option<FieldEvent> {
|
||||
// No-presence ⇒ no phantom event (fabricating one would be dishonest).
|
||||
if !snap.classification.presence {
|
||||
return None;
|
||||
}
|
||||
|
||||
let event = snapshot_to_field_event(snap, &self.signer);
|
||||
|
||||
// §10 network egress gate (ADR-262 §4 P3): only P1/P2 leave the box by
|
||||
// default; P0 raw and P3/P4/P5 (above the default P2 ceiling, or
|
||||
// identity/biometric) are held edge-local. A `Derived` cycle is P4/P5
|
||||
// ⇒ never surfaced as a low-privacy network event.
|
||||
if !network_egress_allowed(event.observation.privacy_class, snap.identity_bound) {
|
||||
tracing::trace!(
|
||||
privacy_class = ?event.observation.privacy_class,
|
||||
"ADR-262 P3: cycle held edge-local (above network policy), not surfaced on /api/field"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
if self.recent.len() == FIELD_RING_CAPACITY {
|
||||
self.recent.pop_front();
|
||||
}
|
||||
self.recent.push_back(event.clone());
|
||||
|
||||
if let Ok(json) = serde_json::to_string(&event) {
|
||||
let _ = self.tx.send(json);
|
||||
}
|
||||
Some(event)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse [`SIGNING_SEED_ENV`] as 64-char hex or a raw 32+ byte UTF-8 value.
|
||||
fn parse_seed(v: &str) -> Option<[u8; 32]> {
|
||||
let v = v.trim();
|
||||
// 64 hex chars → 32 bytes.
|
||||
if v.len() == 64 && v.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
let mut out = [0u8; 32];
|
||||
for (i, chunk) in v.as_bytes().chunks(2).enumerate() {
|
||||
let hi = (chunk[0] as char).to_digit(16)?;
|
||||
let lo = (chunk[1] as char).to_digit(16)?;
|
||||
out[i] = ((hi << 4) | lo) as u8;
|
||||
}
|
||||
return Some(out);
|
||||
}
|
||||
// Otherwise: first 32 bytes of the raw value (must be at least 32 long so a
|
||||
// short/typo'd value fails closed to the dev key rather than a weak key).
|
||||
let bytes = v.as_bytes();
|
||||
if bytes.len() >= 32 {
|
||||
let mut out = [0u8; 32];
|
||||
out.copy_from_slice(&bytes[..32]);
|
||||
return Some(out);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Build a [`SensingSnapshot`] from the real per-cycle values (ADR-262 P3 §4.2).
|
||||
///
|
||||
/// This is the join the ADR mandates: `SensingUpdate` features / classification
|
||||
/// / signal-field **plus** the governed engine's `effective_class` / `demoted`
|
||||
/// / `identity_bound` trust state. All inputs are the same real data the cycle
|
||||
/// already computed — nothing is fabricated. `signal_field` is passed through as
|
||||
/// the honest "strongest field peak" readout (no calibrated coordinates).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[must_use]
|
||||
pub fn build_snapshot(
|
||||
timestamp_ns: u64,
|
||||
node_id: String,
|
||||
features: SensingFeatures,
|
||||
classification: SensingClass,
|
||||
signal_field: Option<SignalField>,
|
||||
trust_class: RuViewPrivacyClass,
|
||||
demoted: bool,
|
||||
identity_bound: bool,
|
||||
) -> SensingSnapshot {
|
||||
SensingSnapshot {
|
||||
timestamp_ns,
|
||||
features,
|
||||
classification,
|
||||
signal_field,
|
||||
trust_class,
|
||||
demoted,
|
||||
identity_bound,
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map RuView's live governed-engine `bfld::PrivacyClass` (the `effective_class`
|
||||
/// on `TrustedOutput`) onto the bridge's [`RuViewPrivacyClass`] input.
|
||||
///
|
||||
/// This is a **lossless, same-meaning** re-encoding of the four byte-level
|
||||
/// classes — both enums are `Raw/Derived/Anonymous/Restricted` in the same
|
||||
/// order. It exists only so `main.rs` can pass the engine's class into the
|
||||
/// bridge without the bridge depending on `wifi-densepose-bfld` (keeping it an
|
||||
/// anti-corruption layer, ADR-262 §5.4). The information-content privacy
|
||||
/// mapping (the §3.3 correctness item) happens *inside* the bridge.
|
||||
#[must_use]
|
||||
pub fn ruview_class_from_bfld(class: wifi_densepose_bfld::PrivacyClass) -> RuViewPrivacyClass {
|
||||
use wifi_densepose_bfld::PrivacyClass as B;
|
||||
match class {
|
||||
B::Raw => RuViewPrivacyClass::Raw,
|
||||
B::Derived => RuViewPrivacyClass::Derived,
|
||||
B::Anonymous => RuViewPrivacyClass::Anonymous,
|
||||
B::Restricted => RuViewPrivacyClass::Restricted,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Handlers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Shared state for the field surface handlers. Generic over the lock guard so
|
||||
/// the module can be tested in isolation with a tiny state (ADR-262 P3 test
|
||||
/// gate) and wired into the full `AppStateInner` in `main.rs` via an adapter.
|
||||
pub type FieldState = Arc<RwLock<FieldSurface>>;
|
||||
|
||||
/// `GET /api/field` — the most recent network-surfaced `FieldEvent`s as JSON,
|
||||
/// plus surface metadata (the signer pubkey + whether a dev key is in use).
|
||||
///
|
||||
/// When no event has been surfaced yet (empty room / above-policy cycles only)
|
||||
/// the `events` array is empty — an **explicit empty payload**, never a
|
||||
/// fabricated event (ADR-262 §4 P3 / §6 honesty).
|
||||
pub async fn api_field(State(state): State<FieldState>) -> Json<serde_json::Value> {
|
||||
let s = state.read().await;
|
||||
Json(serde_json::json!({
|
||||
"spec": "rufield",
|
||||
"endpoint": "/api/field",
|
||||
"signer_pubkey_hex": s.signer_pubkey_hex(),
|
||||
"dev_signing_key": s.using_dev_key(),
|
||||
"events": s.recent(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// `GET /ws/field` — upgrade to a WebSocket that streams each surfaced
|
||||
/// `FieldEvent` (JSON) as the sensing loop emits it. Mirrors `/ws/sensing`:
|
||||
/// subscribe to the broadcast topic and forward.
|
||||
pub async fn ws_field(ws: WebSocketUpgrade, State(state): State<FieldState>) -> impl IntoResponse {
|
||||
let rx = {
|
||||
let s = state.read().await;
|
||||
s.subscribe()
|
||||
};
|
||||
ws.on_upgrade(move |socket| handle_ws_field_client(socket, rx))
|
||||
}
|
||||
|
||||
async fn handle_ws_field_client(mut socket: WebSocket, mut rx: broadcast::Receiver<String>) {
|
||||
// Forward broadcast events; exit on client close or fatal lag.
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(json) => {
|
||||
if socket.send(Message::Text(json)).await.is_err() {
|
||||
break; // client gone
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||
// Slow client missed events — keep going from the latest.
|
||||
continue;
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the additive field-surface router. Mounted into the main HTTP router
|
||||
/// in `main.rs`; also used standalone by the integration tests (ADR-262 P3
|
||||
/// gate, `tower::oneshot`).
|
||||
#[must_use]
|
||||
pub fn router(state: FieldState) -> axum::Router {
|
||||
use axum::routing::get;
|
||||
axum::Router::new()
|
||||
.route("/api/field", get(api_field))
|
||||
.route("/ws/field", get(ws_field))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use wifi_densepose_rufield::{is_fusable, PrivacyClass};
|
||||
|
||||
fn features() -> SensingFeatures {
|
||||
SensingFeatures {
|
||||
mean_rssi: -55.0,
|
||||
variance: 0.4,
|
||||
motion_band_power: 2.0,
|
||||
breathing_band_power: 0.3,
|
||||
dominant_freq_hz: 0.25,
|
||||
change_points: 1,
|
||||
spectral_power: 3.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn present_class() -> SensingClass {
|
||||
SensingClass {
|
||||
motion_level: "low".into(),
|
||||
presence: true,
|
||||
confidence: 0.82,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_seed_hex_and_raw_and_short() {
|
||||
// 64 hex chars → 32 bytes.
|
||||
let hex = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff";
|
||||
let parsed = parse_seed(hex).expect("valid hex seed");
|
||||
assert_eq!(parsed[0], 0x00);
|
||||
assert_eq!(parsed[31], 0xff);
|
||||
// Raw 32-byte value.
|
||||
assert!(parse_seed("0123456789abcdef0123456789abcdef").is_some());
|
||||
// Too short → fail closed (None → dev key).
|
||||
assert!(parse_seed("short").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anonymous_cycle_surfaces_fusable_event() {
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Anonymous, // → P2, network-allowed
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let ev = surface.emit(&snap).expect("anonymous P2 cycle is surfaced");
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert!(is_fusable(&ev), "live event must be ed25519-signed & fusable");
|
||||
assert_eq!(surface.recent().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_cycle_never_surfaces_low_privacy() {
|
||||
// The privacy-safety pin: a Derived (identity) cycle maps to P4/P5 and
|
||||
// is held edge-local — it must NEVER appear on the network surface.
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
for identity_bound in [false, true] {
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Derived,
|
||||
false,
|
||||
identity_bound,
|
||||
);
|
||||
assert!(
|
||||
surface.emit(&snap).is_none(),
|
||||
"Derived cycle (identity_bound={identity_bound}) must be held edge-local"
|
||||
);
|
||||
}
|
||||
assert!(surface.recent().is_empty(), "no Derived event may reach the surface");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ring_buffer_is_bounded() {
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
for i in 0..(FIELD_RING_CAPACITY + 10) {
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000 + i as u64,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Anonymous,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
surface.emit(&snap);
|
||||
}
|
||||
assert_eq!(surface.recent().len(), FIELD_RING_CAPACITY);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! ADR-262 **P3** acceptance gate — the live RuField surface.
|
||||
//!
|
||||
//! In-process integration test (mirrors the `/ws/sensing` / #1050 oneshot
|
||||
//! style with `tower::ServiceExt::oneshot`): drives synthetic sensing cycles
|
||||
//! through the real `FieldSurface` + the real `/api/field` router, and asserts:
|
||||
//!
|
||||
//! 1. an injected `Anonymous` (occupancy) cycle surfaces a **well-formed signed
|
||||
//! `FieldEvent`** — `Modality::WifiCsi`, privacy class consistent with the
|
||||
//! trust (P2, never P1), `is_fusable` (ed25519 receipt verifies), real
|
||||
//! timestamp;
|
||||
//! 2. an empty / no-presence cycle produces **no phantom event** (explicit
|
||||
//! empty payload);
|
||||
//! 3. the **privacy-safety pin** — an injected `Derived` (identity) trust state
|
||||
//! never surfaces as a low-privacy event on `/api/field` (held edge-local).
|
||||
//!
|
||||
//! These gates are plumbing + privacy-safety, NOT accuracy (ADR-262 §0 / §6).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use tokio::sync::RwLock;
|
||||
use tower::ServiceExt; // `oneshot`
|
||||
|
||||
use wifi_densepose_rufield::{is_fusable, verify_event, FieldEvent, Modality, PrivacyClass};
|
||||
use wifi_densepose_sensing_server::rufield_surface::{
|
||||
self, FieldState, FieldSurface, RuViewPrivacyClass, SensingClass, SensingFeatures, SignalField,
|
||||
};
|
||||
|
||||
/// A fixed dev seed for deterministic, signed events under test.
|
||||
const TEST_SEED: &[u8; 32] = b"adr262-p3-integration-test-seed!";
|
||||
|
||||
fn features() -> SensingFeatures {
|
||||
SensingFeatures {
|
||||
mean_rssi: -55.0,
|
||||
variance: 0.4,
|
||||
motion_band_power: 2.0,
|
||||
breathing_band_power: 0.3,
|
||||
dominant_freq_hz: 0.25,
|
||||
change_points: 1,
|
||||
spectral_power: 3.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn class(presence: bool) -> SensingClass {
|
||||
SensingClass {
|
||||
motion_level: if presence { "low".into() } else { "none".into() },
|
||||
presence,
|
||||
confidence: if presence { 0.82 } else { 0.05 },
|
||||
}
|
||||
}
|
||||
|
||||
/// A small 2×1×2 signal field with a clear peak, so the bridge derives a real
|
||||
/// (non-fabricated) position from the strongest cell.
|
||||
fn signal_field() -> SignalField {
|
||||
SignalField {
|
||||
grid_size: [2, 1, 2],
|
||||
values: vec![0.1, 0.2, 0.9, 0.3], // peak at index 2
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a `FieldState` + the real `/api/field` + `/ws/field` router over it.
|
||||
fn surface_router() -> (FieldState, axum::Router) {
|
||||
let state: FieldState = Arc::new(RwLock::new(FieldSurface::from_seed(TEST_SEED, true)));
|
||||
let app = rufield_surface::router(state.clone());
|
||||
(state, app)
|
||||
}
|
||||
|
||||
/// Drive one cycle into the surface (the in-process equivalent of the live
|
||||
/// sensing loop calling `emit()` per cycle).
|
||||
async fn inject(state: &FieldState, trust: RuViewPrivacyClass, presence: bool, identity_bound: bool) {
|
||||
let snap = rufield_surface::build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_node_7".into(),
|
||||
features(),
|
||||
class(presence),
|
||||
Some(signal_field()),
|
||||
trust,
|
||||
false, // demoted
|
||||
identity_bound,
|
||||
);
|
||||
state.write().await.emit(&snap);
|
||||
}
|
||||
|
||||
/// `GET /api/field` and parse the `events` array.
|
||||
async fn get_field_events(app: &axum::Router) -> Vec<FieldEvent> {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/field")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "/api/field must return 200");
|
||||
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap();
|
||||
let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(v["spec"], "rufield");
|
||||
serde_json::from_value(v["events"].clone()).expect("events array deserializes to FieldEvents")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_anonymous_cycle_surfaces_wellformed_signed_event() {
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert_eq!(events.len(), 1, "one occupancy cycle ⇒ exactly one surfaced event");
|
||||
let ev = &events[0];
|
||||
|
||||
// Well-formed: WiFi-CSI modality, real timestamp.
|
||||
assert_eq!(ev.tensor.modality, Modality::WifiCsi);
|
||||
assert_eq!(ev.timestamp_ns, 1_791_986_400_000_000_000);
|
||||
assert!(ev.timestamp_ns > 0, "real (non-zero) timestamp");
|
||||
|
||||
// Privacy consistent with the injected trust: Anonymous → P2, NEVER P1.
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert_ne!(ev.observation.privacy_class, PrivacyClass::P1);
|
||||
|
||||
// Signed + fusable: the ed25519 receipt verifies (real, non-synthetic).
|
||||
assert!(!ev.provenance.synthetic, "live event is non-synthetic");
|
||||
assert!(verify_event(ev).is_ok(), "ed25519 signature must verify");
|
||||
assert!(is_fusable(ev), "verified receipt ⇒ fusable");
|
||||
|
||||
// Real position derived from the signal-field peak (not fabricated).
|
||||
assert!(ev.observation.range_m.is_some(), "field peak ⇒ a real range readout");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_empty_cycle_produces_no_phantom_event() {
|
||||
let (state, app) = surface_router();
|
||||
// A no-presence cycle: nothing to describe.
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, false, false).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert!(
|
||||
events.is_empty(),
|
||||
"no-presence cycle must surface no phantom event (explicit empty payload)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_derived_trust_never_surfaces_low_privacy() {
|
||||
// The privacy-safety pin (ADR-262 §3.3 / §6): a Derived (identity) trust
|
||||
// state maps to P4/P5 and is held edge-local — it must NEVER appear on the
|
||||
// network surface, and certainly never as a low-privacy (P1/P2) event.
|
||||
for identity_bound in [false, true] {
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, identity_bound).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert!(
|
||||
events.is_empty(),
|
||||
"Derived cycle (identity_bound={identity_bound}) must not surface on /api/field"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_mixed_stream_surfaces_only_egress_safe_events() {
|
||||
// Determinism / privacy-safety over a stream: Anonymous cycles surface,
|
||||
// interleaved Derived cycles are dropped — the surface only ever carries
|
||||
// egress-safe (P1/P2) events.
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await; // P2 → surfaced
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, false).await; // P4 → dropped
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await; // P2 → surfaced
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, true).await; // P5 → dropped
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert_eq!(events.len(), 2, "only the two Anonymous cycles surface");
|
||||
for ev in &events {
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert!(is_fusable(ev));
|
||||
}
|
||||
}
|
||||
@@ -174,6 +174,20 @@ impl BreathingExtractor {
|
||||
let output =
|
||||
(1.0 - r) * (input - state.x2) + 2.0 * r * cos_w0 * state.y1 - r * r * state.y2;
|
||||
|
||||
// Self-healing non-finite guard (ADR-158 §A1). A single non-finite
|
||||
// sample — a NaN/inf residual from a corrupt CSI frame, or a transient
|
||||
// overflow — would otherwise be stored into `y1`/`y2` and poison the
|
||||
// resonator recurrence *permanently*: every subsequent output stays
|
||||
// NaN, the `extract()` finite-check drops it, and the history buffer
|
||||
// never refills, so breathing extraction is dead until `reset()`.
|
||||
// Resetting the filter state here lets the resonator recover on the next
|
||||
// clean frame; the 0.0 we return for this frame is still dropped by the
|
||||
// caller's `is_finite()` check, so no spurious sample enters history.
|
||||
if !output.is_finite() {
|
||||
*state = IirState::default();
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
state.x2 = state.x1;
|
||||
state.x1 = input;
|
||||
state.y2 = state.y1;
|
||||
@@ -396,6 +410,75 @@ mod tests {
|
||||
assert!((0.0..=2.0).contains(&fused), "weighted average must be in-range: {fused}");
|
||||
}
|
||||
|
||||
/// ADR-158 §A1 bug-catching test: a single non-finite residual must NOT
|
||||
/// permanently poison the IIR filter state.
|
||||
///
|
||||
/// The resonator recurrence stores `y[n]` into the filter state. Before the
|
||||
/// fix, one NaN/inf residual produced a NaN `output`, the `extract()`
|
||||
/// finite-guard dropped that frame from history — but the NaN was already
|
||||
/// latched into `state.y1`/`y2`, so every subsequent output stayed NaN, the
|
||||
/// finite-guard rejected it too, and the history buffer never refilled.
|
||||
/// Breathing extraction was then dead until `reset()`. A control run on the
|
||||
/// same clean signal yields 15 BPM (0.25 Hz); after a leading NaN frame the
|
||||
/// OLD code returned `None` with `history_len() == 0` forever. This test
|
||||
/// asserts recovery (FAILS on the old code, verified by reverting the
|
||||
/// `bandpass_filter` self-heal).
|
||||
#[test]
|
||||
fn nan_frame_does_not_permanently_poison_filter() {
|
||||
let sr = 10.0;
|
||||
let feed_clean = |ext: &mut BreathingExtractor| {
|
||||
let mut last = None;
|
||||
for i in 0..600 {
|
||||
let t = i as f64 / sr;
|
||||
let s = (2.0 * std::f64::consts::PI * 0.25 * t).sin();
|
||||
last = ext.extract(&[s], &[1.0]);
|
||||
}
|
||||
last
|
||||
};
|
||||
|
||||
// Control: clean signal accumulates history and detects ~15 BPM.
|
||||
let mut control = BreathingExtractor::new(1, sr, 60.0);
|
||||
let control_res = feed_clean(&mut control);
|
||||
assert!(control.history_len() > 0);
|
||||
assert!(control_res.is_some(), "control clean run must produce an estimate");
|
||||
|
||||
// A leading NaN frame must not kill the extractor.
|
||||
let mut ext = BreathingExtractor::new(1, sr, 60.0);
|
||||
ext.extract(&[f64::NAN], &[1.0]);
|
||||
let res = feed_clean(&mut ext);
|
||||
assert!(
|
||||
ext.history_len() > 0,
|
||||
"extractor must recover and refill history after a NaN frame (got {})",
|
||||
ext.history_len()
|
||||
);
|
||||
assert!(res.is_some(), "extractor must recover an estimate after a NaN frame");
|
||||
}
|
||||
|
||||
/// ADR-158 §A1: a mid-stream `inf` must not freeze the history buffer.
|
||||
#[test]
|
||||
fn inf_mid_stream_does_not_freeze_history() {
|
||||
let sr = 10.0;
|
||||
let mut ext = BreathingExtractor::new(1, sr, 60.0);
|
||||
let clean = |ext: &mut BreathingExtractor, count: usize| {
|
||||
for i in 0..count {
|
||||
let t = i as f64 / sr;
|
||||
let s = (2.0 * std::f64::consts::PI * 0.25 * t).sin();
|
||||
ext.extract(&[s], &[1.0]);
|
||||
}
|
||||
};
|
||||
clean(&mut ext, 300);
|
||||
let before = ext.history_len();
|
||||
assert!(before > 0);
|
||||
ext.extract(&[f64::INFINITY], &[1.0]); // poison mid-stream
|
||||
clean(&mut ext, 600);
|
||||
assert!(
|
||||
ext.history_len() > before,
|
||||
"history must keep growing after an inf frame (before={}, after={})",
|
||||
before,
|
||||
ext.history_len()
|
||||
);
|
||||
}
|
||||
|
||||
/// ADR-157 §A3 bug-catching test. Divergence needs the pole magnitude
|
||||
/// `|r| >= 1`, i.e. `bw >= 4`. At `fs = 0.5` Hz with the band widened to
|
||||
/// 0.1-0.9 Hz, `bw = 2*pi*(0.9-0.1)/0.5 = 10.05`, so the OLD pole radius
|
||||
|
||||
@@ -32,6 +32,15 @@ impl Default for IirState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowest physiologically plausible heart rate, in BPM. Estimates below this
|
||||
/// (e.g. a lock onto a breathing harmonic, which the firmware #987 fix also
|
||||
/// guards against) are rejected rather than emitted as a confident vital — a
|
||||
/// false low HR is a safety problem. Value-identical to the prior literal.
|
||||
const HR_PLAUSIBLE_MIN_BPM: f64 = 40.0;
|
||||
/// Highest physiologically plausible heart rate, in BPM. Estimates above this
|
||||
/// are rejected. Value-identical to the prior literal.
|
||||
const HR_PLAUSIBLE_MAX_BPM: f64 = 180.0;
|
||||
|
||||
/// Heart rate extractor using bandpass filtering and autocorrelation
|
||||
/// peak detection.
|
||||
pub struct HeartRateExtractor {
|
||||
@@ -140,8 +149,11 @@ impl HeartRateExtractor {
|
||||
let frequency_hz = self.sample_rate / period_samples as f64;
|
||||
let bpm = frequency_hz * 60.0;
|
||||
|
||||
// Validate BPM is in physiological range (40-180 BPM)
|
||||
if !(40.0..=180.0).contains(&bpm) {
|
||||
// Validate BPM is in the physiological plausibility band. An estimate
|
||||
// outside [HR_PLAUSIBLE_MIN_BPM, HR_PLAUSIBLE_MAX_BPM] is rejected
|
||||
// rather than emitted, so an out-of-band autocorrelation lock can never
|
||||
// surface as a confident heart rate.
|
||||
if !(HR_PLAUSIBLE_MIN_BPM..=HR_PLAUSIBLE_MAX_BPM).contains(&bpm) {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -191,6 +203,20 @@ impl HeartRateExtractor {
|
||||
let output =
|
||||
(1.0 - r) * (input - state.x2) + 2.0 * r * cos_w0 * state.y1 - r * r * state.y2;
|
||||
|
||||
// Self-healing non-finite guard (ADR-158 §A1). A single non-finite
|
||||
// sample — a NaN/inf residual from a corrupt CSI frame, or a transient
|
||||
// overflow — would otherwise be written into `y1`/`y2` and poison the
|
||||
// resonator recurrence *permanently*: every later output stays NaN, the
|
||||
// `extract()` finite-check drops it, `acf0` never recomputes on fresh
|
||||
// data, and heart-rate extraction is dead until `reset()`. Resetting the
|
||||
// filter state here lets the resonator recover on the next clean frame;
|
||||
// the 0.0 returned for this frame is still dropped by the caller's
|
||||
// `is_finite()` check, so no spurious sample enters history.
|
||||
if !output.is_finite() {
|
||||
*state = IirState::default();
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
state.x2 = state.x1;
|
||||
state.x1 = input;
|
||||
state.y2 = state.y1;
|
||||
@@ -420,6 +446,92 @@ mod tests {
|
||||
assert_eq!(ext.n_subcarriers, 56);
|
||||
}
|
||||
|
||||
/// Pin the physiological plausibility band to its documented values. If a
|
||||
/// future edit widens these, an implausible HR could be emitted as a
|
||||
/// confident vital — this characterization test forces that to be a
|
||||
/// deliberate, reviewed change.
|
||||
#[test]
|
||||
fn plausibility_band_constants_pinned() {
|
||||
assert!((HR_PLAUSIBLE_MIN_BPM - 40.0).abs() < f64::EPSILON);
|
||||
assert!((HR_PLAUSIBLE_MAX_BPM - 180.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
/// ADR-158 §A1 bug-catching test: a single non-finite residual must NOT
|
||||
/// permanently poison the IIR filter state.
|
||||
///
|
||||
/// The cardiac resonator latches `y[n]` into `state.y1`/`y2`. Before the
|
||||
/// fix, one NaN/inf residual produced a NaN `output` that was stored into
|
||||
/// the state; the `extract()` finite-guard dropped that frame from history,
|
||||
/// but every subsequent output stayed NaN, so the history buffer never
|
||||
/// refilled and HR extraction was dead until `reset()`. After a leading NaN
|
||||
/// frame, the OLD code returned `None` with `history_len() == 0` forever.
|
||||
/// This asserts recovery (FAILS on the old code).
|
||||
#[test]
|
||||
fn nan_frame_does_not_permanently_poison_filter() {
|
||||
let sr = 50.0;
|
||||
let feed_clean = |ext: &mut HeartRateExtractor| {
|
||||
let mut last = None;
|
||||
for i in 0..1200 {
|
||||
let t = i as f64 / sr;
|
||||
let base = (2.0 * std::f64::consts::PI * 1.2 * t).sin();
|
||||
let r = vec![base * 0.1, base * 0.08, base * 0.12, base * 0.09];
|
||||
last = ext.extract(&r, &[0.0, 0.01, 0.02, 0.03]);
|
||||
}
|
||||
last
|
||||
};
|
||||
|
||||
let mut control = HeartRateExtractor::new(4, sr, 20.0);
|
||||
feed_clean(&mut control);
|
||||
assert!(control.history_len() > 0, "control clean run must accumulate history");
|
||||
|
||||
let mut ext = HeartRateExtractor::new(4, sr, 20.0);
|
||||
ext.extract(&[f64::NAN, 0.1, 0.1, 0.1], &[0.0, 0.01, 0.02, 0.03]);
|
||||
feed_clean(&mut ext);
|
||||
assert!(
|
||||
ext.history_len() > 0,
|
||||
"HR extractor must recover and refill history after a NaN frame (got {})",
|
||||
ext.history_len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Safety negative: pure broadband noise (no cardiac component) must NOT be
|
||||
/// reported as a clinically `Valid` heart rate. A false "HR = 72 bpm" on
|
||||
/// noise is a safety problem (false reassurance / false alert). The
|
||||
/// extractor may still emit a low-confidence guess, but its status must be
|
||||
/// `Degraded`/`Unreliable`, never `Valid`. Mirrors the honest-negative
|
||||
/// requirement in the review brief.
|
||||
#[test]
|
||||
fn pure_noise_is_never_reported_valid() {
|
||||
let mut seed: u64 = 0x1234_5678;
|
||||
let mut rng = || {
|
||||
seed = seed
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
((seed >> 33) as f64 / (1u64 << 31) as f64) - 1.0
|
||||
};
|
||||
let mut ext = HeartRateExtractor::new(8, 50.0, 20.0);
|
||||
let mut last = None;
|
||||
for _ in 0..1500 {
|
||||
let r: Vec<f64> = (0..8).map(|_| rng()).collect();
|
||||
let p: Vec<f64> = (0..8).map(|_| rng()).collect();
|
||||
last = ext.extract(&r, &p);
|
||||
}
|
||||
if let Some(est) = last {
|
||||
assert_ne!(
|
||||
est.status,
|
||||
VitalStatus::Valid,
|
||||
"pure noise must not yield a clinically Valid HR (bpm={}, conf={})",
|
||||
est.value_bpm,
|
||||
est.confidence
|
||||
);
|
||||
assert!(
|
||||
est.confidence < 0.6,
|
||||
"noise HR confidence must stay below the Valid cutoff: {}",
|
||||
est.confidence
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// ADR-157 §A3 bug-catching test.
|
||||
///
|
||||
/// Divergence needs the pole *magnitude* `|r| >= 1`, i.e. `bw >= 4`. With
|
||||
|
||||
Vendored
+1
-1
Submodule vendor/midstream updated: 8f70d2bb9d...92250c20d8
+1
Submodule vendor/rufield added at 509d8ae29e
Vendored
+1
-1
Submodule vendor/ruvector updated: a083bd77fa...abf8e0d6f0
Vendored
+1
-1
Submodule vendor/rvcsi updated: 72891d740f...77c8b6e051
Vendored
+1
-1
Submodule vendor/sublinear-time-solver updated: c25dddf163...47804fc5ca
Reference in New Issue
Block a user