mirror of
https://github.com/ruvnet/RuView
synced 2026-06-15 11:13:20 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ccef656221 | |||
| df617145d6 | |||
| f250149e94 | |||
| faca0530de | |||
| 6f6c867629 | |||
| 95a5ecc746 | |||
| 1f05456588 | |||
| f756a8af49 |
@@ -33,6 +33,8 @@ jobs:
|
||||
working-directory: v2
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: rustup show && rustc --version
|
||||
|
||||
@@ -53,6 +53,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -42,6 +42,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Determine deployment environment
|
||||
id: determine-env
|
||||
@@ -86,6 +88,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
@@ -132,6 +136,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v3
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -82,6 +83,13 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# ADR-262 P1: `wifi-densepose-rufield` path-deps the `vendor/rufield`
|
||||
# submodule. Without a recursive checkout the workspace build fails to
|
||||
# resolve those path deps in CI even though it passes locally.
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# `wifi-densepose-desktop` is a Tauri v2 app — `glib-sys`, `gtk-sys`,
|
||||
# `webkit2gtk-sys`, etc. need the Linux dev libraries via pkg-config or the
|
||||
@@ -202,6 +210,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
continue-on-error: true
|
||||
@@ -267,6 +277,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
@@ -335,10 +347,12 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to Container Registry
|
||||
continue-on-error: true
|
||||
@@ -407,6 +421,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -35,6 +35,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Fetch /traffic/clones + /traffic/views from GitHub
|
||||
env:
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -78,6 +80,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -145,6 +149,8 @@ jobs:
|
||||
vars.HAS_GCP_CREDENTIALS == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download x86_64 artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -20,6 +20,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with: { targets: wasm32-unknown-unknown }
|
||||
|
||||
@@ -26,6 +26,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust + wasm32 target
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -83,6 +85,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -131,6 +135,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
if: github.ref_type == 'tag'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Check firmware version.txt == tag
|
||||
run: |
|
||||
# Tag form: vX.Y.Z-esp32 → expect version.txt to contain X.Y.Z
|
||||
@@ -71,6 +73,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Build firmware (${{ matrix.variant }})
|
||||
working-directory: firmware/esp32-csi-node
|
||||
|
||||
@@ -100,6 +100,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -214,6 +216,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install clang
|
||||
run: |
|
||||
@@ -263,6 +267,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install NVS generator
|
||||
run: pip install esp-idf-nvs-partition-gen
|
||||
@@ -317,6 +323,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Download QEMU artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
@@ -22,6 +22,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
|
||||
@@ -41,6 +41,8 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install mosquitto + clients and start with allow_anonymous
|
||||
run: |
|
||||
|
||||
@@ -26,8 +26,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
|
||||
@@ -76,6 +76,8 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# Linux aarch64 needs QEMU for cross-build on x86_64 runners.
|
||||
- name: Set up QEMU
|
||||
@@ -121,6 +123,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install maturin
|
||||
run: pip install maturin>=1.7
|
||||
- name: Build sdist
|
||||
@@ -144,6 +148,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -29,6 +29,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage viewer for Pages
|
||||
run: |
|
||||
|
||||
@@ -40,6 +40,8 @@ jobs:
|
||||
- { label: 'full+train', flags: '--features full,train' }
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -60,6 +62,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
# v2/rust-toolchain.toml pins channel "1.89" with profile "minimal" (no
|
||||
# clippy). dtolnay@stable installs clippy on the floating "stable"
|
||||
# toolchain, but the override makes cargo use the separate "1.89"
|
||||
@@ -93,6 +97,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
@@ -127,6 +133,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: publish = false is present (no accidental crates.io publish)
|
||||
run: |
|
||||
CARGO=v2/crates/ruview-swarm/Cargo.toml
|
||||
|
||||
@@ -28,6 +28,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
@@ -97,6 +98,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -164,10 +167,12 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build Docker image for scanning
|
||||
continue-on-error: true
|
||||
@@ -245,6 +250,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Run Checkov IaC scan
|
||||
continue-on-error: true
|
||||
@@ -307,6 +314,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run TruffleHog secret scan
|
||||
@@ -341,6 +349,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
@@ -378,6 +388,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Check security policy files
|
||||
continue-on-error: true
|
||||
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
# by the runner, not built on a separate arm64 host).
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
# Bypassing docker/login-action@v3: the action kept emitting
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Stage demos for Pages
|
||||
run: |
|
||||
|
||||
@@ -30,6 +30,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- **ADR-262 P3 — live RuField surface: RuView's running sensing-server now speaks RuField on `/api/field` + `/ws/field`.** Wires the P1 `wifi-densepose-rufield` bridge into the live `wifi-densepose-sensing-server` (the bridge is the only added coupling, ADR-262 §5.4). A new `src/rufield_surface.rs` module (kept out of the 8k-line `main.rs`) holds a `FieldSurface` with a **dedicated ed25519 `Signer`**, a bounded ring buffer of recent signed events (`FIELD_RING_CAPACITY = 64`), and the `/ws/field` broadcast topic; it exposes `GET /api/field` (latest signed `FieldEvent`s + signer pubkey + a `dev_signing_key` flag) and `GET /ws/field` (per-cycle stream, mirroring `/ws/sensing`), plus a standalone `router()` for isolated testing. **Tap:** at the ESP32 governed-trust cycle (`main.rs` `observe_cycle` ~`:5886` / `SensingUpdate` build ~`:5938`), `emit_rufield_event` joins the cycle's real `SensingUpdate` (features/classification/signal_field) with the engine's recorded `effective_class`/`demoted` trust state into a `SensingSnapshot` and surfaces a signed `FieldEvent` — **existing endpoints (`/ws/sensing` etc.) are unchanged; this is purely additive.** **Signer (defers the P2 key decision, §8 Q1):** a **standalone dev/sensing key** from `WDP_RUFIELD_SIGNING_SEED` (64-hex or ≥32-byte value), else a deterministic dev default with a logged `WARN` — reusing the `cog-ha-matter` Ed25519 key is the deferred P2 call, so P3 does not pre-empt it. **Egress privacy (fail-closed):** `network_egress_allowed` is *stricter* than `DefaultPrivacyGuard` for an unattended live surface — only **P1/P2** leave the box; P0 (raw) and P3/P4/P5 are held edge-local, so a `Derived → P4/P5` cycle **never** surfaces; no-presence cycles emit **no phantom event**. **P3 acceptance gates (`tests/rufield_surface_test.rs`, 4 integration via `tower::oneshot` + 4 module unit, 0 failed):** a well-formed **signed** event (`Modality::WifiCsi`, P2 not P1, `is_fusable` ed25519-verified, real timestamp); empty cycle → no phantom; **privacy-safety** — an injected `Derived` trust never surfaces; a mixed stream surfaces only egress-safe events. **Honest scope (ADR-262 §0/§6):** real plumbing on a **live endpoint**, **NOT accuracy** — single-link CSI with its existing caveats (no validated room-coordinate accuracy — `field_localize`), a dedicated dev signing key pending the P2 ownership decision, no accuracy claim. The win is narrowly: "RuView's live sensing now speaks RuField on `/ws/field`."
|
||||
- **ADR-262 P1 — `wifi-densepose-rufield` anti-corruption bridge: RuView WiFi-CSI sensing → signed RuField `FieldEvent`s.** A new v2 workspace member (the *single coupling point* between RuView and the standalone RuField MFS spec, ADR-262 §5.4) that **path-deps** the `vendor/rufield` submodule crates (`rufield-core`/`-provenance`/`-privacy`/`-fusion` — pure-Rust, `--no-default-features`-buildable: serde/sha2/ed25519/toml only, no tch/openblas/ndarray/candle) and **no** RuView internal crate. The bridge takes owned primitives — `SensingSnapshot` mirrors the `/ws/sensing` `SensingUpdate` (features + classification + signal_field) joined with the `TrustedOutput` trust state (`trust_class`/`demoted`/`identity_bound`) — and `snapshot_to_field_event()` emits one **signed** `FieldEvent` (`Modality::WifiCsi`, axis `[Frequency]`): a real `FieldTensor` from the feature scalars with the real `timestamp_ns`; an `Observation` whose `range_m`/`motion_vector`/`space_cell` are derived from the strongest **signal-field peak** when present (else `None` — coordinates are **never fabricated**, per the `field_localize` caveat) and `confidence` from the classification; a real `ProvenanceRef` (sha256 over the tensor bytes, `synthetic=false`) **ed25519-signed** so `rufield_provenance::is_fusable` passes. **The §3.3 privacy mapping is the critical correctness item**, implemented as `map_privacy()` mapping RuView's class onto RuField P0–P5 **by information content, NEVER by byte value** and **fail-closed**: RuView `Derived` (byte `1`, which sorts *below* `Anonymous` byte `2`) carries an identity embedding → maps to **P4** (or **P5** if identity-bound), **never P1** (the single most dangerous mapping mistake); `Raw → P0`, `Anonymous → P2`, `Restricted → P2`; a governed-engine `demoted` cycle floors the egress class to ≥ P2 with raw suppressed. **P1 acceptance gates (15 tests / 0 failed — 5 unit + 9 integration + 1 doc):** round-trip (`SensingSnapshot → FieldEvent →` serde `→` equal), `is_fusable` (verified ed25519 receipt), `RuFieldFusion::ingest` accept + `infer()` runs, **privacy-safety** (`gate_privacy_safety_derived_never_maps_to_low_privacy` — `Derived → P4/P5`, never P1; a table test over every RuView class; fail-closed demotion), and determinism (same snapshot + same signer seed → byte-identical event). **Honest scope:** this is **P1 plumbing** — a tested conversion + a safe privacy mapping. It is **not** wired into the live server (that is P3) and makes **no accuracy claim** (RuField v0.1 is synthetic; RuView's single-link CSI carries its own caveats). CI: the `rust-tests` workflow checkout gains `submodules: recursive` so the path-deps resolve. Python deterministic proof unchanged (off the signal proof path).
|
||||
- **ADR-262 (Proposed): RuField MFS ↔ RuView integration — a live `SensingServerAdapter`, a privacy/provenance bridge, MAPPED not papered-over.** Researched integration design for wiring RuField into RuView. Recommends: a thin **`wifi-densepose-rufield` bridge crate** (anti-corruption layer, path-deps on the `vendor/rufield` submodule — the `vendor/rvcsi` pattern, since rufield crates are unpublished); a **live `SensingServerAdapter`** that taps the real `SensingUpdate` emit site joined with `TrustedOutput` trust state and emits one signed `FieldEvent`/cycle (the file-based `CsiReplayAdapter` stays for offline replay); **vertical fusion composition** (ruvsense fuses *within* WiFi → one `wifi_csi` event → rufield-fusion graph fuses *across* modalities above it); and **one canonical privacy/provenance model** (RuView `effective_class` is source-of-truth, mapped to RuField P0–P5 at egress; reuse the existing `cog-ha-matter` SHA-256+Ed25519 chain for the `ProvenanceReceipt`). **Key honest finding:** RuView has **two privacy enums + three witness mechanisms across two hash algorithms** that do not map 1:1 onto P0–P5, and a real trap — RuView's `Derived` privacy byte (`1`) sorts *below* `Anonymous` (`2`) yet carries identity embeddings, so the bridge must map by **information content** (`Derived → P4/P5`), never by byte value, or it would leak identity as low-privacy P1. 4 independently-shippable phases, each with a test gate (round-trip / `is_fusable` / privacy-monotonicity / ed25519-verify). Honest scope: this is **plumbing architecture, not accuracy** — RuField v0.1 is synthetic and RuView's only real-CSI path is unlabeled replay; the ADR claims only architecture, gated by round-trip/monotonicity/signature tests.
|
||||
- **RuField `CsiReplayAdapter` — first real (non-synthetic) WiFi-CSI adapter (ADR-260 §17).** RuField now ingests **real captured WiFi CSI** instead of only the synthetic simulator. New `rufield-adapters::csi_replay` parses RuView's `.csi.jsonl` recording format (`{timestamp, subcarriers[]}`), normalizes each frame to a `FieldTensor` (`WifiCsi`, real amplitudes + real `timestamp_ns`), establishes a per-subcarrier Welford **empty-room baseline** via `calibrate()`, derives a **physically-grounded CSI-variance motion/presence proxy** (normalized MAD vs baseline → P2 motion/presence, else P1), and emits `FieldEvent`s with a **real sha256 + ed25519 provenance receipt** (`synthetic=false`). **Measured on 199 real captured frames:** 184 presence-proxy / 69 motion-proxy → fed through `RuFieldFusion` → **182 fused inferences (115 breathing, 67 person_present) from real signal.** 12 tests (9 unit + 3 integration over real-CSI fixtures), deterministic (byte-identical stream per file). **Honest caveats (stated everywhere):** it's **replay from file, not live hardware**; recordings are **unlabeled**, so the motion/presence output is a **proxy, NOT validated accuracy** (no pose, no accuracy numbers); live streaming + labeled validation remain roadmap; mmWave/thermal stay synthetic. The win is "RuField ingests real WiFi CSI and produces fused events from it." [`ruvnet/rufield`](https://github.com/ruvnet/rufield) `crates/rufield-adapters`; `vendor/rufield` submodule bumped.
|
||||
- **RuField `rufield-viewer` web dashboard — completes ADR-260 §27.9 (all §27 criteria 1–10 now PASS).** A read-only Axum + vanilla-JS dashboard (no build step — `cargo run -p rufield-viewer`) that streams the deterministic SyntheticSim→fusion camera-free room-intelligence demo: live room-state inferences with confidence, a scrolling event log where every event carries its modality + a colour-coded **P0–P5 privacy badge**, the fusion graph (supporting=green / contradicting=red per inference), and a click-to-open **provenance-receipt modal** (sha256 + ed25519 signer + verified ✓ / fusable ✓) — behind a permanent, undismissable `SYNTHETIC — simulated sensors, no hardware` banner. Endpoints `/` · `/app.js` · `/health` · `/api/run` (full deterministic JSON) · `/events` (SSE). 12 new tests. Honest scope: a read-only SYNTHETIC demo viewer, **not** a device-management console — fleet/real-adapter management is a separate later milestone. Lives in [`ruvnet/rufield`](https://github.com/ruvnet/rufield) (`crates/rufield-viewer`, repo now 7 crates / 72 tests); `vendor/rufield` submodule bumped to include it.
|
||||
- **ADR-261: RuVector graph-ANN index — a real HNSW baseline + a SymphonyQG-style quantized variant, MEASURED (honest negative).** Closes the [ADR-156 §5 #1](docs/adr/ADR-156-ruvector-fusion-beyond-sota.md) gap: the SymphonyQG (SIGMOD 2025) **3.5–17× QPS-over-HNSW** claim was CLAIMED-only because **no HNSW baseline existed to compare against**. This adds one. New pure-Rust, `--no-default-features`-buildable modules in `wifi-densepose-ruvector`: `hnsw.rs` (a correct float HNSW — Malkov & Yashunin: multi-layer NSW graph, `ef_construction`/`ef_search`, Algorithm-4 neighbour selection, **seeded-deterministic** level assignment via SplitMix64, L2 + cosine, full degenerate-case guards), `hnsw_quantized.rs` (the SymphonyQG-style variant — the **same** graph traversed by a cheap **1-bit Hamming** score over the RaBitQ Pass-2 rotated sign code, then **exact-float rerank**), `ann_measure.rs` + `benches/ann_bench.rs` (one shared deterministic planted-cluster fixture; the `ann_bench_report` test is the source of truth). **MEASURED (dim=128, N=10k, K=10, `--release`):** float HNSW = **~25× QPS over linear scan at recall ≥0.99** (the baseline this gap needed; recall@10 correctness gate ≥0.95 holds, L2 + cosine). **Honest negative:** the 1-bit quantized traversal is **too coarse to beat float HNSW at equal recall at this scale** — its best recall is **0.738**, never reaching the ≥0.90 equal-recall point, so there is **no QPS win** over float HNSW; the 3.5–17× is **not reproduced** by our 1-bit construction here. The recall gate also **caught a real index-out-of-bounds bug** in the insert path (disclosed in ADR-261 §4). Caveat: this is **our** HNSW + **our** 1-bit quant, not SymphonyQG's exact system — it tests the *direction* of the claim, with the expected crossover at large N + a multi-bit traversal code. **We did not tune to manufacture a speedup.** +20 tests (ruvector lib 131→151, 0 failed). ADR-156 §5 #1 / §8 backlog: CLAIMED → **MEASURED-direction-tested**. Python deterministic proof unchanged (off the signal proof path).
|
||||
- **ADR-261 Milestone-2: multi-bit quantized HNSW traversal + large-N scaling study — MEASURED (honest negative).** Extends ADR-261's quantized index from 1-bit to **`b`-bit-per-dimension** (`b ∈ {1,2,4}`, 16/32/64 B/node) over the Pass-2 rotated coordinates, and runs a deterministic scaling study (N ∈ {10k, 100k, 250k}) to test M1's *prediction* of a large-N crossover. **Result: no crossover at any measured (N, b), and the trend refutes the prediction.** At N=10k more bits lift the equal-recall QPS ratio (0.19×→0.46×→0.48×) and let b≥2 reach the 0.90 recall bar 1-bit missed — but quant stays slower than float HNSW at equal recall; at N=100k/250k quant recall *collapses* (b=4: 1.000→0.788→0.624, never ≥0.90) while float holds ≥0.92 (denser graph → low-bit codes can't separate near-neighbours, beam goes off-path faster than the float-distance saving repays). Caveat: our HNSW + our per-node multi-bit code, not SymphonyQG's RaBitQ-fused graph — refutes the *direction* at ≤250k, not their million-scale numbers. ruvector lib **151→156** (+5 tests; `scaling_report` `#[ignore]` produced the table). A published negative with the mechanism explained. ADR-261 §11.
|
||||
- **ADR-260: RuField MFS — the open specification for camera-free multimodal field sensing.** A common event / tensor / calibration / privacy / provenance model that sits *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and future quantum sensors (each modality emits a normalized `FieldEvent` → `FieldTensor` → `FusionGraph` → `PrivacyClass` → `ProvenanceReceipt`). Published as a **standalone repo** [`ruvnet/rufield`](https://github.com/ruvnet/rufield) and vendored here as the `vendor/rufield` submodule (the `vendor/rvcsi` pattern — not a `v2/` workspace member). The v0.1 reference stack is a self-contained 6-crate Rust workspace (`rufield-core`, `-provenance` [sha256 + ed25519], `-privacy` [P0–P5 guard], `-adapters` [deterministic `SyntheticSim` across wifi_csi/mmwave_radar/infrared_thermal], `-fusion` [graph + TOML weighted-Bayes rules → 7 room-state inferences], `-bench` [deterministic runner + the §31 acceptance test]). **60 tests / 0 failed, clippy-clean.** §27 acceptance criteria 1–8 and 10 PASS; the live dashboard (9) is deferred. **All benchmark metrics are SYNTHETIC** (scored against the simulator's own ground truth — presence/breathing/bed_exit/room_transition F1 = 1.000, nocturnal_scratch 0.923 reported honestly, p95 latency ~0.01 ms, provenance coverage 100%, 0 privacy violations) — they prove the pipeline recovers known truth, **not** field accuracy; real hardware adapters (ESP32 CSI, mmWave, thermal IR) are a documented roadmap item, none validated in v0.1. The Python deterministic proof is unchanged (rufield is off the signal-processing proof path).
|
||||
|
||||
### Security
|
||||
|
||||
@@ -22,7 +22,8 @@ Dual codebase: Python v1 (`v1/`) and Rust port (`v2/`).
|
||||
| `wifi-densepose-vitals` | ESP32 CSI-grade vital sign extraction (ADR-021) |
|
||||
| `nvsim` | Deterministic NV-diamond magnetometer pipeline simulator (ADR-089) — standalone leaf, WASM-ready |
|
||||
| `vendor/rvcsi` (submodule) | **rvCSI** — edge RF sensing runtime (ADR-095/096): 9 crates (`rvcsi-core`/`-dsp`/`-events`/`-adapter-file`/`-adapter-nexmon`/`-ruvector`/`-runtime`/`-node`/`-cli`). Lives in its own repo ([github.com/ruvnet/rvcsi](https://github.com/ruvnet/rvcsi)), vendored here under `vendor/rvcsi`, published to crates.io as `rvcsi-* 0.3.x` and to npm as `@ruv/rvcsi`. Not a `v2/` workspace member — depend on the published crates (or the submodule's `crates/rvcsi-*` paths). Normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, validate-before-FFI, reusable DSP, typed confidence-scored events, the napi-c Nexmon shim (real nexmon_csi `.pcap` from a Raspberry Pi 5 / 4 / 3B+ — BCM43455c0), the napi-rs SDK, the `rvcsi` CLI, a Claude Code plugin. |
|
||||
| `vendor/rufield` (submodule) | **RuField MFS** — the open spec for camera-free multimodal field sensing (ADR-260). A common `FieldEvent`/`FieldTensor`/`FusionGraph`/`PrivacyClass`/`ProvenanceReceipt` model *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and quantum sensors. Lives in its own repo ([github.com/ruvnet/rufield](https://github.com/ruvnet/rufield)), vendored here under `vendor/rufield`. Not a `v2/` workspace member. v0.1 reference stack = 6 crates (`rufield-core`/`-provenance`/`-privacy`/`-adapters`/`-fusion`/`-bench`), 60 tests/0 failed; all benchmark metrics are **SYNTHETIC** (simulator ground truth, no hardware — real adapters are roadmap). |
|
||||
| `vendor/rufield` (submodule) | **RuField MFS** — the open spec for camera-free multimodal field sensing (ADR-260). A common `FieldEvent`/`FieldTensor`/`FusionGraph`/`PrivacyClass`/`ProvenanceReceipt` model *above* WiFi CSI/CIR/BFLD, UWB, BLE Channel Sounding, mmWave radar, ultrasound, subsonic, infrared, and quantum sensors. Lives in its own repo ([github.com/ruvnet/rufield](https://github.com/ruvnet/rufield)), vendored here under `vendor/rufield`. Not a `v2/` workspace member. v0.1 reference stack = 7 crates (`rufield-core`/`-provenance`/`-privacy`/`-adapters`/`-fusion`/`-bench`/`-viewer`), 72 tests/0 failed; `rufield-viewer` is an Axum + vanilla-JS read-only dashboard (`cargo run -p rufield-viewer`) completing ADR-260 §27.9. The WiFi-CSI modality is now **real-replay-backed** via `CsiReplayAdapter` (ingests real captured `.csi.jsonl` → fused presence/breathing inferences; replay-from-file, unlabeled CSI-variance proxy, not validated accuracy); mmWave/thermal + all synthetic-bench F1 numbers remain **SYNTHETIC** (no live hardware — live streaming + labeled accuracy are roadmap). |
|
||||
| `wifi-densepose-rufield` | ADR-262 P1 **anti-corruption bridge** — converts RuView WiFi-CSI sensing output (`SensingSnapshot` mirroring `SensingUpdate` + `TrustedOutput`, owned primitives, no dep on `wifi-densepose-sensing-server`) into **signed RuField `FieldEvent`s** (`Modality::WifiCsi`, real `timestamp_ns`, sha256 + ed25519 provenance, `synthetic=false`). The single coupling point between RuView and the standalone RuField MFS spec (§5.4); path-deps the `vendor/rufield` submodule crates (`rufield-core`/`-provenance`/`-privacy`/`-fusion`). **Critical §3.3 privacy mapping** (`map_privacy`): maps RuView class → RuField P0–P5 by **information content, never byte value**, fail-closed (`Derived → P4/P5`, never P1; `demoted` floors to ≥ P2). 15 tests / 0 failed (round-trip / `is_fusable` / fusion-ingest / privacy-safety / determinism). P1 plumbing — not wired into the live server (P3), no accuracy claim. |
|
||||
| `ruview-swarm` | Drone swarm control system (ADR-148) — hierarchical-mesh topology, Raft consensus, MARL, CSI sensing payload, MAVLink/PX4 compat, Ruflo AI-agent integration |
|
||||
|
||||
### RuvSense Modules (`signal/src/ruvsense/`)
|
||||
|
||||
@@ -14,6 +14,13 @@ COPY v2/crates/ ./crates/
|
||||
# Copy vendored RuVector crates
|
||||
COPY vendor/ruvector/ /build/vendor/ruvector/
|
||||
|
||||
# Copy vendored RuField submodule — the `wifi-densepose-rufield` bridge crate
|
||||
# (ADR-262) path-deps `../../../vendor/rufield/crates/*`, which from the Docker
|
||||
# build layout (v2/ collapsed into /build) resolves to /vendor/rufield. Copy the
|
||||
# whole tree so the rufield workspace Cargo.toml (workspace-dep inheritance) and
|
||||
# the four bridged crates (rufield-core/-provenance/-privacy/-fusion) all resolve.
|
||||
COPY vendor/rufield/ /vendor/rufield/
|
||||
|
||||
# Build release binaries:
|
||||
# - sensing-server with `mqtt` feature so the HA-DISCO MQTT publisher
|
||||
# (ADR-115) is wired in (auto-discovery topics flow to Home Assistant)
|
||||
|
||||
@@ -102,7 +102,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
| # | Candidate | What | Grade | Verdict |
|
||||
|---|-----------|------|-------|---------|
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **CLAIMED** (author-measured; **not reproduced on our hardware** — reproduction is future work) | **Lead beyond-SOTA candidate for the ruvector ANN path.** Propose as ACCEPTED-future; cite honestly as "claimed by source, reproduction pending." Best fit because the ruvector retrieval path (AETHER re-ID, sketch prefilter) is exactly an ANN problem and SymphonyQG is CPU/edge-portable like our deployment. |
|
||||
| **1** | **SymphonyQG** (SIGMOD 2025, public code) | Unified quantization + graph ANN; source reports **3.5–17× QPS over HNSW at equal recall**, pure-CPU / edge-portable. | **MEASURED-direction-tested** (was CLAIMED) — **[ADR-261](ADR-261-ruvector-graph-ann-index.md)** built the missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant and **measured** the ratio on our hardware. | **DONE — direction REFUTED at our scale (honest negative).** ADR-261 built the real HNSW baseline (**~25× QPS over linear scan at recall ≥0.99**, the substrate this row wanted) and a quantized variant. At N=10k the 1-bit Hamming traversal is **too coarse** — its best recall is 0.738, never reaching the ≥0.90 equal-recall point, so **no QPS win over float HNSW** (the SymphonyQG 3.5–17× is *not* reproduced by our 1-bit construction here). Caveat: **our HNSW + our 1-bit quant, not SymphonyQG's system**; expected crossover at large N + a multi-bit code. We did **not** tune to manufacture a speedup. |
|
||||
| **2** | **Multi-bit / Extended RaBitQ + unbiased estimator** | Extends our existing **1-bit** `sketch.rs` (ADR-084): Pass-2 rotation, multi-bit Pass-3, and the **real RaBitQ unbiased distance estimator** (Gao & Long SIGMOD 2024) reranking the candidate set from the 1-bit code + 8 B/vec side info (§11). | **MEASURED-on-our-hardware** (was CLAIMED) — rotation (§10), multi-bit (§10), and the estimator (§11) all implemented + benchmarked. Rotation lifts strict-K 36%→46%; multi-bit (≤4-bit) reaches 74% strict; **the estimator reaches 49.71% strict (cosine rerank), still short of 90%.** All clear 90% only with over-fetch (estimator improves the factor: 95% at candidate_k=24 vs sign 91.6%). | **DONE — RESOLVED-PARTIAL / NEGATIVE.** Rotation (§10) + estimator (§11) built and MEASURED. The honest negative (no strict-bar 90% from rotation, ≤4-bit, **or the unbiased estimator**) is recorded, not hidden. Over-fetch + Pass-2 is the path that meets the bar (ADR-084's "candidate set" pattern); the estimator lowers the over-fetch factor needed. |
|
||||
| **3** | **GraphPose-Fi-style learned antenna-attention + ChebGConv fusion head** | Would replace the current **untrained identity-projection + mean-pool** "attention" (the `CrossViewpointAttention` default is `ProjectionWeights::identity` — not a *learned* attention) with a learned graph fusion head. | **DATA-GATED** (per ADR-152 measurement (b): architecture is **NOT** the current bottleneck — **data is**) | **ACCEPTED-future, data-gated. Do NOT build now.** ADR-152's measured lesson was that swapping architecture without more/better paired data does not move PCK. Building a learned fusion head before the data exists would repeat the mistake ADR-155 §5 also flagged for GraphPose-Fi. |
|
||||
| — | **Cramér-Rao / sensor-placement** (`geometry.rs` CRB) | Investigated for a 2026 advance beating the textbook Fisher-information CRB already implemented. | **Investigated — NO ACTION** | **Cleared honestly.** No 2026 method beats the closed-form Fisher-information CRB for this 2-D bearing problem; our implementation is already correct SOTA. (Recording a negative result is a deliberate anti-slop signal.) The only CRB change this milestone is the §2.3 *GDOP* honesty fix, which is a labelling/quantity correction, not an algorithmic one. |
|
||||
@@ -138,7 +138,7 @@ The double-clone elimination is also correctness-neutral: all 100 `viewpoint`/`m
|
||||
|
||||
The review surfaced more than this milestone scoped. Tracked here for a future ADR-156 milestone:
|
||||
|
||||
- **SymphonyQG reproduction** (§5 #1) — reproduce the 3.5–17× QPS-over-HNSW claim on our hardware before integrating into the ruvector ANN path. Currently CLAIMED-only.
|
||||
- **SymphonyQG reproduction** (§5 #1) — **RESOLVED-DIRECTION-TESTED** (see [ADR-261](ADR-261-ruvector-graph-ann-index.md)). The missing HNSW baseline + a SymphonyQG-style 1-bit quantized-traversal variant were built and **MEASURED**: float HNSW is ~25× over linear scan at recall ≥0.99 (the baseline this gap needed), but our 1-bit quantized traversal is **too coarse to beat float HNSW at equal recall at N=10k** (best recall 0.738) — the 3.5–17× is **not reproduced** by our construction. Honest negative recorded; expected crossover is large N + a multi-bit traversal code. (Caveat: our HNSW + our 1-bit quant, not SymphonyQG's exact system.)
|
||||
- **Multi-bit / Extended RaBitQ** (§5 #2) — **RESOLVED-PARTIAL** (see §10). Pass-2 randomized rotation (FHT + seeded ±1 sign flips, `src/rotation.rs`) and a multi-bit Pass-3 experiment landed and were MEASURED against the ADR-084 ≥90% bar. **Honest result: rotation helps (+10pp at the strict bar) and Pass-2 reaches 90% with ~3× over-fetch, but NEITHER rotation nor multi-bit (up to 4-bit) clears the strict candidate_k==K 90% bar on the tested anisotropic distribution.** The original `1-bit sign quantization ships first; rotation/more-bits later if benchmark-measured top-K coverage drops below 90%` deferral is therefore retired: the rotation is built, the bar is characterised, and the residual gap is documented rather than deferred.
|
||||
- **Learned cross-viewpoint fusion head** (§5 #3, GraphPose-Fi-style) — **data-gated**: blocked on the paired multi-room data ADR-152 measurement (b) identified as the real bottleneck; do not build the architecture first.
|
||||
- **`CrossViewpointAttention` learned projections** — the default `ProjectionWeights::identity` + mean-pool is honest but unlearned; wiring real learned Q/K/V projections is part of the data-gated item above (no learned weights ⇒ the "attention" is currently a geometric-bias-weighted average, which the code/docs should keep stating plainly).
|
||||
|
||||
@@ -351,12 +351,11 @@ Total test count across the workspace: **60 tests, 0 failed**.
|
||||
| 6 | Benchmark runner produces deterministic reports | **PASS** — identical report across runs (latency is the only wall-clock field) |
|
||||
| 7 | Raw waveform storage disabled by default | **PASS** — P0 network transmission denied by default policy |
|
||||
| 8 | P4 inference requires consent policy approval | **PASS** — P4 without consent → RequiresConsent; breathing/scratch rules carry `requires_consent = true` |
|
||||
| 9 | Dashboard shows live camera-free room intelligence | **DEFERRED** — no `rufield-viewer` dashboard in v0.1; the benchmark + `room_intelligence` example provide a CLI view. Follow-up. |
|
||||
| 9 | Dashboard shows live camera-free room intelligence | **PASS** — `rufield-viewer` (Axum + vanilla JS) streams the deterministic SyntheticSim→fusion demo: live room state, privacy-badged (P0–P5) event log, fusion graph, click-to-open signed-receipt modal, behind a permanent `SYNTHETIC — simulated sensors, no hardware` banner. `cargo run -p rufield-viewer`. Read-only demo viewer (not a device-management console — that's the real-adapter milestone). |
|
||||
| 10 | Spec readable for external implementers | **PASS** — ADR-260 + detailed standalone README with compiling usage examples |
|
||||
|
||||
**Decision:** §27 criteria 1–8 and 10 PASS; criterion 9 (live dashboard) is
|
||||
**deferred** to a follow-up. Per the acceptance rule (1–8, 10 pass; 9 may be
|
||||
deferred), Status is set to **Accepted — v0.1 reference stack**.
|
||||
**Decision:** **all §27 criteria 1–10 PASS** (criterion 9, the live dashboard,
|
||||
was completed by `rufield-viewer`). Status is **Accepted — v0.1 reference stack**.
|
||||
|
||||
### Deterministic benchmark report (SYNTHETIC, seed = 2026)
|
||||
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# ADR-261: RuVector Graph-ANN Index — a real HNSW baseline + a SymphonyQG-style quantized variant, MEASURED
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | `wifi-densepose-ruvector` — `hnsw.rs`, `hnsw_quantized.rs`, `ann_measure.rs`, `benches/ann_bench.rs`, docs |
|
||||
| **Relates to** | ADR-084 (RaBitQ similarity sensor — 1-bit sketch), ADR-156 (RuVector beyond-SOTA sweep — §5 #1 SymphonyQG, §8/§10/§11 RaBitQ Pass-2/multi-bit/estimator), ADR-024 (AETHER re-ID), ADR-016/017 (RuVector integration) |
|
||||
| **Scope** | Build the **missing HNSW graph-ANN baseline** in the ruvector retrieval path, build a **SymphonyQG-style quantized-traversal variant** on the same graph, and **MEASURE** the real recall/QPS ratio between them — closing the ADR-156 §5 #1 gap honestly. Resolves ADR-156 §8 backlog item **"SymphonyQG reproduction"** from **CLAIMED-only** to **MEASURED-direction-tested**. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — the same contract as ADR-154/156:
|
||||
|
||||
- The HNSW index ships a **committed recall@10 correctness gate** (≥ 0.95 vs brute force on a planted-cluster fixture). Low recall means a graph bug; the gate is wired to fail in that case. It **did** fail first — and caught a real index-out-of-bounds bug in the insert path (§4) — which is exactly what a real gate is for.
|
||||
- Every QPS/recall number below is **MEASURED** on this box with a committed, deterministic, `--no-default-features`-runnable measurement (`src/ann_measure.rs`, `ann_bench_report`) and a committed criterion bench (`benches/ann_bench.rs`). Both call **one** shared fixture/measurement module, so the bench and the report can never measure different graphs.
|
||||
- The **headline result is an honest negative**: at our test scale the SymphonyQG-style quantized variant **does not beat float HNSW at equal recall** — the 1-bit Hamming traversal is too coarse to keep recall up. We report the real numbers, explain *why*, and state the expected large-N crossover. **We did not tune the quantized path to manufacture the 3.5–17× the source claims.** A measured negative + a scale caveat is a valid, publishable result.
|
||||
- We are explicit that this is **OUR HNSW + OUR 1-bit quantization, not SymphonyQG's exact system**. It tests the **direction** of the claim on our hardware/data, not a 1:1 reproduction.
|
||||
|
||||
Test machine: Windows 11, `cargo test --release`, `std::time::Instant` wall-clock. Numbers are warm medians on this box; the **ratio** is the claim, not the absolute QPS.
|
||||
|
||||
Reproduce:
|
||||
```bash
|
||||
cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
ann_bench_report -- --nocapture
|
||||
# Larger N: ANN_BENCH_N=50000 cargo test ... --release ann_bench_report -- --nocapture
|
||||
cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
The ruvector crate's retrieval path — AETHER re-ID hot-cache (ADR-024), the `sketch.rs` 1-bit prefilter (ADR-084), room fingerprinting — is, at its core, an **approximate nearest-neighbour (ANN)** problem: dense float embedding in, top-K similar ids out. But **the crate had no graph index**. Every `topk` was either a linear scan (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is `O(N)` per query and does not scale.
|
||||
|
||||
[ADR-156 §5 #1](ADR-156-ruvector-fusion-beyond-sota.md) graded **SymphonyQG** (SIGMOD 2025) the **lead beyond-SOTA ANN candidate**, citing the source's claim of **3.5–17× QPS over HNSW at equal recall**, but marked it **CLAIMED**:
|
||||
|
||||
> *"author-measured; **not reproduced on our hardware** — reproduction is future work."*
|
||||
|
||||
And ADR-156 §8 was blunt about *why* it could not be reproduced: **there was no HNSW baseline to compare against.** You cannot measure a ratio against a baseline that does not exist. This ADR builds that missing baseline, builds the quantized variant that tests the direction of the SymphonyQG bet, and measures the real ratio.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. Add a correct, dependency-free **float HNSW** graph index (`hnsw.rs`): the real Malkov & Yashunin (TPAMI 2018) algorithm — multi-layer navigable small-world graph, `ef_construction` / `ef_search`, the Algorithm-4 neighbour-selection heuristic, seeded-deterministic level assignment, L2 + cosine. This is the **baseline** ADR-156 said was missing.
|
||||
2. Add a **SymphonyQG-style quantized-traversal variant** (`hnsw_quantized.rs`): the *same* graph (same seed, same structure), but the beam search scores candidates with a **cheap 1-bit Hamming distance** over the RaBitQ Pass-2 rotated sign code (reusing `rotation.rs` + the sign-quantization of `sketch.rs`), then **exact-float reranks** the final candidate set. This is the SymphonyQG bet — cheaper per-node scoring, recovered by a final exact rerank.
|
||||
3. **Measure** linear vs float-HNSW vs quantized-HNSW (recall@10, QPS, equal-recall ratios) on one deterministic planted-cluster fixture, and record the honest verdict against the SymphonyQG 3.5–17× claim.
|
||||
|
||||
### Why 1-bit Hamming for the quantized traversal
|
||||
|
||||
The crate already had the exact pieces SymphonyQG fuses: a deterministic orthogonal rotation (`rotation.rs`, RaBitQ Pass-2) and sign-quantization (`sketch.rs`). A 1-bit code compares by POPCNT Hamming — a few machine words, no per-dimension float work — so it is the cheapest possible traversal score and the most direct test of "can a quantized score keep the beam on the right path." The cost (measured below): the 1-bit code is a *coarse* angle proxy (ADR-156 §10 measured ~46% strict-K coverage for sign-only), and that coarseness is what limits recall here.
|
||||
|
||||
---
|
||||
|
||||
## 3. Design
|
||||
|
||||
### 3.1 `hnsw.rs` — float HNSW (the baseline)
|
||||
|
||||
- **Graph.** `links[id][layer]` adjacency; layer 0 holds every node, higher layers exponentially sparser. `m_max` is `2·M` on layer 0, `M` above (the paper's asymmetric degree cap).
|
||||
- **Insert.** Greedy-descend the upper layers to a good entry point, then for each layer from the node's level down to 0: `search_layer` for `ef_construction` candidates, `select_neighbours` (Algorithm 4 — keep a candidate only if it is closer to the new node than to any already-selected neighbour, giving diverse navigable edges), wire bidirectional edges, re-prune any neighbour that overflows `m_max`. The node is pushed into the arrays **before** wiring so every `links[*]` index is valid mid-insert (§4 — the bug the gate caught).
|
||||
- **Search.** Greedy-descend layers `>0`, then best-first beam search of width `ef` on layer 0; return the closest `k`. Iterative (explicit heaps + visited set) — **no recursion**, bounded by the beam and the visited set.
|
||||
- **Determinism.** Level assignment is the only randomness and is driven by a **seeded SplitMix64** (the exact pattern from `rotation.rs`) — never `Date::now`/OS RNG/unseeded `rand`. Same `(seed, params, insertion order)` ⇒ bit-identical graph and search (pinned by `hnsw_is_deterministic_for_seed`).
|
||||
- **Robustness.** Empty index, `k==0`, `k>n`, single node, zero-dim, ragged query, `ef<k` all return cleanly — pinned by `*_no_panic` tests.
|
||||
|
||||
### 3.2 `hnsw_quantized.rs` — the SymphonyQG-style variant
|
||||
|
||||
Same graph as the float index (identical seed/structure — the **only** variable is the scoring), plus a per-node `ceil(D/8)`-byte 1-bit Pass-2 sign code (`D = next_pow2(dim)`). `search_quantized(query, k, ef, rerank)`:
|
||||
1. Encode the query to its 1-bit code (one rotation + sign pack).
|
||||
2. Greedy-descend + beam-search the graph scoring every visited node by **POPCNT Hamming** (query-code XOR node-code) — no per-dim float work.
|
||||
3. **Exact-float rerank** the top `rerank` Hamming candidates with the true L2/cosine metric, return the best `k`.
|
||||
|
||||
### 3.3 Security / robustness
|
||||
|
||||
Both indices: bounded **iterative** traversal (no unbounded recursion), no panic on empty/degenerate/ragged/zero-dim input (the metric compares over the shorter prefix; zero-norm cosine returns max distance, not NaN). The 1-bit encode handles padded dims via the existing `Rotation::apply_padded`.
|
||||
|
||||
---
|
||||
|
||||
## 4. The bug the correctness gate caught (disclosed, not hidden)
|
||||
|
||||
The first run of the recall@10 gate **panicked**: `index out of bounds: the len is 33 but the index is 33` in `search_layer`. Root cause: `insert` wired bidirectional edges (`links[nbr][l].push(id)`) **before** pushing the new node's own `links[id]` row into the array. A later traversal step in the *same* insert could hop to a neighbour that now pointed at `id` and read `links[id]` — which did not exist yet. Fix: push the node (with empty per-layer link lists) into `vectors`/`links`/`levels` **up front**, then wire edges into its existing slot. The new node has no incoming edges and empty outgoing lists until wiring, so it is unreachable by the searches that run first — pushing early is safe and keeps every index valid. This is exactly why the recall gate exists: a silent low-recall graph and an out-of-bounds panic are both "slop" the gate forces into the open.
|
||||
|
||||
---
|
||||
|
||||
## 5. The SymphonyQG claim being tested
|
||||
|
||||
| Source | Claim | Grade (before this ADR) |
|
||||
|--------|-------|-------------------------|
|
||||
| SymphonyQG, SIGMOD 2025 | **3.5–17× QPS over HNSW at equal recall**, via quantization unified with graph traversal, pure-CPU/edge-portable | **CLAIMED** — author-measured, *not reproduced on our hardware (no HNSW baseline existed)* |
|
||||
|
||||
The bet: a quantized traversal score is cheap enough — and accurate enough to keep the beam on-path — that you pay far less per visited node and recover the small recall loss with a final exact rerank.
|
||||
|
||||
---
|
||||
|
||||
## 6. MEASURED results
|
||||
|
||||
Fixture: planted-cluster synthetic, **dim=128, N=10,000, 64 clusters, 200 queries, K=10, noise=0.35**, L2 metric, `M=16`, `ef_construction=200`. Graph seed `0x6261524741484E53`, rotation seed `0x5EEDC0DE12345678`. `--release`, warm wall-clock on the test machine. (The fixture and both indices are shared by the criterion bench.)
|
||||
|
||||
| Method | recall@10 | QPS | latency (µs) |
|
||||
|--------|-----------|-----|--------------|
|
||||
| **linear scan (brute force)** | 1.0000 | 1,022 | 978 |
|
||||
| **float-HNSW** ef=16 | 0.9945 | **25,744** | 39 |
|
||||
| float-HNSW ef=32 | 0.9990 | 21,470 | 47 |
|
||||
| float-HNSW ef=64 | 1.0000 | 18,779 | 53 |
|
||||
| float-HNSW ef=128 | 1.0000 | 12,722 | 79 |
|
||||
| float-HNSW ef=256 | 1.0000 | 5,742 | 174 |
|
||||
| quant-HNSW ef=32 rr=20 | 0.1620 | 30,005 | 33 |
|
||||
| quant-HNSW ef=32 rr=100 | 0.2615 | 36,388 | 28 |
|
||||
| quant-HNSW ef=64 rr=100 | 0.4865 | 20,603 | 49 |
|
||||
| quant-HNSW ef=128 rr=100 | 0.6785 | 13,718 | 73 |
|
||||
| quant-HNSW ef=256 rr=100 | **0.7380** | 6,578 | 152 |
|
||||
|
||||
### Equal-recall QPS ratios
|
||||
|
||||
| Target recall | Fastest float-HNSW | Fastest quant-HNSW meeting it | quant/float | float/linear |
|
||||
|---------------|--------------------|-------------------------------|-------------|--------------|
|
||||
| ≥ 0.90 | ef=16 → 25,744 QPS | **none** (best quant recall = 0.738) | — | **25.19×** |
|
||||
| ≥ 0.95 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
| ≥ 0.99 | ef=16 → 25,744 QPS | **none** | — | **25.19×** |
|
||||
|
||||
---
|
||||
|
||||
## 7. Honest verdict
|
||||
|
||||
**The HNSW baseline is a decisive win over linear scan: ~25× QPS at recall ≥ 0.99** (ef=16: 0.9945 recall, 25,744 QPS vs linear 1,022 QPS). The correctness gate (recall@10 ≥ 0.95 vs brute force, both L2 and cosine) holds. This is the baseline ADR-156 §5 #1 said did not exist — it now does.
|
||||
|
||||
**The SymphonyQG-style quantized variant does NOT beat float HNSW at our scale — direction REFUTED at N=10k.** The 1-bit Hamming traversal is too coarse: its best achievable recall is **0.738** (ef=256, rr=100), and it never reaches even the 0.90 equal-recall point where a fair QPS comparison could be made. Where the quantized score *is* faster (ef=32: ~30–36k QPS, beating float's 25.7k), its recall collapses to 0.16–0.26 — a meaningless win. There is **no equal-recall operating point** at which quantized is faster, so the SymphonyQG 3.5–17× claim is **not reproduced** by our 1-bit construction here.
|
||||
|
||||
**Why** (so the negative is understood, not just stated):
|
||||
1. The 1-bit sign code is a **coarse angle proxy** — ADR-156 §10 already measured it at ~46% strict-K coverage. Driving graph *traversal* by that coarse score steers the beam onto the wrong nodes, and the exact-float rerank can only recover what the beam actually visited. At N=10k, near-neighbours have nearly-identical sign codes, so Hamming cannot separate them.
|
||||
2. At this scale **float distance is already cheap**: one 128-d L2 is a handful of µs; the per-node float compute the quantization saves is small relative to the recall it costs. SymphonyQG's win shows up at **much larger N** (millions), where (a) the float-distance fraction of query time dominates and (b) their *multi-bit RaBitQ-fused* code (not our 1-bit sign code) keeps recall high. **Expected crossover: large N + a higher-bit code.** ADR-156 §10 already measured that a ≤4-bit code reaches ~74% strict coverage vs 1-bit's ~46%, so a multi-bit traversal score is the obvious next lever — deferred, not claimed.
|
||||
|
||||
**Caveat (stated plainly):** this is **our** HNSW + **our** 1-bit quantization, not SymphonyQG's system. We tested the *direction* of the claim ("does quantized traversal + rerank beat float HNSW at equal recall?") on our hardware/data and got a **measured no at N=10k**. That neither confirms nor refutes SymphonyQG's own published numbers on their system/scale — it refutes the direction *for our construction at our scale*, and identifies the two levers (scale, code bit-depth) a real reproduction would need.
|
||||
|
||||
---
|
||||
|
||||
## 8. Validation
|
||||
|
||||
- **`cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --lib`** — **156 passed / 0 failed, 1 ignored** (M1 added 20: 10 `hnsw`, 7 `hnsw_quantized`, 3 `ann_measure`; M2 added 5 multi-bit/scaling tests; `scaling_report` is the `#[ignore]` measurement that produced the §11 table).
|
||||
- **`cargo test --workspace --no-default-features`** — GREEN (see §10 for the count).
|
||||
- **Correctness gate verified to bite:** the recall@10 gate **panicked** on the first (buggy) insert path (§4); after the fix it passes at 0.99+ recall (L2 and cosine).
|
||||
- **`cargo test -p wifi-densepose-ruvector --no-default-features --release ann_bench_report -- --nocapture`** — prints the §6 table; the numbers above are copied verbatim from that run.
|
||||
- **`cargo bench -p wifi-densepose-ruvector --bench ann_bench`** — compiles and runs the same fixture through criterion.
|
||||
- **`python archive/v1/data/proof/verify.py`** — **VERDICT: PASS** (the Rust ANN work is independent of the Python signal-proof pipeline; hash unchanged).
|
||||
|
||||
---
|
||||
|
||||
## 9. Consequences
|
||||
|
||||
**Positive.** ruvector now has a real, deterministic, pure-Rust HNSW graph index (25× over linear scan at high recall) usable by the AETHER re-ID / sketch-prefilter path — the ANN substrate ADR-156 §5 #1 wanted. The SymphonyQG claim is no longer CLAIMED-only: we built the missing baseline and **measured** the direction, with the bug-caught-by-the-gate disclosed.
|
||||
|
||||
**Negative / honest.** The 1-bit quantized variant is **not** an equal-recall QPS win at our scale; it is shipped as a measured experiment with a clearly-stated ceiling, not as a recommended default. Anyone reaching for it must read §7.
|
||||
|
||||
**Resolved by Milestone-2 (§11, MEASURED — no longer deferred).**
|
||||
- **Multi-bit traversal score** — implemented (`b ∈ {1,2,4}` bits/dim over the Pass-2 rotated coordinates) and measured. It *does* lift quantized recall (at N=10k, b=4 reaches the 0.90 equal-recall regime where 1-bit could not), but still does not beat float HNSW QPS.
|
||||
- **Large-N crossover measurement** — measured at N ∈ {10k, 100k, 250k}. **The predicted large-N crossover did NOT materialize — it moved the wrong way** (quant recall *collapses* as N grows). See §11.
|
||||
|
||||
**Deferred (not silently dropped).**
|
||||
- **Wiring HNSW into the live re-ID path** (AETHER hot-cache / sketch prefilter) behind a flag.
|
||||
- **N ≥ 1M + SymphonyQG's exact RaBitQ-fused construction** — our impl refutes the *direction* at ≤250k; a true 1:1 reproduction at million-scale with their fused codes remains a separate, larger build.
|
||||
|
||||
---
|
||||
|
||||
## 10. What changed, file by file
|
||||
|
||||
- `hnsw.rs` (new) — float HNSW: graph, seeded-deterministic level assignment, Algorithm-2 beam search, Algorithm-4 neighbour selection, L2/cosine, brute-force ground truth, full degenerate-case guards; 10 tests incl. the recall@10 correctness gate (L2 + cosine) and determinism. The insert-order bug fix (§4).
|
||||
- `hnsw_quantized.rs` (new) — SymphonyQG-style quantized-traversal index over the shared graph: 1-bit Pass-2 code per node, Hamming-scored greedy + beam, exact-float rerank; 7 tests incl. the rerank-recall gate and determinism.
|
||||
- `ann_measure.rs` (new) — shared deterministic fixture + recall/QPS measurement for linear / float-HNSW / quant-HNSW, the `ann_bench_report` test (the §6 source of truth), `ANN_BENCH_N` override.
|
||||
- `benches/ann_bench.rs` (new) + `Cargo.toml` `[[bench]]` — criterion bench over the same fixture/indices.
|
||||
- `lib.rs` — `pub mod hnsw / hnsw_quantized / ann_measure`; re-export `HnswIndex`, `HnswParams`, `Metric`, `QuantizedHnswIndex`.
|
||||
- `ADR-156-ruvector-fusion-beyond-sota.md` §5 #1 + §8 backlog — SymphonyQG regraded **CLAIMED → MEASURED-direction-tested (refuted at N=10k for our 1-bit construction)**, pointing here.
|
||||
- `CHANGELOG.md` — `[Unreleased]` entry.
|
||||
|
||||
---
|
||||
|
||||
## 11. Milestone-2 — multi-bit traversal + large-N scaling study (MEASURED)
|
||||
|
||||
M1 (§7) refuted the SymphonyQG direction at N=10k with a 1-bit code, and *predicted* a crossover at "large N + a higher-bit code." M2 builds both levers and measures them — so the prediction is tested, not assumed.
|
||||
|
||||
**Built:** `hnsw_quantized.rs` generalized from 1-bit to a **`b`-bit-per-dimension** code (`b ∈ {1,2,4}`, a mid-rise quantizer over the same `RANGE=3.0` rotated coordinates as ADR-156 §10's `measure_multibit`); `ann_measure.rs` gained `run_scaling_study` / `best_float_op` / `best_quant_op` + a deterministic `scaling_report` (`#[ignore]`, `--release`) and a CI-safe `scaling_study_small_is_consistent`. Memory: **16 / 32 / 64 bytes/node** for b = 1 / 2 / 4.
|
||||
|
||||
**MEASURED** (dim=128, 64 clusters, 200 queries, K=10, L2, M=16, ef_construction=200, seeded, `--release`, this box; target recall ≥ 0.90):
|
||||
|
||||
| N | bits | B/node | quant best recall | float @ target | quant @ target | quant/float |
|
||||
|--:|--:|--:|--:|--|--|--:|
|
||||
| 10,000 | 1 | 16 | 1.000 | 23,155 QPS @ r=0.995 | 4,482 QPS @ r=0.965 | **0.19×** |
|
||||
| 10,000 | 2 | 32 | 1.000 | 23,155 QPS @ r=0.995 | 10,658 QPS @ r=0.908 | **0.46×** |
|
||||
| 10,000 | 4 | 64 | 1.000 | 23,155 QPS @ r=0.995 | 11,217 QPS @ r=0.946 | **0.48×** |
|
||||
| 100,000 | 1 / 2 / 4 | 16/32/64 | 0.207 / 0.346 / 0.788 | 2,493 QPS @ r=0.938 | none (never ≥ 0.90) | — |
|
||||
| 250,000 | 1 / 2 / 4 | 16/32/64 | 0.108 / 0.210 / 0.624 | 1,593 QPS @ r=0.925 | none | — |
|
||||
|
||||
**Verdict — NO crossover at any measured (N, b) up to 250k, and the trend REFUTES the large-N prediction:**
|
||||
1. **Multi-bit helps at small N but not enough.** At N=10k, more bits lift the equal-recall QPS ratio 0.19× → 0.46× → 0.48× (and let b≥2 actually *reach* the 0.90 bar that 1-bit missed) — but quant stays **below 1.0×**, i.e. slower than float HNSW at equal recall.
|
||||
2. **The predicted large-N crossover moved the wrong way.** As N grows 10k → 100k → 250k, quant's best achievable recall **collapses** (b=4: 1.000 → 0.788 → 0.624) and never reaches the 0.90 comparison point, while float HNSW holds ≥0.92. A denser graph packs near-neighbours whose low-bit codes are nearly identical, so the approximate score steers the beam off-path faster than the bigger float-distance savings can repay. The "crossover at millions" intuition is **not supported by our construction's trend** — if anything it diverges.
|
||||
3. **Caveat unchanged:** this is our HNSW + our per-node multi-bit code, not SymphonyQG's RaBitQ-fused graph. The result refutes the *direction* for our construction at ≤250k; it does not disprove their published numbers on their system at their scale. A real 1:1 reproduction is the deferred million-scale build.
|
||||
|
||||
This is a **published negative with the mechanism explained** — the multi-bit + scaling levers were built and measured rather than asserted, and the honest outcome (no crossover, trend diverging) is recorded, not hidden.
|
||||
@@ -0,0 +1,207 @@
|
||||
# ADR-262: RuField MFS ↔ RuView integration — a live SensingServerAdapter, a privacy/provenance bridge, MAPPED not papered-over
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed — **P1 + P3 implemented** (live `/api/field` + `/ws/field`; P3 signs with a **dedicated dev/sensing key**, deferring the §8 Q1 `cog-ha-matter` key-ownership decision to P2) |
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codebase target** | New thin bridge crate `wifi-densepose-rufield` (v2 workspace member); taps `wifi-densepose-sensing-server` emit path + `wifi-densepose-engine` `TrustedOutput`; depends on `vendor/rufield/crates/rufield-*` via path (the `vendor/rvcsi` pattern) |
|
||||
| **Relates to** | ADR-260 (RuField MFS spec + v0.1 reference stack), ADR-261 (RuVector graph-ANN), ADR-141 (BFLD privacy control-plane / modes / attestation), ADR-137 (fusion-engine quality scoring / contradiction), ADR-032 (multistatic mesh security hardening / witness), ADR-116 (cog tamper-evident audit log — `cog-ha-matter` SHA-256+Ed25519), ADR-095/096 (`rvcsi` vendored-submodule precedent) |
|
||||
| **Scope** | Decide **how** RuView's live WiFi-CSI sensing-server emits RuField `FieldEvent`s, **whether** RuView's ruvsense fusion composes with or is wrapped by rufield-fusion, and **how** to reconcile RuView's existing privacy/witness/provenance machinery with RuField's P0–P5 + ed25519 `ProvenanceReceipt`. The privacy/provenance reconciliation is the crux. |
|
||||
|
||||
---
|
||||
|
||||
## 0. PROOF discipline (this ADR's contract)
|
||||
|
||||
This project has been publicly accused of "AI slop." This ADR answers with **evidence, not adjectives** — every "RuView already does X" carries a `file:line`, and every external/SOTA claim is graded.
|
||||
|
||||
- **No accuracy is claimed.** RuField v0.1 is **SYNTHETIC** end-to-end by its own admission (ADR-260 "Honest statement", line 386–390: *"Every metric here is simulator-based. No ESP32 CSI, mmWave, or thermal capture was used."*). RuView's only real-CSI rufield path today would be **replay of recorded `.csi.jsonl`, unlabeled** — `rufield-adapters::CsiReplayAdapter`'s own module doc (`vendor/rufield/crates/rufield-adapters/src/csi_replay.rs:19-31`) states it is *"real signal, replay from file not live hardware, unlabeled ⇒ proxy not validated accuracy."* This ADR therefore proposes **plumbing**, and grades its own claims as "ARCHITECTURE" (a design decision, testable by a round-trip/compile gate) vs "ACCURACY" (which it explicitly does not assert).
|
||||
- The privacy/provenance section reports an **honest conflict**: RuView has **three** witness mechanisms across two hash algorithms, and **two** privacy enums, none of which map 1:1 onto RuField's P0–P5. We map them and recommend the cleanest reconciliation rather than asserting they already align.
|
||||
- Each phase below ships an **independently testable gate** (a round-trip test, a privacy-monotonicity test, a signature-verify test) so the integration is provable, not aspirational.
|
||||
|
||||
---
|
||||
|
||||
## 0.1 Implementation status
|
||||
|
||||
**P1 (§4) is implemented** as the `wifi-densepose-rufield` bridge crate (`v2/crates/wifi-densepose-rufield/`, a new v2 workspace member; path-deps the `vendor/rufield` submodule per §5.4):
|
||||
|
||||
- **Input** — `SensingSnapshot` (owned primitives mirroring `SensingUpdate` features/classification/signal_field joined with the `TrustedOutput` `trust_class`/`demoted`/`identity_bound`); the bridge does **not** depend on `wifi-densepose-sensing-server` (anti-corruption layer).
|
||||
- **Conversion** — `snapshot_to_field_event(&snap, &Signer)` emits a signed `FieldEvent` (`Modality::WifiCsi`, axis `[Frequency]`, real `timestamp_ns`); position derived from the signal-field peak when present (never fabricated); real sha256 `ProvenanceRef` + ed25519 signature, `synthetic = false`.
|
||||
- **Privacy (§3.3 crux)** — `map_privacy()` maps by information content, **fail-closed**: `Raw → P0`, `Derived → P4` (or `P5` if identity-bound — **never P1**), `Anonymous → P2`, `Restricted → P2`; a `demoted` cycle floors egress to ≥ P2.
|
||||
- **Gates that pass** (`tests/p1_gates.rs`, 15 tests / 0 failed = 5 unit + 9 integration + 1 doc): round-trip (snapshot → `FieldEvent` → serde → equal); `is_fusable` (verified ed25519 receipt); `RuFieldFusion::ingest` accept + `infer()` runs; **privacy-safety** (`gate_privacy_safety_derived_never_maps_to_low_privacy` — `Derived → P4/P5`, never P1; full §3.3 table; fail-closed demotion); determinism (same snapshot + same signer seed → byte-identical event).
|
||||
|
||||
**P3 (§4) is implemented** as the live RuField surface in `wifi-densepose-sensing-server` (the bridge is now wired into the running server):
|
||||
|
||||
- **Tap** — at the ESP32 governed-trust cycle (`main.rs` `observe_cycle` ~`:5886` / `SensingUpdate` build ~`:5938`), a new `emit_rufield_event` joins the cycle's `SensingUpdate` (features / classification / signal_field) with the engine's recorded `effective_class` / `demoted` trust state into a `wifi_densepose_rufield::SensingSnapshot`, then `snapshot_to_field_event(&snap, &signer)`. Existing endpoints (`/ws/sensing` etc.) are **unchanged** — purely additive.
|
||||
- **Surface** — `GET /api/field` (latest signed `FieldEvent`s + signer pubkey + a `dev_signing_key` flag) and `GET /ws/field` (broadcast stream, mirroring `/ws/sensing`), both mounted on the HTTP port and `/ws/field` also on the WS port. A small bounded ring buffer (`FIELD_RING_CAPACITY = 64`) holds recent **network-surfaced** events. New handler code lives in `src/rufield_surface.rs`, not in the 8k-line `main.rs`.
|
||||
- **Signer (defers the P2 key decision)** — a **dedicated standalone `Signer`** held in server state, seeded from `WDP_RUFIELD_SIGNING_SEED` (64-hex or ≥32-byte value), else a deterministic dev default with a logged `WARN`. Reusing the `cog-ha-matter` Ed25519 key (§8 Q1) is the **deferred P2** decision — P3 uses a standalone sensing key so it does not pre-empt that call.
|
||||
- **Egress privacy (fail-closed)** — `network_egress_allowed` is *stricter* than `DefaultPrivacyGuard` for an unattended live surface: only **P1/P2** leave the box; P0 (raw) and P3/P4/P5 (identity/biometric/aggregate above the default P2 ceiling) are held edge-local. A `Derived` cycle maps to P4/P5 and is therefore **never** surfaced. No-presence cycles emit nothing (no phantom events).
|
||||
- **Gates that pass** (`tests/rufield_surface_test.rs`, 4 integration via `tower::oneshot` + 4 module unit, 0 failed): a well-formed **signed** event (`Modality::WifiCsi`, P2 not P1, `is_fusable` ed25519-verified, real timestamp); **empty cycle → no phantom**; **privacy-safety** — an injected `Derived` trust never surfaces on `/api/field`; a mixed stream surfaces only egress-safe events.
|
||||
|
||||
**Deferred:** the §3.3 *provenance carrier* recommendation (reuse the `cog-ha-matter` SHA-256+Ed25519 chain + embed the BLAKE3 engine witness) is **not** in P1/P3 — both take a dedicated `Signer` (the §8 open question 1 key-ownership decision is unresolved; P3 uses a standalone dev/sensing key precisely so it does not pre-empt P2). P2's `cog-ha-matter` key reuse + BLAKE3-embed, and P4 (multi-modality), remain future work. **No accuracy is claimed** (§0 / §6) — P1/P3 are tested plumbing on a live endpoint + a safe privacy mapping; the live surface is single-link CSI with its existing caveats (no validated room-coordinate accuracy — `field_localize`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Context — two architectures, mapped
|
||||
|
||||
### 1.1 RuField MFS (ADR-260, `vendor/rufield/`)
|
||||
|
||||
A standalone pure-Rust Cargo workspace (serde, serde_json, toml, sha2, ed25519-dalek; **no tch/ndarray/candle**), vendored here as a git submodule (`git submodule status vendor/rufield` → `ba66e2e…`), **not** a v2 workspace member — exactly the `vendor/rvcsi` precedent (ADR-095/096). **Not published to crates.io**: every internal dep is a path dep with a nominal `version = "0.1.0"` (`vendor/rufield/Cargo.toml:31-37`); the `docs.rs/rufield-*` URLs are aspirational.
|
||||
|
||||
The data model (graded ARCHITECTURE, evidence read directly):
|
||||
|
||||
- **`FieldEvent`** (`vendor/rufield/crates/rufield-core/src/event.rs:96-112`): `spec_version, event_id, timestamp_ns: u64, sensor: SensorDescriptor, tensor: FieldTensor, observation: Observation, provenance: ProvenanceRef`.
|
||||
- **`Observation`** (`event.rs:25-51`): `zone_id, space_cell, range_m, velocity_mps, motion_vector, confidence: f32, features: BTreeMap<String,f32>` (the derived P1 scalars the fusion engine actually reads), `labels: Vec<String>` (ground-truth, **never read by fusion**), `privacy_class: PrivacyClass`.
|
||||
- **`PrivacyClass`** (`rufield-core/src/privacy.rs:8-25`): `P0..P5`, `#[serde(rename_all="UPPERCASE")]`, `Ord` by declaration order so **P0 < P1 < … < P5** — higher = more private; `level()->u8` returns 0..=5 (`privacy.rs:27-40`).
|
||||
- **`ProvenanceRef`** (on-wire, `event.rs:73-93`): `raw_hash, firmware_hash` (`sha256:…`), `model_id, calibration_id, synthetic: bool`, optional `signature_hex` / `signer_pubkey_hex` (detached ed25519).
|
||||
- The four traits (`rufield-core/src/traits.rs`): **`FieldAdapter`** (`:26-38`, `next_event() -> Result<Option<FieldEvent>>`), **`FieldEncoder`** (`:41-51`, **unimplemented in v0.1** — an open seam), **`FusionEngine`** (`:54-63`, `ingest(event)` + `infer(&query)`), **`PrivacyGuard`** (`:86-97`, `authorize(class, Destination, consent, identity_bound) -> PrivacyDecision{Allow|Deny|RequiresConsent}`).
|
||||
- **`CsiReplayAdapter`** (`rufield-adapters/src/csi_replay.rs`): constructed from **already-loaded text** (`from_jsonl(&str)` `:249-251`; `from_jsonl_with(text, device_id, &[u8;32])` `:254-323`) — **not** a path/`Read`/`Iterator`. Deserializes `CsiFrameRecord { timestamp: f64 (seconds), subcarriers: Vec<f64> }` (`:74-80`), buffers all frames into a `Vec<CsiFrame>`, then streams via a cursor (`next_event` `:550-557`). Maps each frame → `FieldEvent` with `Modality::WifiCsi`, axes `[Frequency]`, a Welford motion proxy, observation `privacy_class = P2 if presence else P1` (`:439-443`), real `sha256` raw-hash, and a **real ed25519 signature** (`signer.sign_event` `:507-510`). `max_privacy_class = P2`.
|
||||
- **`RuFieldFusion`** (`rufield-fusion/src/engine.rs:55-78`): `ingest()` **rejects non-fusable events on its first line** — `if !is_fusable(&event) { return Err(NotFusable) }` (`:212-215`) — then reads `event.observation.features` into a bounded temporal window; `infer()` applies TOML rules (`WeightedBayes` noisy-OR / `TemporalWindow`) → `Vec<FieldInference>`. TOML rule struct: `inputs, method, feature, threshold, privacy_max, window_ms, requires_consent` (`rules.rs:17-35`).
|
||||
- **`is_fusable`** (`rufield-provenance/src/lib.rs:179-184`): `synthetic == true` **OR** `verify_event().is_ok()` — the §11 invariant. Signing key is `ed25519_dalek 2.1`, deterministic from a 32-byte seed; raw hash is `sha256_hex` → `"sha256:<hex>"` (`:26-35`).
|
||||
- **`DefaultPrivacyGuard`** (`rufield-privacy/src/lib.rs:38-110`): default `network_max = P2`, `allow_p0_network = false`. P5-no-identity → `Deny`; P4-no-consent → `RequiresConsent`; `EdgeLocal` → `Allow`; `Network` denies P0 and `class > network_max`.
|
||||
- **`rufield-viewer`** (Axum 0.7): **self-contained, consumes `SyntheticSim` only** — all routes are read-only GET/SSE (`GET /api/run`, `GET /events`); **there is no ingest endpoint** (`vendor/rufield/crates/rufield-viewer/src/server.rs:63-72`). Feeding it a live stream requires adding a route.
|
||||
|
||||
### 1.2 RuView (the integration target)
|
||||
|
||||
- **Sensing-server is Axum** (`v2/crates/wifi-densepose-sensing-server/src/main.rs:7498-7629`), two listeners (WS `:8765`, HTTP). CSI does **not** arrive over WS/HTTP — it arrives over **UDP** from ESP32 nodes (`use tokio::net::UdpSocket`, `main.rs:53`; `recv_from` loop `main.rs:5286-5299`), parsed by magic `0xC511_0001` → **`Esp32Frame`** (`types.rs:84-100`: `node_id, n_subcarriers, ppdu_type, amplitudes: Vec<f64>, phases: Vec<f64>`, rssi/freq/sequence) → pushed into per-node `NodeState.frame_history: VecDeque<Vec<f64>>` (`main.rs:441-497`).
|
||||
- **`/ws/sensing` emits a `SensingUpdate`** (`main.rs:267-317`), broadcast over a `tokio::sync::broadcast` channel (`s.tx.send(json)` `main.rs:5938-5991`; the WS handler just subscribes and forwards, `main.rs:3021-3073`). `SensingUpdate` carries `nodes`, `features`, `classification {motion_level, presence, confidence}`, `signal_field`, `persons: Vec<PersonDetection>` (17 COCO keypoints + `position:[f64;3]` from `field_localize`, `main.rs:403-428`), pose, vitals. **`field_localize` (PR #1050) is a module, not a route** (`mod field_localize` `main.rs:17`; honesty caveat `field_localize.rs:16-27` — a single ESP32 link cannot resolve true room position, `position` is "strongest field peak").
|
||||
- **ruvsense fusion is strictly WITHIN-WiFi-modality.** `MultistaticFuser::fuse(&[MultiBandCsiFrame]) -> FusedSensingFrame` (`v2/crates/wifi-densepose-signal/src/ruvsense/multistatic.rs:285-288`) attention-weights **multiple WiFi CSI nodes/viewpoints** (every input is ESP32 CSI; `multistatic_bridge.rs:50-62` builds the frames from `NodeState` amplitude with `HardwareType::Esp32S3`). `coherence_gate.rs:18-37` is the `GateDecision{Accept|PredictOnly|Reject|Recalibrate}`; `pose_tracker.rs:255-263` is the 17-keypoint Kalman tracker with 128-dim AETHER re-ID; `field_model.rs:301-308` does SVD room-eigenstructure perturbation extraction. **No camera/mmWave/audio enters this path** — ruvsense is a multi-link WiFi-CSI fuser.
|
||||
- **The governed-trust cycle** runs in the separate **`wifi-densepose-engine`** crate. `StreamingEngine::process_cycle` (`v2/crates/wifi-densepose-engine/src/lib.rs:409`, `run_cycle` `:434-533`) produces **`TrustedOutput`** (`:82-112`): `semantic_id, quality: QualityScore, effective_class: PrivacyClass, demoted: bool, provenance: SemanticProvenance, witness: [u8;32]` (BLAKE3 over `evidence‖model‖calibration‖privacy_decision‖class`, `witness_of` `:598-613`), `recalibration_recommended`. **Crucially, none of this trust metadata is on the `SensingUpdate` wire today** — it is exposed only out-of-band on `GET /api/v1/status` (`main.rs:4173-4178`) and as a single live effect: `EngineBridge::suppress_raw_outputs()` strips per-node amplitude when `effective_class >= Restricted` (`engine_bridge.rs:240-243`, applied `main.rs:5908-5932`). The honest scope is stated in `engine_bridge.rs:14-27`: the governed engine runs *alongside* the bare fusion path; derived outputs are "published ungoverned."
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
1. **Build a thin RuView-side bridge crate `wifi-densepose-rufield`** (a new v2 workspace member) that depends on `vendor/rufield/crates/rufield-core` (+ `rufield-provenance`, `rufield-privacy`, `rufield-fusion`) **via path** — mirroring the `vendor/rvcsi` pattern. RuView does **not** depend on published rufield crates (there are none) and does **not** vendor rufield into the v2 workspace; rufield stays a standalone submodule and the bridge is the only coupling point (an anti-corruption layer).
|
||||
2. **Emit `FieldEvent`s from the live server via an in-process `SensingServerAdapter`**, not by re-using the file-based `CsiReplayAdapter` on the hot path. The bridge taps the existing `SensingUpdate` build site and the `EngineBridge` trust state, joins them, and emits one signed `FieldEvent` per cycle on a new `tokio::broadcast` topic / optional `/ws/field` endpoint. `CsiReplayAdapter` is retained for the **offline/replay** path (recorded `.csi.jsonl` → events) because it already reads RuView's recording format (`recording.rs` writes `{session}.csi.jsonl`).
|
||||
3. **Compose the two fusion engines vertically, do not merge them.** ruvsense stays the **WiFi-modality node** (multi-link fusion → one fused WiFi belief); rufield-fusion sits **above** it as the **cross-modality** graph. ruvsense's `FusedSensingFrame`/`TrustedOutput` becomes one `FieldEvent` (modality `wifi_csi`); rufield fuses it against future mmWave/thermal/`rvcsi` events. They do not conflict because ruvsense has no cross-modality fusion to collide with (§1.2 evidence).
|
||||
4. **Reconcile privacy/provenance with ONE canonical model + a documented mapping** (§3, the crux): RuView's `effective_class` is the **source of truth**, mapped onto RuField `PrivacyClass` at the bridge; RuView's existing **`cog-ha-matter` SHA-256+Ed25519 witness chain** (already RuField's exact crypto) is adopted as the carrier for RuField `ProvenanceReceipt`, with the live BLAKE3 engine witness embedded as a hashed field. We do **not** maintain two parallel signed-receipt systems.
|
||||
|
||||
---
|
||||
|
||||
## 3. Privacy & provenance reconciliation (the crux)
|
||||
|
||||
This is the most important section. RuView and RuField genuinely **overlap and partially conflict**. We map both honestly.
|
||||
|
||||
### 3.1 What RuView actually has (implemented, with evidence)
|
||||
|
||||
- **TWO privacy enums, not one ladder.** `PrivacyClass` — **4 variants** `Raw=0, Derived=1, Anonymous=2, Restricted=3` (`v2/crates/wifi-densepose-bfld/src/lib.rs:103-116`, `#[repr(u8)]`, higher byte = more private, **non-monotonic in information** — `Derived=1` carries *more* identity than `Anonymous=2`). And `PrivacyMode` — **5 variants** `RawResearch, PrivateHome, EnterpriseAnonymous, CareWithConsent, StrictNoIdentity` (`bfld/src/privacy_mode.rs:18-31`), each mapping to a `PrivacyClass` via `target_class()` (`:63-70`; two modes collapse to `Anonymous`).
|
||||
- **THREE witness mechanisms across TWO hash algorithms:**
|
||||
- BFLD `PrivacyAttestationProof` — **BLAKE3, unsigned**, attests mode/class continuity only; **built but NOT on the live path** (ADR-141 status line ~597; `bfld/src/privacy_mode.rs:121-148`).
|
||||
- Engine-cycle `TrustedOutput.witness: [u8;32]` — **BLAKE3, unsigned**, over the full trust decision; **LIVE every cycle** (`wifi-densepose-engine/src/lib.rs:598-613`).
|
||||
- `cog-ha-matter::WitnessChain` — **SHA-256 hash chain + Ed25519 signatures** (`v2/crates/cog-ha-matter/src/witness.rs:138-151`; `witness_signing.rs:39-76`), JSONL-persisted, `verify()` + `verify_signature()`. Implemented for ADR-116 (cog/Matter audit log); **standalone, not wired to BFLD/engine**. Its `WitnessHash` newtype doc explicitly anticipates a hash-algo migration (`witness.rs:37-41`).
|
||||
- **No numeric trust score.** "Trust" in code = `base_coherence: f32∈[0,1]` + `penalized_coherence()` (`signal/.../fusion_quality.rs:99,122-126`) + a **boolean** `forces_privacy_demotion()` (`:116`). Demotion is monotonic and irreversible (`demote_one` clamps at Restricted, `engine/src/lib.rs:617-619`).
|
||||
- **Structured provenance exists, but no signed "receipt" on the sensing path.** `SemanticProvenance { evidence, model_version, calibration_version, privacy_decision }` (`v2/crates/wifi-densepose-worldgraph/src/model.rs:137-147`) is attached to every belief and is the *input* to the BLAKE3 witness — but it is unsigned and not called a receipt.
|
||||
|
||||
### 3.2 Side-by-side, graded
|
||||
|
||||
| Dimension | RuView (file:line) | RuField | Alignment |
|
||||
|---|---|---|---|
|
||||
| Privacy ladder | `PrivacyClass` 4 (`bfld/lib.rs:103`) **or** `PrivacyMode` 5 (`bfld/privacy_mode.rs:18`) | `PrivacyClass` 6 (P0–P5, `rufield-core/privacy.rs:8`) | **PARTIAL→CONFLICT** — no clean 1:1; counts differ (4/5 vs 6); RuView class ordering non-monotonic |
|
||||
| Demotion direction | higher = more private, irreversible (`engine/lib.rs:617`) | higher P# = more private, `Ord` by decl order (`privacy.rs:8-25`) | **STRONG** (same direction) |
|
||||
| Provenance receipt | `SemanticProvenance` unsigned (`worldgraph/model.rs:137`) | `ProvenanceRef` + ed25519 (`event.rs:73`) | **PARTIAL** — structured but unsigned |
|
||||
| Witness crypto (live path) | BLAKE3 `[u8;32]`, unsigned (`engine/lib.rs:598`) | sha256 + ed25519 (`rufield-provenance/lib.rs:26,135`) | **CONFLICT** (algo + signing) |
|
||||
| Witness crypto (cog-ha-matter) | **SHA-256 + Ed25519** (`cog-ha-matter/witness.rs`, `witness_signing.rs`) | **sha256 + ed25519** | **STRONG** — RuField's exact crypto, already in-repo, but unwired and in another bounded context |
|
||||
| Trust / confidence | `penalized_coherence: f32` + boolean demote (`fusion_quality.rs:122`) | `confidence: f32` per observation | **WEAK** — RuView has no graded trust object; confidence maps, demotion is binary |
|
||||
|
||||
### 3.3 The recommendation (the key call)
|
||||
|
||||
**Adopt ONE canonical model with a documented, lossy-but-monotonic mapping — do not run two parallel schemes.** Concretely:
|
||||
|
||||
1. **Privacy: RuView `effective_class` is the source of truth; the bridge maps it onto RuField `PrivacyClass`** at the egress boundary. The honest mapping (graded ARCHITECTURE — it is a *policy* decision, and it is **monotonicity-testable**, not an accuracy claim):
|
||||
|
||||
| RuView `PrivacyClass` | → RuField | Rationale |
|
||||
|---|---|---|
|
||||
| `Raw` (raw CSI amplitude) | `P0` | raw waveform |
|
||||
| `Derived` (identity embedding, LAN-only) | `P4` *(or P5 if identity-bound)* | derived **identity** features ⇒ biometric/identity tier, **not** P1 — RuView's non-monotonic `Derived=1` is the trap; map by *information content*, not byte value |
|
||||
| `Anonymous` (occupancy/aggregate) | `P2`/`P3` | occupancy → P2, room-count aggregate → P3 |
|
||||
| `Restricted` (zeroized) | `P2`-capped, raw suppressed | matches `suppress_raw_outputs` (`engine_bridge.rs:240`) |
|
||||
|
||||
The bridge **must** map `Derived → P4/P5`, never P1, because RuView's `Derived` carries `identity_embedding` (§3.1) — this is the single most dangerous mapping mistake and gets a dedicated test (P2 in §4). `PrivacyMode` (5) is the better *operator-facing* join to RuField's 6 levels but the **class** is what gates egress, so the class mapping is canonical.
|
||||
|
||||
2. **Provenance: adopt `cog-ha-matter`'s SHA-256+Ed25519 chain as the carrier for RuField `ProvenanceReceipt`** — it is already RuField's exact crypto (graded STRONG above), already implemented, already tamper-evident. The bridge constructs the RuField `ProvenanceRef` by: `raw_hash = sha256(csi bytes)`, `model_id`/`calibration_id` from `SemanticProvenance`, and **embeds the live BLAKE3 engine witness `[u8;32]` as a hashed provenance field** (it is already computed every cycle — do not throw it away), then **signs with ed25519** so `is_fusable` passes for live (non-synthetic) events. We do **not** add a second BLAKE3-vs-ed25519 argument: BLAKE3 stays RuView's internal fast cycle-fingerprint; ed25519 is the *external* attestation RuField requires. One signer, one chain.
|
||||
|
||||
3. **Trust: map `penalized_coherence` → `Observation.confidence`; keep demotion binary.** RuView has no graded trust object to reconcile; the coherence scalar is the honest analog and the demotion boolean already drives `effective_class`.
|
||||
|
||||
This is a **bridge-with-canonical-source**, not "keep both forever." RuView owns the privacy decision (it has the live governed cycle); RuField owns the *external wire shape* (P0–P5 + signed receipt). The bridge is the one-directional translation, and it is the only place the two schemes meet.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phased plan (each phase independently shippable + testable)
|
||||
|
||||
**P1 — `SensingServerAdapter` emitting `FieldEvent`s (ARCHITECTURE).**
|
||||
New crate `wifi-densepose-rufield` with a `SensingServerAdapter` that consumes a `(SensingUpdate, TrustedOutput)` pair (tapped at `main.rs:5886`/`:5938`) and emits a signed `FieldEvent` (`Modality::WifiCsi`, axes `[Frequency]`, observation features from `SensingUpdate.features`, `confidence` from `penalized_coherence`). Offline path: keep `CsiReplayAdapter` for recorded `.csi.jsonl`. **Gate:** a round-trip test — emit a `FieldEvent` from a fixture `SensingUpdate`, assert it serializes, `is_fusable` passes (ed25519-signed), and `RuFieldFusion::ingest` accepts it. No server changes required beyond exposing the tap; the adapter is a library.
|
||||
|
||||
**P2 — privacy/provenance bridge (the crux, ARCHITECTURE).**
|
||||
Implement the §3.3 mapping: `effective_class → PrivacyClass`, `cog-ha-matter` ed25519 signer for the receipt, BLAKE3 witness embedded. **Gates (three, all monotonicity/safety, not accuracy):** (a) `Derived → P4|P5` never P1 (the dangerous-mapping test); (b) privacy monotonicity — `demoted == true` ⇒ emitted `PrivacyClass >= P2` and raw suppressed; (c) signature round-trip — sign with the cog-ha-matter key, `rufield_provenance::verify_event` passes. This phase is shippable without P3 (events emitted on an internal topic, not yet on the public wire).
|
||||
|
||||
**P3 — surface in `/ws` + viewer (ARCHITECTURE).**
|
||||
Add an opt-in `/ws/field` endpoint (or a `field_events` array on `SensingUpdate` behind a flag) carrying the signed `FieldEvent` + a privacy badge. Add an ingest route to `rufield-viewer` (it has none today — `server.rs:63-72`) so it can replay RuView's live feed instead of only `SyntheticSim`. **Gate:** a WS integration test asserting a connected client receives a privacy-badged, signature-verifiable `FieldEvent`; a viewer test asserting the new ingest route renders a live event. The `cognitum` appliance can speak RuField by consuming this endpoint (it already runs `ruview-vitals-worker`); deferred to its own ADR.
|
||||
|
||||
**P4 — fusion composition + multi-modality (ARCHITECTURE, optional).**
|
||||
Wire a second modality (cheapest: an `rvcsi`-sourced event, or recorded mmWave) into `RuFieldFusion` alongside the WiFi event, proving cross-modality fusion above ruvsense. **Gate:** a fusion test with two modalities producing ≥1 cross-modal inference, with provenance coverage 100%.
|
||||
|
||||
---
|
||||
|
||||
## 5. Decision matrix
|
||||
|
||||
### 5.1 Data-path emission (P1)
|
||||
|
||||
| Option | Latency | Reuse | Live-fit | Risk | Verdict |
|
||||
|---|---|---|---|---|---|
|
||||
| Re-use `CsiReplayAdapter` on hot path | poor (file buffer, `&str` ctor) | high | **bad** — it's a file-cursor, not a live source | low | **Reject for live** (keep for replay) |
|
||||
| In-process `SensingServerAdapter` (tap `SensingUpdate`+`TrustedOutput`) | good | medium | **good** — taps the real emit + real trust state | low | **CHOSEN** |
|
||||
| Server publishes `FieldEvent` on its own topic (no adapter trait) | good | low | good | medium (bypasses `FieldAdapter` contract) | Reject — loses the trait seam |
|
||||
|
||||
### 5.2 Fusion relationship (P3/P4)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Merge ruvsense into rufield-fusion | **Reject** — different scopes; ruvsense is within-WiFi multi-link, rufield is cross-modality |
|
||||
| rufield-fusion wraps ruvsense (vertical compose) | **CHOSEN** — ruvsense → one WiFi `FieldEvent` → rufield cross-modality graph |
|
||||
| Run both as peers, reconcile after | Reject — duplicates fusion semantics, two contradiction models |
|
||||
|
||||
### 5.3 Privacy/provenance reconciliation (P2)
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| (a) Map RuView classes onto RuField P0–P5, RuView canonical | **CHOSEN (privacy)** — `effective_class` is the live source of truth |
|
||||
| (b) Adopt RuField ed25519 receipts as RuView's provenance | **CHOSEN (provenance)** — via the already-present `cog-ha-matter` SHA-256+Ed25519 chain |
|
||||
| (c) Keep both schemes with a permanent bridge | **Reject** — two signed-receipt systems is the duplication we must not ship |
|
||||
|
||||
### 5.4 Dependency direction
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| Depend on published rufield crates | **Reject** — not published (`vendor/rufield/Cargo.toml:31-37`) |
|
||||
| Make rufield a v2 workspace member | **Reject** — breaks the standalone-spec/`rvcsi` precedent |
|
||||
| Thin `wifi-densepose-rufield` bridge → path deps on submodule | **CHOSEN** — anti-corruption layer, single coupling point |
|
||||
|
||||
---
|
||||
|
||||
## 6. Security & honesty notes
|
||||
|
||||
- **No accuracy claim.** Live RuField events from RuView are derived from the same single-link CSI whose own caveats are on record (`field_localize.rs:16-27`); the offline path is unlabeled replay (`csi_replay.rs:19-31`). This ADR ships **plumbing with monotonicity/signature gates**, not validated F1.
|
||||
- **The dangerous mapping is `Derived → P1`.** RuView's `Derived` byte value (1) is numerically below `Anonymous` (2) but carries identity (`bfld/lib.rs`); a naive byte-mapping would leak identity-bearing features as low-privacy P1. P2's gate (a) exists specifically to prevent this.
|
||||
- **One signer, not two.** Adding a second ed25519 keypair alongside `cog-ha-matter`'s would create two roots of trust. The bridge reuses the cog-ha-matter signing key (`witness_signing.rs`).
|
||||
- **`is_fusable` is a real gate, not decoration** (`rufield-provenance/lib.rs:179-184`): live events that fail to sign are rejected by `RuFieldFusion::ingest` — we must not paper over a signing failure with `synthetic = true` on a real event (that would be the §11 invariant violation the spec forbids).
|
||||
- BLAKE3 stays internal; ed25519 is the external attestation. We do not relitigate RuView's BLAKE3 cycle-witness — it is embedded, not replaced.
|
||||
|
||||
## 7. Consequences
|
||||
|
||||
**Positive:** RuView becomes one honest adapter in the larger RuField ecosystem (ADR-260 goal §9) without forking its fusion or privacy engine; the three witness mechanisms get a single external attestation path; cross-modality fusion becomes possible above the existing WiFi fusion; the `cognitum` appliance gains a standard wire format. The bridge is the only coupling point, so rufield can evolve as a standalone spec.
|
||||
|
||||
**Negative:** a fourth crate to maintain; the privacy mapping is lossy (4/5 → 6) and must be kept honest by tests; reusing the `cog-ha-matter` key crosses a bounded-context boundary (cog/Matter ↔ sensing) that ADR-116 kept separate — that coupling needs review. The live trust metadata (`witness`, `effective_class`) is **currently decoupled** from `SensingUpdate` (§1.2), so P1 must do real join work, not a field read.
|
||||
|
||||
## 8. Open questions
|
||||
|
||||
1. **Signer ownership:** should the bridge reuse the `cog-ha-matter` Ed25519 key, or mint a dedicated RuView-sensing key with its own rotation? (Reuse couples bounded contexts; a new key adds a second root of trust.)
|
||||
2. **`PrivacyMode` vs `PrivacyClass` as the canonical map target:** class gates egress (chosen), but the 5-mode ladder is the cleaner join to 6 levels — do we expose mode in the receipt too?
|
||||
3. **Where does the BLAKE3 engine witness live in the RuField receipt** — a `firmware_hash`-style field, an extension field, or a `CalibrationReceipt.data_hash`? (RuField's `ProvenanceRef` has no spare slot; needs a spec extension or reuse of `model_id`.)
|
||||
4. **Should `field_localize` positions ride in `Observation.space_cell`/`motion_vector`** given the explicit single-link caveat, or stay RuView-only until multi-node calibration lands?
|
||||
5. **`rvcsi` relationship:** `rvcsi` has its own `CsiFrame`/`CsiWindow` and could implement `FieldAdapter` directly — should the second modality in P4 be `rvcsi`, making RuField the convergence point for *both* vendored sensing runtimes?
|
||||
6. **Transport:** RuField ADR-260 §29 leaves default transport open (MQTT/NATS/WS/MCP). RuView is WS + UDP + broadcast; does `/ws/field` suffice, or does the appliance need MQTT to match the cog stack?
|
||||
|
||||
## 9. Recommendation
|
||||
|
||||
Proceed with P1+P2 behind a feature flag. They are independently shippable, carry real gates (round-trip, monotonicity, signature-verify), and require no change to RuView's fusion or privacy engine — only a tap and a translation. Defer P3/P4 and the appliance/transport questions to follow-up ADRs once the bridge round-trips on recorded `.csi.jsonl` and on one live cycle.
|
||||
@@ -522,6 +522,25 @@ Base URL: `http://localhost:3000` (Docker) or `http://localhost:8080` (binary de
|
||||
| `GET` | `/api/v1/mesh` | ADR-110 fleet-wide mesh sync map ([iter 29](adr/ADR-110-esp32-c6-firmware-extension.md)) | `{"nodes":{"9":{...},"12":{...}},"total":2}` |
|
||||
| `GET` | `/api/v1/nodes/:id/sync` | Single-node mesh sync snapshot (or 404) | `{"offset_us":1163565,"is_leader":false,...}` |
|
||||
| `GET` | `/api/v1/mesh/metrics` | ADR-110 mesh state in Prometheus exposition format ([iter 36](adr/ADR-110-esp32-c6-firmware-extension.md)) | `wifi_densepose_mesh_offset_us{node="9"} 1163565\n…` |
|
||||
| `GET` | `/api/field` | ADR-262 P3 — latest **signed RuField `FieldEvent`s** from the live sensing cycle, plus the signer pubkey + a `dev_signing_key` flag. Only egress-safe (P1/P2) events are surfaced; identity/biometric (P4/P5) and raw (P0) are held edge-local | `{"spec":"rufield","signer_pubkey_hex":"…","dev_signing_key":true,"events":[…]}` |
|
||||
|
||||
### RuField surface (ADR-262 P3)
|
||||
|
||||
RuView's live WiFi-CSI sensing now also speaks the standalone **RuField MFS** wire format. Each governed sensing cycle is converted (via the `wifi-densepose-rufield` anti-corruption bridge) into a **signed** `FieldEvent` (`Modality::WifiCsi`, ed25519 `ProvenanceRef`) and surfaced on two additive endpoints:
|
||||
|
||||
- `GET /api/field` — the most recent signed events (JSON).
|
||||
- `GET /ws/field` — a WebSocket that streams each cycle's signed event (mirrors `/ws/sensing`).
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/field | python -m json.tool # latest signed FieldEvents
|
||||
python -c "import asyncio,websockets; asyncio.run((lambda: websockets.connect('ws://localhost:8765/ws/field'))())" # stream
|
||||
```
|
||||
|
||||
Privacy is fail-closed: only egress-safe **P1/P2** events leave the box — raw (P0) and identity/biometric/aggregate (P3–P5) cycles are held **edge-local** and never appear on these endpoints; a no-presence cycle emits **no event**.
|
||||
|
||||
**Signing key:** the surface signs with a **dedicated dev/sensing key**, seeded from `WDP_RUFIELD_SIGNING_SEED` (a 64-char hex string or a ≥32-byte value); when unset it falls back to a deterministic dev default and logs a `WARN` (the `dev_signing_key` flag in `/api/field` reflects this). This is a standalone key pending the ADR-262 §8 Q1 key-ownership decision — set `WDP_RUFIELD_SIGNING_SEED` for any real deployment.
|
||||
|
||||
> **Honesty (ADR-262 §0/§6):** this is real plumbing on a live endpoint, **not an accuracy claim.** It is the single-link CSI sensing with its existing caveats (no validated room-coordinate accuracy — positions are the "strongest field peak", not calibrated triangulation).
|
||||
|
||||
### Example: Get fleet mesh state (ADR-110)
|
||||
|
||||
|
||||
Generated
+49
@@ -7085,6 +7085,42 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-fusion"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"toml 0.8.23",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-privacy"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rufield-provenance"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"rufield-core",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc"
|
||||
version = "0.24.0"
|
||||
@@ -11045,6 +11081,18 @@ dependencies = [
|
||||
"tower-http",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"rufield-core",
|
||||
"rufield-fusion",
|
||||
"rufield-privacy",
|
||||
"rufield-provenance",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wifi-densepose-ruvector"
|
||||
version = "0.3.2"
|
||||
@@ -11094,6 +11142,7 @@ dependencies = [
|
||||
"wifi-densepose-engine",
|
||||
"wifi-densepose-geo",
|
||||
"wifi-densepose-hardware",
|
||||
"wifi-densepose-rufield",
|
||||
"wifi-densepose-signal",
|
||||
"wifi-densepose-wifiscan",
|
||||
"wifi-densepose-worldgraph",
|
||||
|
||||
@@ -72,6 +72,11 @@ members = [
|
||||
"crates/homecore-assist", # ADR-133 — HOMECORE voice assistant + ruflo bridge
|
||||
"crates/homecore-server", # iter-9 — HOMECORE integration binary (all 8 crates wired together)
|
||||
"crates/ruview-swarm", # ADR-148 — drone swarm control system
|
||||
# ADR-262 P1 — anti-corruption bridge converting RuView WiFi-CSI sensing
|
||||
# output into signed RuField FieldEvents. Path-deps the `vendor/rufield`
|
||||
# submodule crates (rufield-core/-provenance/-privacy/-fusion); single
|
||||
# coupling point between RuView and the standalone RuField MFS spec.
|
||||
"crates/wifi-densepose-rufield",
|
||||
]
|
||||
# ADR-040: WASM edge crate targets wasm32-unknown-unknown (no_std),
|
||||
# excluded from workspace to avoid breaking `cargo test --workspace`.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "wifi-densepose-rufield"
|
||||
version = "0.3.0"
|
||||
edition = "2021"
|
||||
description = "ADR-262 anti-corruption bridge: converts RuView WiFi-CSI sensing output into signed RuField FieldEvents (P0–P5 privacy mapping + ed25519 provenance)"
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
# ADR-262 §5.4: this crate is the single coupling point ("anti-corruption
|
||||
# layer") between RuView and the standalone RuField MFS spec. It depends on the
|
||||
# `vendor/rufield` submodule crates **via path** (the `vendor/rvcsi` pattern) —
|
||||
# RuView does NOT depend on published rufield crates (there are none) and does
|
||||
# NOT make rufield a v2 workspace member. The four crates below are pure-Rust
|
||||
# (serde / serde_json / toml / sha2 / ed25519-dalek only — no tch / openblas /
|
||||
# ndarray / candle), so they build under `--no-default-features`.
|
||||
[dependencies]
|
||||
rufield-core = { path = "../../../vendor/rufield/crates/rufield-core" }
|
||||
rufield-provenance = { path = "../../../vendor/rufield/crates/rufield-provenance" }
|
||||
rufield-privacy = { path = "../../../vendor/rufield/crates/rufield-privacy" }
|
||||
rufield-fusion = { path = "../../../vendor/rufield/crates/rufield-fusion" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,206 @@
|
||||
//! The conversion: `SensingSnapshot` → signed `FieldEvent` (ADR-262 P1).
|
||||
//!
|
||||
//! This is the in-process `SensingServerAdapter` core (ADR-262 §4 P1 / §5.1):
|
||||
//! it consumes a `(SensingUpdate, TrustedOutput)` join — modelled here as a
|
||||
//! [`SensingSnapshot`] of owned primitives — and emits one signed
|
||||
//! [`FieldEvent`] (`Modality::WifiCsi`, axis `[Frequency]`) per cycle.
|
||||
|
||||
use crate::privacy::egress_class;
|
||||
use crate::snapshot::{SensingSnapshot, SignalField};
|
||||
use rufield_core::{
|
||||
FieldAxis, FieldEvent, FieldTensor, Modality, Observation, PrivacyClass, ProvenanceRef,
|
||||
SensorDescriptor,
|
||||
};
|
||||
use rufield_provenance::{sha256_hex, Signer};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Model id stamped on emitted events (ADR-262 — derived features come from
|
||||
/// RuView's `/ws/sensing` pipeline, not a trained encoder).
|
||||
const MODEL_ID: &str = "ruview_sensing_server_v1";
|
||||
|
||||
/// Firmware hash placeholder until the real ESP32 firmware image hash is wired
|
||||
/// through (ADR-262 §8 open question 3 — the BLAKE3 engine witness slot). A
|
||||
/// stable `sha256:` over the model id keeps it a real digest, not a fake.
|
||||
fn firmware_hash() -> String {
|
||||
sha256_hex(MODEL_ID.as_bytes())
|
||||
}
|
||||
|
||||
/// Squash a non-negative power-like scalar into `[0, 1]` deterministically.
|
||||
/// `x / (x + 1)` — monotone, no panics, no calibration claim.
|
||||
fn squash(x: f64) -> f32 {
|
||||
if !x.is_finite() || x <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(x / (x + 1.0)) as f32
|
||||
}
|
||||
|
||||
/// Build the `Observation.features` map the RuField fusion engine reads
|
||||
/// (`rufield-fusion/engine.rs:217-228`: `motion_energy`, `breathing_band`,
|
||||
/// `transient`, `presence`, `range_m`, plus `posture_height`).
|
||||
fn build_features(snap: &SensingSnapshot, range_m: Option<f32>) -> BTreeMap<String, f32> {
|
||||
let f = &snap.features;
|
||||
let mut m = BTreeMap::new();
|
||||
m.insert("motion_energy".to_string(), squash(f.motion_band_power));
|
||||
m.insert("breathing_band".to_string(), squash(f.breathing_band_power));
|
||||
m.insert("transient".to_string(), squash(f.change_points as f64));
|
||||
m.insert(
|
||||
"presence".to_string(),
|
||||
if snap.classification.presence { 1.0 } else { 0.0 },
|
||||
);
|
||||
if let Some(r) = range_m {
|
||||
m.insert("range_m".to_string(), r);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// Derive a real range (metres) and motion vector from the strongest signal
|
||||
/// field peak, if a field is present. Returns `(range_m, motion_vector,
|
||||
/// space_cell)` — all `None` when there is no field (we do NOT fabricate
|
||||
/// coordinates, per ADR-262 §4 P1).
|
||||
fn derive_position(
|
||||
field: Option<&SignalField>,
|
||||
) -> (Option<f32>, Option<[f32; 3]>, Option<[i32; 3]>) {
|
||||
let Some(field) = field else {
|
||||
return (None, None, None);
|
||||
};
|
||||
let Some(cell) = field.peak_cell() else {
|
||||
return (None, None, None);
|
||||
};
|
||||
// Range from origin in grid-cell units (real readout, not calibrated
|
||||
// metres — the honesty caveat from `field_localize.rs:16-27`).
|
||||
let [x, y, z] = cell;
|
||||
let range = ((x * x + y * y + z * z) as f32).sqrt();
|
||||
let mag = if range > 0.0 { range } else { 1.0 };
|
||||
let motion_vector = [x as f32 / mag, y as f32 / mag, z as f32 / mag];
|
||||
(Some(range), Some(motion_vector), Some(cell))
|
||||
}
|
||||
|
||||
/// Stable, deterministic event id from `(node_id, timestamp_ns)`. No RNG, so
|
||||
/// the same snapshot always yields the same id (required for the determinism
|
||||
/// gate).
|
||||
fn event_id(snap: &SensingSnapshot) -> String {
|
||||
format!("ruview-{}-{}", snap.node_id, snap.timestamp_ns)
|
||||
}
|
||||
|
||||
/// Convert a [`SensingSnapshot`] to a **signed** [`FieldEvent`] (ADR-262 P1).
|
||||
///
|
||||
/// 1. Builds a `FieldTensor` (`Modality::WifiCsi`, axis `[Frequency]`) whose
|
||||
/// values are the RuView feature scalars, with the real `timestamp_ns`.
|
||||
/// 2. Builds an `Observation` — `motion_vector`/`range_m`/`space_cell` derived
|
||||
/// from the signal-field peak when present (else `None`; coordinates are
|
||||
/// never fabricated), `confidence` from the classification, labels from
|
||||
/// motion-level/presence.
|
||||
/// 3. Stamps the §3.3 egress privacy class (information-content mapping with
|
||||
/// the demotion floor) on both tensor and observation.
|
||||
/// 4. Builds a real `ProvenanceRef` (sha256 raw hash over the tensor/feature
|
||||
/// bytes, `synthetic = false`) and **signs** it with the supplied ed25519
|
||||
/// [`Signer`] so `rufield_provenance::is_fusable` passes.
|
||||
///
|
||||
/// Determinism: with no RNG anywhere and a deterministic ed25519 signer, the
|
||||
/// same `snap` + same signer seed yields a byte-identical event.
|
||||
#[must_use]
|
||||
pub fn snapshot_to_field_event(snap: &SensingSnapshot, signer: &Signer) -> FieldEvent {
|
||||
let class = egress_class(snap.trust_class, snap.identity_bound, snap.demoted);
|
||||
|
||||
let (range_m, motion_vector, space_cell) = derive_position(snap.signal_field.as_ref());
|
||||
|
||||
// ── 1. Tensor ──────────────────────────────────────────────────────────
|
||||
// The frequency-domain feature scalars, in a stable order.
|
||||
let f = &snap.features;
|
||||
let values: Vec<f32> = vec![
|
||||
f.mean_rssi as f32,
|
||||
f.variance as f32,
|
||||
f.motion_band_power as f32,
|
||||
f.breathing_band_power as f32,
|
||||
f.dominant_freq_hz as f32,
|
||||
f.spectral_power as f32,
|
||||
];
|
||||
let confidence = (snap.classification.confidence as f32).clamp(0.0, 1.0);
|
||||
let noise_floor = f.variance.max(0.0) as f32;
|
||||
let calibration_id = format!("ruview_node_{}", snap.node_id);
|
||||
|
||||
// `FieldTensor::new` only errors on a shape/axis mismatch; our shape
|
||||
// exactly matches `values.len()` and one axis, so this is infallible here.
|
||||
let tensor = FieldTensor::new(
|
||||
snap.timestamp_ns,
|
||||
Modality::WifiCsi,
|
||||
vec![FieldAxis::Frequency],
|
||||
vec![values.len()],
|
||||
values,
|
||||
confidence,
|
||||
noise_floor,
|
||||
Some(calibration_id.clone()),
|
||||
class,
|
||||
)
|
||||
.expect("feature tensor shape is well-formed by construction");
|
||||
|
||||
// ── 2. Observation ─────────────────────────────────────────────────────
|
||||
let observation = Observation {
|
||||
zone_id: Some(snap.node_id.clone()),
|
||||
space_cell,
|
||||
range_m,
|
||||
velocity_mps: None,
|
||||
motion_vector,
|
||||
confidence,
|
||||
features: build_features(snap, range_m),
|
||||
labels: build_labels(snap),
|
||||
privacy_class: class,
|
||||
};
|
||||
|
||||
// ── 3. Provenance (real sha256 over the tensor bytes) ───────────────────
|
||||
let raw_hash = sha256_hex(
|
||||
&serde_json::to_vec(&tensor).expect("tensor serializes to JSON for hashing"),
|
||||
);
|
||||
let provenance = ProvenanceRef {
|
||||
raw_hash,
|
||||
firmware_hash: firmware_hash(),
|
||||
model_id: MODEL_ID.to_string(),
|
||||
calibration_id,
|
||||
synthetic: false, // a real (non-synthetic) live/replay event
|
||||
signature_hex: None,
|
||||
signer_pubkey_hex: None,
|
||||
};
|
||||
|
||||
let sensor = SensorDescriptor {
|
||||
modality: "wifi_csi".to_string(),
|
||||
vendor: "esp32".to_string(),
|
||||
device_id: snap.node_id.clone(),
|
||||
placement: "unknown".to_string(),
|
||||
clock_domain: "local".to_string(),
|
||||
};
|
||||
|
||||
let mut event = FieldEvent::new(
|
||||
event_id(snap),
|
||||
snap.timestamp_ns,
|
||||
sensor,
|
||||
tensor,
|
||||
observation,
|
||||
provenance,
|
||||
);
|
||||
|
||||
// ── 4. Sign (ed25519) so `is_fusable` passes for this real event ────────
|
||||
signer
|
||||
.sign_event(&mut event)
|
||||
.expect("ed25519 signing of a serializable event is infallible");
|
||||
|
||||
event
|
||||
}
|
||||
|
||||
/// Labels from the classification. These are descriptive (`person_present`,
|
||||
/// `motion_<level>`); the RuField fusion engine never reads labels
|
||||
/// (`event.rs:45-48`), so this carries no identity.
|
||||
fn build_labels(snap: &SensingSnapshot) -> Vec<String> {
|
||||
let mut labels = Vec::new();
|
||||
if snap.classification.presence {
|
||||
labels.push("person_present".to_string());
|
||||
}
|
||||
labels.push(format!("motion_{}", snap.classification.motion_level));
|
||||
labels
|
||||
}
|
||||
|
||||
/// Convenience: the privacy class that *would* be stamped for a snapshot,
|
||||
/// without building the whole event. Useful for egress badges (P3) and tests.
|
||||
#[must_use]
|
||||
pub fn snapshot_egress_class(snap: &SensingSnapshot) -> PrivacyClass {
|
||||
egress_class(snap.trust_class, snap.identity_bound, snap.demoted)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//! # wifi-densepose-rufield
|
||||
//!
|
||||
//! ADR-262 **anti-corruption bridge**: converts RuView's live WiFi-CSI sensing
|
||||
//! output into signed RuField [`FieldEvent`](rufield_core::FieldEvent)s.
|
||||
//!
|
||||
//! This crate is the **single coupling point** (ADR-262 §5.4) between RuView and
|
||||
//! the standalone RuField MFS spec (`vendor/rufield`, ADR-260). It depends on
|
||||
//! the four pure-Rust rufield crates **via path** — `rufield-core`,
|
||||
//! `-provenance`, `-privacy`, `-fusion` — and on **no** RuView internal crate.
|
||||
//! Inputs are owned primitives ([`SensingSnapshot`]) that mirror what RuView's
|
||||
//! sensing cycle produces, so the bridge never imports `SensingUpdate` /
|
||||
//! `TrustedOutput` directly.
|
||||
//!
|
||||
//! ## What P1 ships (honesty — ADR-262 §0 / §6)
|
||||
//!
|
||||
//! This is **P1 plumbing**: a tested `SensingSnapshot → FieldEvent` conversion
|
||||
//! plus the **fail-closed privacy mapping** that is the §3.3 correctness item.
|
||||
//! It is **not** wired into the live server (that is P3) and makes **no accuracy
|
||||
//! claim** — RuField v0.1 is synthetic end-to-end and RuView's single-link CSI
|
||||
//! carries its own caveats. The gates here are round-trip / fusability /
|
||||
//! privacy-safety / determinism, not validated F1.
|
||||
//!
|
||||
//! ## The critical correctness item: the privacy mapping (§3.3)
|
||||
//!
|
||||
//! RuView's `Derived` class has byte value `1` (below `Anonymous = 2`) yet
|
||||
//! carries an identity embedding. The bridge maps it to **P4/P5 by information
|
||||
//! content, never P1** — see [`map_privacy`]. Mapping off the byte would leak
|
||||
//! identity as low-privacy; [`map_privacy`] (and its dedicated test
|
||||
//! `derived_identity_never_maps_to_low_privacy`) exist specifically to prevent
|
||||
//! that.
|
||||
//!
|
||||
//! ## Example
|
||||
//!
|
||||
//! ```
|
||||
//! use wifi_densepose_rufield::{
|
||||
//! snapshot_to_field_event, SensingSnapshot, SensingFeatures, SensingClass,
|
||||
//! RuViewPrivacyClass,
|
||||
//! };
|
||||
//! use rufield_provenance::{Signer, is_fusable};
|
||||
//!
|
||||
//! let snap = SensingSnapshot {
|
||||
//! timestamp_ns: 1_791_986_400_000_000_000,
|
||||
//! features: SensingFeatures {
|
||||
//! mean_rssi: -55.0,
|
||||
//! variance: 0.4,
|
||||
//! motion_band_power: 2.0,
|
||||
//! breathing_band_power: 0.3,
|
||||
//! dominant_freq_hz: 0.25,
|
||||
//! change_points: 1,
|
||||
//! spectral_power: 3.0,
|
||||
//! },
|
||||
//! classification: SensingClass {
|
||||
//! motion_level: "low".into(),
|
||||
//! presence: true,
|
||||
//! confidence: 0.82,
|
||||
//! },
|
||||
//! signal_field: None,
|
||||
//! trust_class: RuViewPrivacyClass::Anonymous,
|
||||
//! demoted: false,
|
||||
//! identity_bound: false,
|
||||
//! node_id: "esp32_room_01".into(),
|
||||
//! };
|
||||
//!
|
||||
//! let signer = Signer::from_seed(b"adr-262-bridge-seed-32-bytes-ok!");
|
||||
//! let event = snapshot_to_field_event(&snap, &signer);
|
||||
//! assert!(is_fusable(&event)); // ed25519-signed, non-synthetic ⇒ fusable
|
||||
//! ```
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod bridge;
|
||||
pub mod privacy;
|
||||
pub mod snapshot;
|
||||
|
||||
pub use bridge::{snapshot_egress_class, snapshot_to_field_event};
|
||||
pub use privacy::{apply_demotion_floor, egress_class, map_privacy};
|
||||
pub use snapshot::{
|
||||
RuViewPrivacyClass, SensingClass, SensingFeatures, SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
// Re-export the rufield surface a bridge consumer needs, so callers depend on
|
||||
// one crate.
|
||||
pub use rufield_core::{Destination, FieldEvent, Modality, PrivacyClass, PrivacyDecision};
|
||||
pub use rufield_fusion::RuFieldFusion;
|
||||
pub use rufield_privacy::{DefaultPrivacyGuard, PrivacyPolicy};
|
||||
pub use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
|
||||
/// Whether a mapped [`PrivacyClass`] may be surfaced on a **network** egress
|
||||
/// (ADR-262 §4 P3 — the live `/api/field` / `/ws/field` surface must respect
|
||||
/// the same default §10 network policy `/ws/sensing` honours, never emitting
|
||||
/// above-policy data).
|
||||
///
|
||||
/// **Fail-closed for a live, unattended surface.** The live RuView surface has
|
||||
/// **no per-event consent or identity-binding ceremony** — so this is *stricter*
|
||||
/// than [`DefaultPrivacyGuard::authorize`]: it requires BOTH that the default
|
||||
/// guard would `Allow` the class onto [`Destination::Network`] with **no consent
|
||||
/// granted**, AND that the class is at or below the default network ceiling
|
||||
/// ([`PrivacyClass::P2`]). The second clause deliberately drops P4/P5 even
|
||||
/// though the guard's consent/identity *exceptions* would let an explicitly
|
||||
/// consented/identity-bound P4/P5 through — because the live surface cannot
|
||||
/// honestly assert that consent. Net effect: only **P1/P2** leave the box; P0
|
||||
/// (raw) and P3/P4/P5 are held edge-local.
|
||||
///
|
||||
/// This is the privacy-safety pin for the live surface: a `Derived` cycle maps
|
||||
/// to P4 (or P5 when identity-bound) via [`map_privacy`] and is therefore
|
||||
/// **never** surfaced as a network event — neither as a low-privacy P1 (the
|
||||
/// §3.3 mapping trap) nor at all.
|
||||
#[must_use]
|
||||
pub fn network_egress_allowed(class: PrivacyClass, identity_bound: bool) -> bool {
|
||||
use rufield_core::PrivacyGuard;
|
||||
let guard_allows = matches!(
|
||||
DefaultPrivacyGuard::default().authorize(
|
||||
class,
|
||||
Destination::Network,
|
||||
false, // no per-event consent on the live network surface (fail-closed)
|
||||
identity_bound,
|
||||
),
|
||||
PrivacyDecision::Allow
|
||||
);
|
||||
// Additionally cap at the default network ceiling: an unattended live
|
||||
// surface never asserts the P4-consent / P5-identity exception.
|
||||
guard_allows && class <= PrivacyClass::P2
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//! The ADR-262 §3.3 privacy mapping — the critical correctness item.
|
||||
//!
|
||||
//! RuView's effective `PrivacyClass` (4 byte-level classes) is the source of
|
||||
//! truth; the bridge maps it onto RuField's `PrivacyClass` (P0–P5) **at the
|
||||
//! egress boundary, by information content, NEVER by byte value**.
|
||||
//!
|
||||
//! ## The trap (ADR-262 §3, §6)
|
||||
//!
|
||||
//! RuView's `Derived` has byte value `1`, which sorts *below* `Anonymous`
|
||||
//! (byte `2`). A naive byte-mapping (`Derived = 1 → P1`) would leak
|
||||
//! identity-bearing features (`identity_embedding`, `identity_risk_score`) as a
|
||||
//! **low-privacy P1** event. Because `Derived` carries derived *identity*, it
|
||||
//! must map to the **biometric/identity tier (P4/P5)** — never P1. This is the
|
||||
//! single most dangerous mapping mistake; it gets a dedicated test
|
||||
//! (`derived_identity_never_maps_to_low_privacy`).
|
||||
//!
|
||||
//! ## Fail-closed
|
||||
//!
|
||||
//! [`RuViewPrivacyClass`] is a closed enum, so there is no runtime "unknown"
|
||||
//! value to receive — but the mapping is written `match`-exhaustively with an
|
||||
//! explicit, documented arm per class, and the `demoted`/`identity_bound`
|
||||
//! overlays only ever move the result **toward more privacy**, never less.
|
||||
|
||||
use crate::snapshot::RuViewPrivacyClass;
|
||||
use rufield_core::PrivacyClass;
|
||||
|
||||
/// Map a RuView effective `PrivacyClass` onto a RuField `PrivacyClass`
|
||||
/// (ADR-262 §3.3), by information content.
|
||||
///
|
||||
/// | RuView (byte) | → RuField | Rationale |
|
||||
/// |---|---|---|
|
||||
/// | `Raw` (0) | `P0` | raw CSI waveform |
|
||||
/// | `Derived` (1) | `P4` (or `P5` if `identity_bound`) | derived **identity** features ⇒ biometric/identity tier, **not** P1 |
|
||||
/// | `Anonymous` (2) | `P2` | occupancy / motion only |
|
||||
/// | `Restricted` (3) | `P2` (raw suppressed) | matches `suppress_raw_outputs` |
|
||||
///
|
||||
/// `identity_bound` only promotes `Derived` (already identity-derived) from P4
|
||||
/// to P5; it can never lower the class.
|
||||
#[must_use]
|
||||
pub fn map_privacy(ruview_class: RuViewPrivacyClass, identity_bound: bool) -> PrivacyClass {
|
||||
match ruview_class {
|
||||
// Raw CSI amplitude → raw waveform tier.
|
||||
RuViewPrivacyClass::Raw => PrivacyClass::P0,
|
||||
|
||||
// THE CRITICAL ARM (§3.3 / §6): `Derived` carries identity. Map by
|
||||
// information content to the biometric/identity tier P4, and to P5 when
|
||||
// the surface is bound to a named identity. NEVER P1.
|
||||
RuViewPrivacyClass::Derived => {
|
||||
if identity_bound {
|
||||
PrivacyClass::P5
|
||||
} else {
|
||||
PrivacyClass::P4
|
||||
}
|
||||
}
|
||||
|
||||
// Anonymous occupancy / motion aggregate → P2.
|
||||
RuViewPrivacyClass::Anonymous => PrivacyClass::P2,
|
||||
|
||||
// Restricted: occupancy with risk score / hash stripped and raw
|
||||
// suppressed. Capped at P2 (occupancy tier), matching
|
||||
// `EngineBridge::suppress_raw_outputs` (`engine_bridge.rs:240`).
|
||||
RuViewPrivacyClass::Restricted => PrivacyClass::P2,
|
||||
}
|
||||
}
|
||||
|
||||
/// The §4 P2 gate (b) monotonicity overlay: a governed-engine **demotion**
|
||||
/// (`TrustedOutput.demoted == true`) must never let the emitted class fall
|
||||
/// below P2 (occupancy floor), and raw is suppressed.
|
||||
///
|
||||
/// This is applied *after* [`map_privacy`] and can only raise the class
|
||||
/// (toward more privacy) — it is fail-closed by construction.
|
||||
#[must_use]
|
||||
pub fn apply_demotion_floor(class: PrivacyClass, demoted: bool) -> PrivacyClass {
|
||||
if demoted && class < PrivacyClass::P2 {
|
||||
PrivacyClass::P2
|
||||
} else {
|
||||
class
|
||||
}
|
||||
}
|
||||
|
||||
/// The full egress class for a snapshot: information-content mapping with the
|
||||
/// demotion floor overlaid. This is what the bridge stamps on the emitted
|
||||
/// `FieldEvent`.
|
||||
#[must_use]
|
||||
pub fn egress_class(
|
||||
ruview_class: RuViewPrivacyClass,
|
||||
identity_bound: bool,
|
||||
demoted: bool,
|
||||
) -> PrivacyClass {
|
||||
apply_demotion_floor(map_privacy(ruview_class, identity_bound), demoted)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn derived_maps_to_identity_tier_not_p1() {
|
||||
// The single most dangerous mapping mistake: Derived (byte 1) must NOT
|
||||
// become P1. It carries identity ⇒ P4, or P5 if identity-bound.
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, true), PrivacyClass::P5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_table_matches_adr_262_section_3_3() {
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Raw, false), PrivacyClass::P0);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Derived, false), PrivacyClass::P4);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Anonymous, false), PrivacyClass::P2);
|
||||
assert_eq!(map_privacy(RuViewPrivacyClass::Restricted, false), PrivacyClass::P2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapping_ignores_non_monotonic_byte_value() {
|
||||
// Derived's byte (1) is *below* Anonymous's byte (2), but Derived's
|
||||
// mapped class must be *above* Anonymous's mapped class — proving the
|
||||
// mapping uses information content, not the byte.
|
||||
assert!(RuViewPrivacyClass::Derived.raw_byte() < RuViewPrivacyClass::Anonymous.raw_byte());
|
||||
assert!(
|
||||
map_privacy(RuViewPrivacyClass::Derived, false)
|
||||
> map_privacy(RuViewPrivacyClass::Anonymous, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demotion_floor_only_raises_privacy() {
|
||||
// Raw → P0, but a demoted cycle floors to P2 with raw suppressed.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, true), PrivacyClass::P2);
|
||||
// Already-high classes are never lowered by the floor.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P5, true), PrivacyClass::P5);
|
||||
// No demotion ⇒ unchanged.
|
||||
assert_eq!(apply_demotion_floor(PrivacyClass::P0, false), PrivacyClass::P0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_bound_only_promotes() {
|
||||
// identity_bound never lowers privacy; it only promotes Derived P4→P5.
|
||||
for c in [
|
||||
RuViewPrivacyClass::Raw,
|
||||
RuViewPrivacyClass::Derived,
|
||||
RuViewPrivacyClass::Anonymous,
|
||||
RuViewPrivacyClass::Restricted,
|
||||
] {
|
||||
assert!(map_privacy(c, true) >= map_privacy(c, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Owned, primitive input types for the ADR-262 bridge.
|
||||
//!
|
||||
//! These deliberately **mirror** the shapes RuView's sensing cycle produces
|
||||
//! (the `/ws/sensing` `SensingUpdate` build site at
|
||||
//! `wifi-densepose-sensing-server/src/main.rs:~5938` and the `TrustedOutput`
|
||||
//! trust state surfaced via `EngineBridge` at `main.rs:~5886`) **without
|
||||
//! importing** RuView's internal crates. Keeping the bridge an anti-corruption
|
||||
//! layer (ADR-262 §5.4) means it takes owned primitives, not `SensingUpdate`
|
||||
//! or `TrustedOutput` directly — so this crate never depends on
|
||||
//! `wifi-densepose-sensing-server`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The CSI feature scalars RuView publishes on every `/ws/sensing` cycle.
|
||||
///
|
||||
/// Mirrors `FeatureInfo` (`main.rs:368-377`). All values are in RuView's own
|
||||
/// units; the bridge normalizes them into `Observation.features` for fusion.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingFeatures {
|
||||
/// Mean RSSI across the CSI window (dBm).
|
||||
pub mean_rssi: f64,
|
||||
/// CSI amplitude variance.
|
||||
pub variance: f64,
|
||||
/// Motion-band spectral power (drives `motion_energy`).
|
||||
pub motion_band_power: f64,
|
||||
/// Breathing-band spectral power (drives `breathing_band`).
|
||||
pub breathing_band_power: f64,
|
||||
/// Dominant frequency of the CSI window (Hz).
|
||||
pub dominant_freq_hz: f64,
|
||||
/// Number of change points detected in the window (drives `transient`).
|
||||
pub change_points: usize,
|
||||
/// Total spectral power of the window.
|
||||
pub spectral_power: f64,
|
||||
}
|
||||
|
||||
/// The RuView classification block. Mirrors `ClassificationInfo`
|
||||
/// (`main.rs:379-384`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingClass {
|
||||
/// Coarse motion level label (e.g. `"none"`, `"low"`, `"high"`).
|
||||
pub motion_level: String,
|
||||
/// Whether a person is present.
|
||||
pub presence: bool,
|
||||
/// Classification confidence `0.0..=1.0`.
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
/// A RuView signal field — a floor-plane grid of field values. Mirrors
|
||||
/// `SignalField` (`main.rs:386-390`). The bridge derives a real position from
|
||||
/// the strongest field peak (like `field_localize`) and **never fabricates**
|
||||
/// coordinates when this is absent.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SignalField {
|
||||
/// Grid dimensions `[x, y, z]`.
|
||||
pub grid_size: [usize; 3],
|
||||
/// Row-major flattened field values; `len() == grid_size.product()`.
|
||||
pub values: Vec<f64>,
|
||||
}
|
||||
|
||||
impl SignalField {
|
||||
/// Index `[x, y, z]` of the strongest field cell, or `None` if the grid is
|
||||
/// empty / all-NaN. This is the honest "strongest field peak" readout that
|
||||
/// `field_localize` (`field_localize.rs:16-27`) exposes — **not** calibrated
|
||||
/// triangulation.
|
||||
#[must_use]
|
||||
pub fn peak_cell(&self) -> Option<[i32; 3]> {
|
||||
let [nx, ny, nz] = self.grid_size;
|
||||
if nx == 0 || ny == 0 || nz == 0 || self.values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut best_idx: Option<usize> = None;
|
||||
let mut best_val = f64::NEG_INFINITY;
|
||||
for (i, &v) in self.values.iter().enumerate() {
|
||||
if v.is_finite() && v > best_val {
|
||||
best_val = v;
|
||||
best_idx = Some(i);
|
||||
}
|
||||
}
|
||||
let idx = best_idx?;
|
||||
// Row-major: idx = ((x * ny) + y) * nz + z.
|
||||
let z = idx % nz;
|
||||
let y = (idx / nz) % ny;
|
||||
let x = idx / (nz * ny);
|
||||
Some([x as i32, y as i32, z as i32])
|
||||
}
|
||||
}
|
||||
|
||||
/// RuView's effective privacy class (the `effective_class` / privacy byte on
|
||||
/// `TrustedOutput`).
|
||||
///
|
||||
/// This **mirrors** `wifi_densepose_bfld::PrivacyClass` (`bfld/lib.rs:103-116`,
|
||||
/// `#[repr(u8)]`) — the four byte-level classes. The byte values are
|
||||
/// **deliberately non-monotonic in information content**: `Derived = 1` carries
|
||||
/// an identity embedding yet sorts *below* `Anonymous = 2`. The bridge's
|
||||
/// `map_privacy` must therefore map by information content, NEVER by byte value
|
||||
/// (ADR-262 §3.3 — the central correctness item).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RuViewPrivacyClass {
|
||||
/// Byte `0` — raw CSI amplitude, local-only.
|
||||
Raw,
|
||||
/// Byte `1` — derived **identity** features (identity_embedding +
|
||||
/// identity_risk_score), LAN-only. The dangerous one (§3.3).
|
||||
Derived,
|
||||
/// Byte `2` — aggregate occupancy / motion, no identity.
|
||||
Anonymous,
|
||||
/// Byte `3` — care/regulated: occupancy minus risk score and hash;
|
||||
/// raw suppressed.
|
||||
Restricted,
|
||||
}
|
||||
|
||||
impl RuViewPrivacyClass {
|
||||
/// The raw byte value used by RuView's `#[repr(u8)]` enum
|
||||
/// (`bfld/lib.rs:103`). Exposed only so callers can demonstrate the
|
||||
/// non-monotonicity trap in tests; the bridge never maps off this byte.
|
||||
#[must_use]
|
||||
pub fn raw_byte(self) -> u8 {
|
||||
match self {
|
||||
RuViewPrivacyClass::Raw => 0,
|
||||
RuViewPrivacyClass::Derived => 1,
|
||||
RuViewPrivacyClass::Anonymous => 2,
|
||||
RuViewPrivacyClass::Restricted => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One sensing cycle, as a bridge input. Mirrors the join of `SensingUpdate`
|
||||
/// (features + classification + signal_field) and the `TrustedOutput` trust
|
||||
/// state (`trust_class`) that ADR-262 §1.2 / P1 say must be done at the bridge.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SensingSnapshot {
|
||||
/// Capture time, nanoseconds since Unix epoch (the real `SensingUpdate`
|
||||
/// timestamp, ns).
|
||||
pub timestamp_ns: u64,
|
||||
/// CSI feature scalars (`/ws/sensing` feature set).
|
||||
pub features: SensingFeatures,
|
||||
/// Classification (motion level / presence / confidence).
|
||||
pub classification: SensingClass,
|
||||
/// Optional signal field for a real position readout.
|
||||
pub signal_field: Option<SignalField>,
|
||||
/// RuView's effective privacy class (the source-of-truth, §3.3).
|
||||
pub trust_class: RuViewPrivacyClass,
|
||||
/// Whether the governed engine demoted this cycle (`TrustedOutput.demoted`).
|
||||
/// When `true` the emitted event must be `>= P2` and raw suppressed
|
||||
/// (§3.3 / §4 P2 gate (b)).
|
||||
pub demoted: bool,
|
||||
/// Whether this cycle's identity surface is bound to an enrolled identity
|
||||
/// (RuView's `identity_bound`). Promotes `Derived` to P5 when set.
|
||||
pub identity_bound: bool,
|
||||
/// Stable node id (e.g. `"esp32_room_01"`).
|
||||
pub node_id: String,
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! ADR-262 P1 acceptance gates. Each test below IS an acceptance criterion.
|
||||
//!
|
||||
//! - round-trip: snapshot → FieldEvent → serde → equal
|
||||
//! - is_fusable: emitted event passes the §11 fusability invariant
|
||||
//! - fusion ingest accept: `RuFieldFusion::ingest` accepts it + `infer` runs
|
||||
//! - privacy safety: `Derived` never maps to a low-privacy class (the §3.3 trap)
|
||||
//! - determinism: same snapshot + same signer seed → identical event
|
||||
|
||||
use rufield_core::{FusionEngine, InferenceQuery, PrivacyClass};
|
||||
use rufield_fusion::RuFieldFusion;
|
||||
use rufield_provenance::{is_fusable, verify_event, Signer};
|
||||
use wifi_densepose_rufield::{
|
||||
map_privacy, snapshot_to_field_event, RuViewPrivacyClass, SensingClass, SensingFeatures,
|
||||
SensingSnapshot, SignalField,
|
||||
};
|
||||
|
||||
const SEED: &[u8; 32] = b"adr-262-bridge-seed-32-bytes-ok!";
|
||||
|
||||
fn signer() -> Signer {
|
||||
Signer::from_seed(SEED)
|
||||
}
|
||||
|
||||
/// A representative snapshot with a real signal field (so a position is derived).
|
||||
fn sample_snapshot() -> SensingSnapshot {
|
||||
SensingSnapshot {
|
||||
timestamp_ns: 1_791_986_400_123_456_789,
|
||||
features: SensingFeatures {
|
||||
mean_rssi: -52.5,
|
||||
variance: 0.73,
|
||||
motion_band_power: 2.4,
|
||||
breathing_band_power: 0.6,
|
||||
dominant_freq_hz: 0.27,
|
||||
change_points: 2,
|
||||
spectral_power: 4.1,
|
||||
},
|
||||
classification: SensingClass {
|
||||
motion_level: "high".into(),
|
||||
presence: true,
|
||||
confidence: 0.88,
|
||||
},
|
||||
signal_field: Some(SignalField {
|
||||
grid_size: [2, 1, 2],
|
||||
// peak at flat index 2 → cell [1,0,0]
|
||||
values: vec![0.1, 0.2, 0.9, 0.3],
|
||||
}),
|
||||
trust_class: RuViewPrivacyClass::Anonymous,
|
||||
demoted: false,
|
||||
identity_bound: false,
|
||||
node_id: "esp32_room_01".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_round_trip_serde_equal() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let json = serde_json::to_string(&ev).expect("serialize");
|
||||
let back: rufield_core::FieldEvent = serde_json::from_str(&json).expect("deserialize");
|
||||
assert_eq!(ev, back, "FieldEvent must round-trip through serde unchanged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_is_fusable_verified_receipt() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// Real (non-synthetic) event must carry a verifying ed25519 signature.
|
||||
assert!(!ev.provenance.synthetic, "live event must NOT be marked synthetic");
|
||||
assert!(ev.provenance.signature_hex.is_some(), "must be signed");
|
||||
assert!(verify_event(&ev).is_ok(), "signature must verify");
|
||||
assert!(is_fusable(&ev), "verified receipt ⇒ fusable (§11 invariant)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_fusion_ingest_accepts_and_infers() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
let mut engine = RuFieldFusion::new();
|
||||
engine.ingest(ev).expect("fusion engine must accept the signed event");
|
||||
// infer() must run without error (may or may not produce inferences).
|
||||
let inferences = engine
|
||||
.infer(&InferenceQuery::all())
|
||||
.expect("infer() must run");
|
||||
// The graph recorded the event/sensor provenance nodes.
|
||||
assert!(
|
||||
engine.graph().node_count() >= 2,
|
||||
"ingest should record sensor + event nodes"
|
||||
);
|
||||
let _ = inferences; // count is not an accuracy claim
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_privacy_safety_derived_never_maps_to_low_privacy() {
|
||||
// THE critical §3.3 gate. Derived carries identity ⇒ P4/P5, NEVER P1.
|
||||
let p4 = map_privacy(RuViewPrivacyClass::Derived, false);
|
||||
let p5 = map_privacy(RuViewPrivacyClass::Derived, true);
|
||||
assert_eq!(p4, PrivacyClass::P4);
|
||||
assert_eq!(p5, PrivacyClass::P5);
|
||||
assert!(p4 >= PrivacyClass::P4, "Derived must be in the identity tier");
|
||||
assert_ne!(p4, PrivacyClass::P1, "Derived must NEVER be P1");
|
||||
|
||||
// And end-to-end: an emitted event from a Derived snapshot must be P4/P5.
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Derived;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P4,
|
||||
"emitted Derived event must be P4 or P5, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
assert_eq!(ev.observation.privacy_class, ev.tensor.privacy_class);
|
||||
}
|
||||
|
||||
/// Full §3.3 table over every RuView class → expected RuField class.
|
||||
#[test]
|
||||
fn gate_privacy_table_over_every_ruview_class() {
|
||||
let cases = [
|
||||
(RuViewPrivacyClass::Raw, false, PrivacyClass::P0),
|
||||
(RuViewPrivacyClass::Derived, false, PrivacyClass::P4),
|
||||
(RuViewPrivacyClass::Derived, true, PrivacyClass::P5),
|
||||
(RuViewPrivacyClass::Anonymous, false, PrivacyClass::P2),
|
||||
(RuViewPrivacyClass::Restricted, false, PrivacyClass::P2),
|
||||
];
|
||||
for (ruview, id_bound, expected) in cases {
|
||||
assert_eq!(
|
||||
map_privacy(ruview, id_bound),
|
||||
expected,
|
||||
"{ruview:?} (identity_bound={id_bound}) must map to {expected:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fail-closed: a demoted Raw snapshot must NOT emit P0 (raw) — it floors to P2.
|
||||
#[test]
|
||||
fn gate_demotion_is_fail_closed() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.trust_class = RuViewPrivacyClass::Raw; // would be P0
|
||||
snap.demoted = true; // governed engine demotion
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(
|
||||
ev.observation.privacy_class >= PrivacyClass::P2,
|
||||
"demoted cycle must floor to >= P2, got {:?}",
|
||||
ev.observation.privacy_class
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_determinism_same_seed_identical_event() {
|
||||
let snap = sample_snapshot();
|
||||
let a = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
let b = snapshot_to_field_event(&snap, &Signer::from_seed(SEED));
|
||||
assert_eq!(a, b, "same snapshot + same signer seed ⇒ identical event");
|
||||
// Including the signature (ed25519 is deterministic).
|
||||
assert_eq!(a.provenance.signature_hex, b.provenance.signature_hex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fabricated_position_when_field_absent() {
|
||||
let mut snap = sample_snapshot();
|
||||
snap.signal_field = None;
|
||||
let ev = snapshot_to_field_event(&snap, &signer());
|
||||
assert!(ev.observation.range_m.is_none(), "no field ⇒ no fabricated range");
|
||||
assert!(ev.observation.space_cell.is_none(), "no field ⇒ no fabricated cell");
|
||||
assert!(
|
||||
ev.observation.motion_vector.is_none(),
|
||||
"no field ⇒ no fabricated motion vector"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_real_position_from_field_peak() {
|
||||
let ev = snapshot_to_field_event(&sample_snapshot(), &signer());
|
||||
// peak at flat index 2, grid [2,1,2] (row-major) → cell [1,0,0]
|
||||
assert_eq!(ev.observation.space_cell, Some([1, 0, 0]));
|
||||
assert_eq!(ev.observation.range_m, Some(1.0));
|
||||
}
|
||||
@@ -47,3 +47,7 @@ harness = false
|
||||
[[bench]]
|
||||
name = "fusion_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "ann_bench"
|
||||
harness = false
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Criterion bench for the ADR-261 graph-ANN index: linear scan vs float HNSW
|
||||
//! vs quantized HNSW, on the shared `ann_measure` fixture.
|
||||
//!
|
||||
//! The authoritative recall/QPS numbers in ADR-261 come from the
|
||||
//! `--no-default-features --release` test report
|
||||
//! (`ann_bench_report` in `src/ann_measure.rs`), which is deterministic and
|
||||
//! gate-runnable. This criterion bench times the same operations through the
|
||||
//! criterion harness for stable per-op medians:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo bench -p wifi-densepose-ruvector --bench ann_bench
|
||||
//! ```
|
||||
//!
|
||||
//! Build is excluded from the timed region (done once in setup); only the query
|
||||
//! path is measured. The fixture and both indices are identical to the report's,
|
||||
//! so the bench and the report can never measure different graphs.
|
||||
|
||||
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||
use wifi_densepose_ruvector::ann_measure::{
|
||||
build_indices, build_quant_bits, queries, AnnBenchParams,
|
||||
};
|
||||
|
||||
fn bench_ann(c: &mut Criterion) {
|
||||
// Modest N so the bench builds quickly; the report covers the larger N.
|
||||
let p = AnnBenchParams::default_fixture(10_000);
|
||||
let (float_idx, quant_idx, vectors) = build_indices(p);
|
||||
// Multi-bit quant variants over the SAME graph/fixture (ADR-261 §11).
|
||||
let quant_2bit = build_quant_bits(p, &vectors, 2);
|
||||
let quant_4bit = build_quant_bits(p, &vectors, 4);
|
||||
let qs = queries(p);
|
||||
let k = p.k;
|
||||
|
||||
let mut group = c.benchmark_group("ann_query");
|
||||
group.sample_size(20);
|
||||
|
||||
// Linear scan (brute force) — the no-index baseline.
|
||||
group.bench_function("linear_scan", |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.brute_force(black_box(q), k).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
|
||||
// Float HNSW at a mid beam width.
|
||||
for &ef in &[64usize, 128] {
|
||||
group.bench_function(format!("float_hnsw_ef{ef}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink.wrapping_add(float_idx.search(black_box(q), k, ef).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Quantized HNSW (1-bit) at matched beam widths + rerank.
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_1bit_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(quant_idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Multi-bit quant HNSW (ADR-261 §11): 2-bit and 4-bit traversal codes at a
|
||||
// mid beam width, so the criterion medians show the per-bit QPS cost the
|
||||
// scaling study reports against recall.
|
||||
for (label, idx) in [("2bit", &quant_2bit), ("4bit", &quant_4bit)] {
|
||||
for &ef in &[64usize, 128] {
|
||||
let rr = k * 5;
|
||||
group.bench_function(format!("quant_hnsw_{label}_ef{ef}_rr{rr}"), |b| {
|
||||
b.iter(|| {
|
||||
let mut sink = 0u64;
|
||||
for q in &qs {
|
||||
sink = sink
|
||||
.wrapping_add(idx.search_quantized(black_box(q), k, ef, rr).len() as u64);
|
||||
}
|
||||
black_box(sink)
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_ann);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,684 @@
|
||||
//! Deterministic, `--no-default-features`-runnable **ANN benchmark measurement**
|
||||
//! for ADR-261 — the single source of truth for the QPS/recall numbers the ADR
|
||||
//! quotes for **linear scan**, **float HNSW**, and **quantized HNSW**.
|
||||
//!
|
||||
//! Both the criterion bench (`benches/ann_bench.rs`) and the in-crate report test
|
||||
//! ([`tests::ann_bench_report`]) call into here, so they can never silently
|
||||
//! measure different things. The numbers in ADR-261 §6 come from running:
|
||||
//!
|
||||
//! ```text
|
||||
//! cd v2 && cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
//! ann_bench_report -- --nocapture
|
||||
//! ```
|
||||
//!
|
||||
//! # What is measured, and the honesty contract
|
||||
//!
|
||||
//! On one fixed planted-cluster fixture (documented dim/N/K/seed), for each
|
||||
//! method we measure:
|
||||
//! - **recall@10** vs the brute-force exact top-10 (the ground truth),
|
||||
//! - **QPS** = queries / total wall-clock query time (warm; build excluded),
|
||||
//! at matched recall operating points found by sweeping `ef` (HNSW) and
|
||||
//! `(ef, rerank)` (quantized).
|
||||
//!
|
||||
//! The reported **ratio** is the claim, not the absolute QPS (which is
|
||||
//! machine-specific). We do **not** tune the quantized path to manufacture a
|
||||
//! win: if at our scale quantized does not beat float HNSW, the report says so
|
||||
//! and the ADR records the honest negative + the expected larger-N crossover.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::hnsw::{HnswIndex, HnswParams, Metric};
|
||||
use crate::hnsw_quantized::QuantizedHnswIndex;
|
||||
|
||||
/// SplitMix64 — the crate-wide deterministic PRNG (mirrors `coverage.rs`).
|
||||
#[inline]
|
||||
fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
#[inline]
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
((split_mix64(state) >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
#[inline]
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// ANN benchmark fixture parameters, documented in the ADR-261 report.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AnnBenchParams {
|
||||
/// Embedding dimension.
|
||||
pub dim: usize,
|
||||
/// Number of indexed vectors (N).
|
||||
pub n: usize,
|
||||
/// Number of planted clusters (near-neighbour structure).
|
||||
pub clusters: usize,
|
||||
/// Number of queries timed.
|
||||
pub n_queries: usize,
|
||||
/// Top-K.
|
||||
pub k: usize,
|
||||
/// Intra-cluster Gaussian jitter.
|
||||
pub noise: f32,
|
||||
/// Master fixture seed.
|
||||
pub seed: u64,
|
||||
/// Graph construction/level seed.
|
||||
pub graph_seed: u64,
|
||||
/// Rotation seed for the quantized 1-bit codes.
|
||||
pub rot_seed: u64,
|
||||
}
|
||||
|
||||
impl AnnBenchParams {
|
||||
/// The default ADR-261 fixture: AETHER-shape 128-d, planted clusters.
|
||||
pub fn default_fixture(n: usize) -> Self {
|
||||
Self {
|
||||
dim: 128,
|
||||
n,
|
||||
clusters: 64,
|
||||
n_queries: 200,
|
||||
k: 10,
|
||||
noise: 0.35,
|
||||
seed: 0xADADADAD_0000_0261,
|
||||
graph_seed: 0x6261_5247_4148_4E53,
|
||||
rot_seed: 0x5EED_C0DE_1234_5678,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The fixture vectors for `p` (deterministic planted clusters).
|
||||
pub fn fixture(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n)
|
||||
.map(|i| {
|
||||
let c = i % p.clusters;
|
||||
let mut s = p.seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The timed query set for `p` (drawn from the same clusters, disjoint seed).
|
||||
pub fn queries(p: AnnBenchParams) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..p.clusters)
|
||||
.map(|c| {
|
||||
let mut s = p.seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..p.dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..p.n_queries)
|
||||
.map(|q| {
|
||||
let c = q % p.clusters;
|
||||
let mut s = p.seed ^ 0xDEAD_0000_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
(0..p.dim)
|
||||
.map(|d| centres[c][d] + gauss(&mut s) * p.noise)
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-method measurement: recall@K and QPS.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct MethodResult {
|
||||
/// Mean recall@K vs brute-force ground truth.
|
||||
pub recall: f64,
|
||||
/// Queries per second (warm wall-clock).
|
||||
pub qps: f64,
|
||||
/// Mean query latency in microseconds.
|
||||
pub latency_us: f64,
|
||||
}
|
||||
|
||||
/// Ground-truth brute-force top-K id sets for every query (computed once).
|
||||
/// Public so the criterion bench and the report test share one definition.
|
||||
pub fn ground_truth(idx: &HnswIndex, queries: &[Vec<f32>], k: usize) -> Vec<HashSet<u32>> {
|
||||
queries
|
||||
.iter()
|
||||
.map(|q| idx.brute_force(q, k).into_iter().map(|(id, _)| id).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Measure **linear scan** (brute force): recall is 1.0 by definition; QPS is the
|
||||
/// timed exact scan. This is the no-index baseline.
|
||||
pub fn measure_linear(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.brute_force(q, k);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **float HNSW** at a given beam width `ef`.
|
||||
pub fn measure_float_hnsw(
|
||||
idx: &HnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = idx.search(q, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure **quantized HNSW** at a given `(ef, rerank)`.
|
||||
pub fn measure_quantized_hnsw(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
queries: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
rerank: usize,
|
||||
) -> MethodResult {
|
||||
let mut recall_acc = 0.0f64;
|
||||
let start = Instant::now();
|
||||
let mut sink = 0u64;
|
||||
for (qi, q) in queries.iter().enumerate() {
|
||||
let got = qidx.search_quantized(q, k, ef, rerank);
|
||||
let hit = got.iter().filter(|(id, _)| truth[qi].contains(id)).count();
|
||||
recall_acc += hit as f64 / k as f64;
|
||||
sink = sink.wrapping_add(got.len() as u64);
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
std::hint::black_box(sink);
|
||||
MethodResult {
|
||||
recall: recall_acc / queries.len() as f64,
|
||||
qps: queries.len() as f64 / elapsed,
|
||||
latency_us: elapsed / queries.len() as f64 * 1e6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build both indices for `p` (shared insertion order + graph seed so the float
|
||||
/// and quantized graphs are identical — the only variable is scoring). The
|
||||
/// quantized index uses the legacy **1-bit** code (ADR-261 §6); use
|
||||
/// [`build_indices_bits`] for the multi-bit scaling study (§11).
|
||||
pub fn build_indices(p: AnnBenchParams) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
build_indices_bits(p, 1)
|
||||
}
|
||||
|
||||
/// Build the float HNSW + a `bits`-bit quantized HNSW over the same fixture,
|
||||
/// sharing the graph seed and insertion order so the *only* variable between the
|
||||
/// float and quantized search is the traversal score. `bits ∈ {1, 2, 4}` (clamped
|
||||
/// in [`QuantizedHnswIndex::build_bits`]). The float index is **independent of
|
||||
/// `bits`** — callers sweeping `bits` should build the float index once and reuse
|
||||
/// it (the quantized graph is identical across `bits`; only the per-node code
|
||||
/// changes).
|
||||
pub fn build_indices_bits(
|
||||
p: AnnBenchParams,
|
||||
bits: u32,
|
||||
) -> (HnswIndex, QuantizedHnswIndex, Vec<Vec<f32>>) {
|
||||
let vectors = fixture(p);
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
let mut float_idx = HnswIndex::new(p.dim, Metric::L2, params);
|
||||
for v in &vectors {
|
||||
float_idx.insert(v);
|
||||
}
|
||||
let quant_idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
p.dim,
|
||||
Metric::L2,
|
||||
params,
|
||||
p.rot_seed,
|
||||
bits,
|
||||
p.k * 4,
|
||||
);
|
||||
(float_idx, quant_idx, vectors)
|
||||
}
|
||||
|
||||
/// Build only the `bits`-bit quantized index for `p`, reusing a fixture the
|
||||
/// caller already has (avoids regenerating `N×dim` floats per bit-depth in the
|
||||
/// scaling sweep). The graph seed/insertion order match [`build_indices_bits`],
|
||||
/// so this quantized graph is identical to that one's at the same `p`.
|
||||
pub fn build_quant_bits(p: AnnBenchParams, vectors: &[Vec<f32>], bits: u32) -> QuantizedHnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: p.graph_seed,
|
||||
};
|
||||
QuantizedHnswIndex::build_bits(vectors, p.dim, Metric::L2, params, p.rot_seed, bits, p.k * 4)
|
||||
}
|
||||
|
||||
/// The fastest operating point of a method that meets `target` recall, as
|
||||
/// `(qps, recall, label)`; `None` if no swept op met it.
|
||||
type BestOp = Option<(f64, f64, String)>;
|
||||
|
||||
/// Sweep float HNSW over a fixed `ef` ladder; return the fastest op meeting
|
||||
/// `target` recall.
|
||||
pub fn best_float_op(
|
||||
idx: &HnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> BestOp {
|
||||
let mut best: BestOp = None;
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(idx, qs, truth, k, ef);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef}")));
|
||||
}
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Sweep quant HNSW over a fixed `(ef, rerank)` ladder; return the fastest op
|
||||
/// meeting `target` recall, plus the best recall reached anywhere on the ladder
|
||||
/// (so a not-found verdict can report how close it got).
|
||||
pub fn best_quant_op(
|
||||
qidx: &QuantizedHnswIndex,
|
||||
qs: &[Vec<f32>],
|
||||
truth: &[HashSet<u32>],
|
||||
k: usize,
|
||||
target: f64,
|
||||
) -> (BestOp, f64) {
|
||||
let mut best: BestOp = None;
|
||||
let mut best_recall_seen = 0.0f64;
|
||||
for &ef in &[32usize, 64, 128, 256, 512] {
|
||||
for &rr in &[k * 2, k * 5, k * 10, k * 20] {
|
||||
let r = measure_quantized_hnsw(qidx, qs, truth, k, ef, rr);
|
||||
best_recall_seen = best_recall_seen.max(r.recall);
|
||||
if r.recall >= target && best.as_ref().map(|b| r.qps > b.0).unwrap_or(true) {
|
||||
best = Some((r.qps, r.recall, format!("ef={ef} rr={rr}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
(best, best_recall_seen)
|
||||
}
|
||||
|
||||
/// One row of the ADR-261 §11 scaling study: at a fixed `(N, b)`, the equal-recall
|
||||
/// (≥ `target`) operating points for float vs quant HNSW and their QPS ratio.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScalingRow {
|
||||
/// Indexed vector count.
|
||||
pub n: usize,
|
||||
/// Traversal-code bit-depth (1, 2, or 4).
|
||||
pub bits: u32,
|
||||
/// Packed bytes per node of the quant code at this `b`.
|
||||
pub bytes_per_node: usize,
|
||||
/// Fastest float-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub float_op: BestOp,
|
||||
/// Fastest quant-HNSW op meeting `target` recall (qps, recall, label).
|
||||
pub quant_op: BestOp,
|
||||
/// Best recall the quant ladder reached at this `(N, b)` (≤ `target` ⇒ no op).
|
||||
pub quant_best_recall: f64,
|
||||
/// quant/float QPS ratio at equal recall, if both met `target`.
|
||||
pub ratio: Option<f64>,
|
||||
}
|
||||
|
||||
/// Run the ADR-261 §11 multi-bit scaling study: for each `N ∈ ns` and each
|
||||
/// `b ∈ bits_set`, measure the equal-recall (≥ `target`) QPS ratio of quant-HNSW
|
||||
/// vs float-HNSW on the shared fixture. Deterministic and `--no-default-features`
|
||||
/// runnable. Returns one [`ScalingRow`] per `(N, b)`; the caller prints the table
|
||||
/// and decides the crossover verdict. The float index is built once per `N` and
|
||||
/// reused across `b` (the quant graph is identical across `b`).
|
||||
pub fn run_scaling_study(
|
||||
base: AnnBenchParams,
|
||||
ns: &[usize],
|
||||
bits_set: &[u32],
|
||||
target: f64,
|
||||
) -> Vec<ScalingRow> {
|
||||
let mut rows = Vec::new();
|
||||
for &n in ns {
|
||||
let p = AnnBenchParams { n, ..base };
|
||||
let (float_idx, _q1, vectors) = build_indices_bits(p, 1);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let float_op = best_float_op(&float_idx, &qs, &truth, p.k, target);
|
||||
for &b in bits_set {
|
||||
let qidx = build_quant_bits(p, &vectors, b);
|
||||
let (quant_op, quant_best_recall) =
|
||||
best_quant_op(&qidx, &qs, &truth, p.k, target);
|
||||
let ratio = match (&float_op, &quant_op) {
|
||||
(Some((fqps, _, _)), Some((qqps, _, _))) => Some(qqps / fqps),
|
||||
_ => None,
|
||||
};
|
||||
rows.push(ScalingRow {
|
||||
n,
|
||||
bits: qidx.bits(),
|
||||
bytes_per_node: qidx.bytes_per_node(),
|
||||
float_op: float_op.clone(),
|
||||
quant_op,
|
||||
quant_best_recall,
|
||||
ratio,
|
||||
});
|
||||
}
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fixture_and_queries_are_deterministic() {
|
||||
let p = AnnBenchParams::default_fixture(500);
|
||||
assert_eq!(fixture(p), fixture(p));
|
||||
assert_eq!(queries(p), queries(p));
|
||||
let p2 = AnnBenchParams {
|
||||
seed: p.seed ^ 1,
|
||||
..p
|
||||
};
|
||||
assert_ne!(fixture(p)[0], fixture(p2)[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linear_recall_is_one() {
|
||||
// Linear scan IS the ground truth, so recall must be exactly 1.0.
|
||||
let p = AnnBenchParams::default_fixture(800);
|
||||
let (float_idx, _q, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
let r = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
assert!((r.recall - 1.0).abs() < 1e-9, "linear recall {} != 1.0", r.recall);
|
||||
assert!(r.qps > 0.0);
|
||||
}
|
||||
|
||||
/// The ADR-261 measurement report. Prints the linear / float-HNSW /
|
||||
/// quantized-HNSW recall@10 + QPS table and the QPS ratios at matched recall.
|
||||
/// Run with `--release --nocapture` for the numbers the ADR quotes.
|
||||
#[test]
|
||||
fn ann_bench_report() {
|
||||
// N here is the small/CI-friendly default so the standard (debug) test
|
||||
// gate stays fast; the ADR's headline numbers are taken at the larger N
|
||||
// under --release (documented in the ADR with the exact command). This
|
||||
// test asserts only structural invariants so it is gate-safe at any N.
|
||||
let n: usize = std::env::var("ANN_BENCH_N")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(10_000);
|
||||
let p = AnnBenchParams::default_fixture(n);
|
||||
let (float_idx, quant_idx, _v) = build_indices(p);
|
||||
let qs = queries(p);
|
||||
let truth = ground_truth(&float_idx, &qs, p.k);
|
||||
|
||||
println!("\n=== ADR-261 ANN benchmark (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} N={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
p.dim, p.n, p.clusters, p.n_queries, p.k, p.noise, p.graph_seed, p.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 (debug build unless --release)");
|
||||
println!(
|
||||
"{:<28} {:>9} {:>12} {:>12}",
|
||||
"method", "recall@10", "QPS", "lat(us)"
|
||||
);
|
||||
|
||||
let lin = measure_linear(&float_idx, &qs, &truth, p.k);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
"linear scan (brute)", lin.recall, lin.qps, lin.latency_us
|
||||
);
|
||||
|
||||
// Float HNSW across an ef sweep.
|
||||
let mut float_ops: Vec<(usize, MethodResult)> = Vec::new();
|
||||
for &ef in &[16usize, 32, 64, 128, 256] {
|
||||
let r = measure_float_hnsw(&float_idx, &qs, &truth, p.k, ef);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("float-HNSW ef={ef}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
float_ops.push((ef, r));
|
||||
}
|
||||
|
||||
// Quantized HNSW across (ef, rerank) sweep.
|
||||
let mut quant_ops: Vec<((usize, usize), MethodResult)> = Vec::new();
|
||||
for &ef in &[32usize, 64, 128, 256] {
|
||||
for &rr in &[p.k * 2, p.k * 5, p.k * 10] {
|
||||
let r = measure_quantized_hnsw(&quant_idx, &qs, &truth, p.k, ef, rr);
|
||||
println!(
|
||||
"{:<28} {:>8.4} {:>12.1} {:>12.1}",
|
||||
format!("quant-HNSW ef={ef} rr={rr}"),
|
||||
r.recall,
|
||||
r.qps,
|
||||
r.latency_us
|
||||
);
|
||||
quant_ops.push(((ef, rr), r));
|
||||
}
|
||||
}
|
||||
|
||||
// Equal-recall comparison: pick, for a target recall, the FASTEST op of
|
||||
// each method that meets it, then report the QPS ratios.
|
||||
println!("\n--- equal-recall QPS ratios ---");
|
||||
for &target in &[0.90f64, 0.95, 0.99] {
|
||||
let best_float = float_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
let best_quant = quant_ops
|
||||
.iter()
|
||||
.filter(|(_, r)| r.recall >= target)
|
||||
.max_by(|a, b| a.1.qps.partial_cmp(&b.1.qps).unwrap());
|
||||
match (best_float, best_quant) {
|
||||
(Some((fef, fr)), Some(((qef, qrr), qr))) => {
|
||||
let ratio = qr.qps / fr.qps;
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant ef={} rr={} {:.0} QPS | quant/float={:.2}x | float/linear={:.2}x",
|
||||
target, fef, fr.qps, qef, qrr, qr.qps, ratio, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
(Some((fef, fr)), None) => {
|
||||
let hnsw_vs_lin = fr.qps / lin.qps;
|
||||
println!(
|
||||
"recall>={:.2}: float ef={} {:.0} QPS | quant: NO op met this recall | float/linear={:.2}x",
|
||||
target, fef, fr.qps, hnsw_vs_lin
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
println!("recall>={:.2}: neither method met this recall at the swept ops", target);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("=========================================================\n");
|
||||
|
||||
// Structural assertions (gate-safe, any N):
|
||||
// - linear scan is exact,
|
||||
// - the best float-HNSW op clears the correctness gate,
|
||||
// - quantized's best op is at least useful (recall well above random).
|
||||
assert!((lin.recall - 1.0).abs() < 1e-9);
|
||||
let best_float_recall = float_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
assert!(
|
||||
best_float_recall >= 0.95,
|
||||
"best float-HNSW recall {best_float_recall:.4} below 0.95 gate"
|
||||
);
|
||||
let best_quant_recall = quant_ops.iter().map(|(_, r)| r.recall).fold(0.0, f64::max);
|
||||
// Honest floor: the 1-bit Hamming traversal is a COARSE angle proxy, so
|
||||
// at large N its best recall lands well below the float gate (MEASURED
|
||||
// ~0.74 at N=10k — see ADR-261 §6). We assert only that it is clearly
|
||||
// useful (>> random: random top-10 of N=10k is ~0.001), which catches a
|
||||
// fully-broken traversal/rerank without pretending the quantized variant
|
||||
// matches float HNSW. The honest negative IS the result.
|
||||
assert!(
|
||||
best_quant_recall >= 0.30,
|
||||
"best quant-HNSW recall {best_quant_recall:.4} below the 0.30 not-broken floor"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ADR-261 §11 **multi-bit scaling study**. Sweeps `N` and `b ∈ {1,2,4}`,
|
||||
/// printing the `(N, b) → recall / QPS / quant-vs-float ratio at equal recall`
|
||||
/// surface and the crossover verdict. This is the source of truth for the §11
|
||||
/// table. Run for the published numbers with:
|
||||
///
|
||||
/// ```text
|
||||
/// cd v2 && ANN_SCALE_NS=10000,100000,250000 \
|
||||
/// cargo test -p wifi-densepose-ruvector --no-default-features --release \
|
||||
/// scaling_report -- --nocapture --ignored
|
||||
/// ```
|
||||
///
|
||||
/// Marked `#[ignore]` so the default (debug) gate stays fast: it builds and
|
||||
/// queries several indices up to large `N`, which is minutes under `--release`
|
||||
/// and far too slow in debug. The CI-safe structural invariants are checked by
|
||||
/// `scaling_study_small_is_consistent` below at tiny `N`.
|
||||
#[test]
|
||||
#[ignore = "scaling study — run explicitly with --release --ignored; minutes at large N"]
|
||||
fn scaling_report() {
|
||||
// N ladder: default 10k→100k→250k (a clean 25× span that builds+queries in
|
||||
// a few minutes under --release on the test box). Override with
|
||||
// ANN_SCALE_NS=a,b,c. The largest feasible N is documented in the ADR with
|
||||
// the measured build/query time at the cap.
|
||||
let ns: Vec<usize> = std::env::var("ANN_SCALE_NS")
|
||||
.ok()
|
||||
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
|
||||
.unwrap_or_else(|| vec![10_000, 100_000, 250_000]);
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let target = 0.90f64;
|
||||
let base = AnnBenchParams::default_fixture(ns[0]);
|
||||
|
||||
println!("\n=== ADR-261 §11 multi-bit scaling study (planted-cluster synthetic) ===");
|
||||
println!(
|
||||
"dim={} clusters={} queries={} K={} noise={} graph_seed=0x{:X} rot_seed=0x{:X}",
|
||||
base.dim, base.clusters, base.n_queries, base.k, base.noise, base.graph_seed, base.rot_seed
|
||||
);
|
||||
println!("metric=L2 M=16 ef_construction=200 target recall >= {target:.2} (use --release for QPS)");
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10} {:>22} {:>22} {:>12}",
|
||||
"N", "bits", "B/node", "q_best_rec", "float@target", "quant@target", "quant/float"
|
||||
);
|
||||
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, target);
|
||||
for row in &rows {
|
||||
let float_s = row
|
||||
.float_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let quant_s = row
|
||||
.quant_op
|
||||
.as_ref()
|
||||
.map(|(q, r, l)| format!("{l} {q:.0}QPS r={r:.3}"))
|
||||
.unwrap_or_else(|| "none".to_string());
|
||||
let ratio_s = row
|
||||
.ratio
|
||||
.map(|x| format!("{x:.2}x"))
|
||||
.unwrap_or_else(|| "—".to_string());
|
||||
println!(
|
||||
"{:<9} {:>4} {:>9} {:>10.3} {:>22} {:>22} {:>12}",
|
||||
row.n, row.bits, row.bytes_per_node, row.quant_best_recall, float_s, quant_s, ratio_s
|
||||
);
|
||||
}
|
||||
|
||||
// Crossover verdict: report whether the quant/float ratio EVER exceeds 1.0
|
||||
// at equal recall, and the per-bit trend of the best-quant-recall as N grows
|
||||
// (is quant getting closer to the equal-recall regime, or not).
|
||||
println!("\n--- crossover verdict (quant-HNSW > float-HNSW at equal recall?) ---");
|
||||
let crossover: Vec<&ScalingRow> = rows
|
||||
.iter()
|
||||
.filter(|r| r.ratio.map(|x| x > 1.0).unwrap_or(false))
|
||||
.collect();
|
||||
if crossover.is_empty() {
|
||||
println!("NO crossover at any measured (N, b): quant never met target recall AND beat float QPS.");
|
||||
} else {
|
||||
for r in &crossover {
|
||||
println!(
|
||||
"CROSSOVER at N={} b={}: quant/float = {:.2}x at recall >= {target:.2}",
|
||||
r.n, r.bits, r.ratio.unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
for &b in &bits_set {
|
||||
let trend: Vec<(usize, f64)> = rows
|
||||
.iter()
|
||||
.filter(|r| r.bits == b)
|
||||
.map(|r| (r.n, r.quant_best_recall))
|
||||
.collect();
|
||||
let trend_s: Vec<String> = trend
|
||||
.iter()
|
||||
.map(|(n, r)| format!("N={n}:{r:.3}"))
|
||||
.collect();
|
||||
println!("b={b} best-quant-recall trend: {}", trend_s.join(" "));
|
||||
}
|
||||
println!("======================================================================\n");
|
||||
|
||||
// Structural invariants (gate-safe at any N): at least one float op met
|
||||
// target at every N (the baseline must work), and quant recall is in range.
|
||||
for &n in &ns {
|
||||
let any_float = rows.iter().any(|r| r.n == n && r.float_op.is_some());
|
||||
assert!(any_float, "no float-HNSW op met target recall at N={n} — baseline broken");
|
||||
}
|
||||
for r in &rows {
|
||||
assert!(
|
||||
(0.0..=1.0).contains(&r.quant_best_recall),
|
||||
"quant recall out of range at N={} b={}: {}",
|
||||
r.n,
|
||||
r.bits,
|
||||
r.quant_best_recall
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// CI-safe structural check for the scaling study at tiny `N` (debug-fast):
|
||||
/// the study runs end-to-end, bytes/node scales with `b`, and the float
|
||||
/// baseline meets target at the smallest N. Does **not** assert any crossover
|
||||
/// (that is the §11 measured question, answered by `scaling_report`).
|
||||
#[test]
|
||||
fn scaling_study_small_is_consistent() {
|
||||
let base = AnnBenchParams::default_fixture(1500);
|
||||
let ns = [1500usize, 3000];
|
||||
let bits_set = [1u32, 2, 4];
|
||||
let rows = run_scaling_study(base, &ns, &bits_set, 0.90);
|
||||
assert_eq!(rows.len(), ns.len() * bits_set.len());
|
||||
// Bytes/node scales with b at dim=128 (D=128): 16 / 32 / 64.
|
||||
for r in rows.iter().filter(|r| r.n == 1500) {
|
||||
let expect = match r.bits {
|
||||
1 => 16,
|
||||
2 => 32,
|
||||
_ => 64,
|
||||
};
|
||||
assert_eq!(r.bytes_per_node, expect, "B/node wrong for b={}", r.bits);
|
||||
}
|
||||
// Float baseline must meet target at the smallest N.
|
||||
assert!(
|
||||
rows.iter().any(|r| r.n == 1500 && r.float_op.is_some()),
|
||||
"float baseline failed target at small N"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,826 @@
|
||||
//! A correct, dependency-free **float HNSW** graph-ANN index — ADR-261.
|
||||
//!
|
||||
//! # Why this exists
|
||||
//!
|
||||
//! The ruvector crate's retrieval path (AETHER re-ID hot-cache, the `sketch.rs`
|
||||
//! 1-bit prefilter, room fingerprinting) is, at its core, an **approximate
|
||||
//! nearest-neighbour** problem: dense float embedding in, top-K similar ids out.
|
||||
//! Until now the crate had **no graph index** — every `topk` was a linear scan
|
||||
//! (`O(N·d)` per query) or a 1-bit Hamming prefilter over a linear scan. That is
|
||||
//! fine at the small N the unit fixtures use, but it is `O(N)` per query and does
|
||||
//! not scale.
|
||||
//!
|
||||
//! [ADR-156 §5 #1](../../../../../docs/adr/ADR-156-ruvector-fusion-beyond-sota.md)
|
||||
//! lists **SymphonyQG** (SIGMOD 2025) as the lead beyond-SOTA ANN candidate,
|
||||
//! claiming **3.5–17× QPS over HNSW at equal recall** — but graded that claim
|
||||
//! **CLAIMED**, *"not reproduced on our hardware (no HNSW baseline exists to
|
||||
//! compare against)."* You cannot measure a ratio against a baseline you do not
|
||||
//! have. This module **builds that missing HNSW baseline**; [`crate::hnsw_quantized`]
|
||||
//! builds the quantized-rerank variant that tests the *direction* of the
|
||||
//! SymphonyQG bet. ADR-261 reports the **measured** ratio.
|
||||
//!
|
||||
//! # The algorithm (Malkov & Yashunin, TPAMI 2018)
|
||||
//!
|
||||
//! HNSW = a multi-layer navigable small-world graph. Each inserted point gets a
|
||||
//! random **level** `ℓ` (geometrically distributed, mean `1/ln(M)`); it appears
|
||||
//! in all layers `0..=ℓ`. Layer 0 holds every point; higher layers are
|
||||
//! exponentially sparser "express lanes". A search:
|
||||
//!
|
||||
//! 1. Enters at the top layer's single entry point.
|
||||
//! 2. **Greedy-descends** each layer above 0: repeatedly hop to the neighbour
|
||||
//! closest to the query until no neighbour is closer, then drop a layer.
|
||||
//! 3. At layer 0, runs a **best-first beam search** with beam width `ef`,
|
||||
//! keeping the `ef` closest candidates seen, and returns the closest `k`.
|
||||
//!
|
||||
//! Construction inserts each point by searching for its `ef_construction`
|
||||
//! nearest existing neighbours at each of its layers, then connecting it to a
|
||||
//! pruned subset chosen by the **neighbour-selection heuristic** (Algorithm 4 in
|
||||
//! the paper): prefer neighbours that are closer to the new point than to any
|
||||
//! already-selected neighbour, which keeps the graph navigable (diverse edges)
|
||||
//! instead of clumping all edges toward one cluster.
|
||||
//!
|
||||
//! # Determinism (the proof contract)
|
||||
//!
|
||||
//! Level assignment is the only randomness, and it is driven by a **seeded
|
||||
//! SplitMix64** PRNG (the exact pattern from [`crate::rotation`]) — never
|
||||
//! `Date::now`, an OS RNG, or `rand` without a seed. Two indices built from the
|
||||
//! same `(seed, params, insertion order)` are bit-identical, pinned by
|
||||
//! [`tests::hnsw_is_deterministic_for_seed`]. This matters for reproducible
|
||||
//! benchmarks: the recall/QPS numbers in ADR-261 must be regenerable.
|
||||
//!
|
||||
//! # Robustness (no panic on degenerate input)
|
||||
//!
|
||||
//! Empty index, `k > n`, `k == 0`, a single node, zero-dimension vectors,
|
||||
//! ragged-length queries, and `ef < k` are all handled without panicking —
|
||||
//! pinned by the `*_no_panic` / degenerate tests. Graph traversal is bounded by
|
||||
//! the visited-set and the candidate beam, so there is no unbounded recursion
|
||||
//! (the search is iterative, using explicit heaps).
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{BinaryHeap, HashSet};
|
||||
|
||||
/// Distance metric for the index. Both are computed over `Vec<f32>` with an
|
||||
/// `f64` accumulator for numerical stability on long vectors.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Metric {
|
||||
/// Squared euclidean distance `Σ (a_i − b_i)²`. Monotone in euclidean
|
||||
/// distance, so top-K ranking is identical; we skip the sqrt.
|
||||
L2,
|
||||
/// Cosine **distance** `1 − cos(a, b)`. Smaller = more similar. This is
|
||||
/// AETHER's actual angular metric and what the `sketch.rs` sign code
|
||||
/// approximates, so it is the default for ruvector re-ID.
|
||||
Cosine,
|
||||
}
|
||||
|
||||
impl Metric {
|
||||
/// Distance between two equal-length slices under this metric.
|
||||
///
|
||||
/// Ragged lengths are handled charitably (compared over the shorter prefix);
|
||||
/// a degenerate (zero-norm) cosine input yields the maximum cosine distance
|
||||
/// `1.0` rather than a NaN. Never panics.
|
||||
#[inline]
|
||||
pub fn distance(self, a: &[f32], b: &[f32]) -> f32 {
|
||||
let n = a.len().min(b.len());
|
||||
match self {
|
||||
Metric::L2 => {
|
||||
let mut acc = 0.0f64;
|
||||
for i in 0..n {
|
||||
let d = a[i] as f64 - b[i] as f64;
|
||||
acc += d * d;
|
||||
}
|
||||
acc as f32
|
||||
}
|
||||
Metric::Cosine => {
|
||||
let mut dot = 0.0f64;
|
||||
let mut na = 0.0f64;
|
||||
let mut nb = 0.0f64;
|
||||
for i in 0..n {
|
||||
let (x, y) = (a[i] as f64, b[i] as f64);
|
||||
dot += x * y;
|
||||
na += x * x;
|
||||
nb += y * y;
|
||||
}
|
||||
let denom = (na * nb).sqrt();
|
||||
if denom < 1e-12 {
|
||||
1.0
|
||||
} else {
|
||||
(1.0 - dot / denom) as f32
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construction / search hyper-parameters for an [`HnswIndex`].
|
||||
///
|
||||
/// Defaults follow the paper's recommended starting points (`M = 16`,
|
||||
/// `ef_construction = 200`). `ef_search` is the query-time beam width; larger
|
||||
/// `ef_search` trades QPS for recall — the knob the ADR-261 benchmark sweeps to
|
||||
/// find the equal-recall operating point.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct HnswParams {
|
||||
/// Max neighbours per node on layers ≥ 1. Layer 0 uses `2·M` (`m_max0`),
|
||||
/// the paper's standard asymmetry (the base layer needs higher degree).
|
||||
pub m: usize,
|
||||
/// Candidate list size during construction (`efConstruction`). Larger =
|
||||
/// better-connected graph, slower build.
|
||||
pub ef_construction: usize,
|
||||
/// Default beam width at query time (`ef`). Overridable per-query in
|
||||
/// [`HnswIndex::search`].
|
||||
pub ef_search: usize,
|
||||
/// Seed for the level-assignment PRNG. Fixed ⇒ reproducible graph.
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
impl Default for HnswParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed: 0x1157_0000_0000_0001u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A min-distance ordering wrapper: a `BinaryHeap<Candidate>` is a **max-heap**,
|
||||
/// so we negate the comparison to make `peek()` the *closest* candidate when we
|
||||
/// want a min-heap, or use it directly for a max-heap of the *farthest*. We keep
|
||||
/// two explicit newtypes to make the intent unmistakable at each call site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Scored {
|
||||
dist: f32,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl PartialEq for Scored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.dist == other.dist && self.id == other.id
|
||||
}
|
||||
}
|
||||
impl Eq for Scored {}
|
||||
|
||||
/// Max-heap ordering: larger `dist` is "greater" ⇒ at the top. Ties broken by
|
||||
/// id so the order is total and deterministic.
|
||||
impl Ord for Scored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.dist
|
||||
.partial_cmp(&other.dist)
|
||||
.unwrap_or(Ordering::Equal)
|
||||
.then(self.id.cmp(&other.id))
|
||||
}
|
||||
}
|
||||
impl PartialOrd for Scored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// `Reverse`-equivalent for a min-heap (closest at top) without pulling in
|
||||
/// `std::cmp::Reverse` boilerplate at every site.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct MinScored(Scored);
|
||||
impl PartialEq for MinScored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.0 == other.0
|
||||
}
|
||||
}
|
||||
impl Eq for MinScored {}
|
||||
impl Ord for MinScored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
other.0.cmp(&self.0) // reversed
|
||||
}
|
||||
}
|
||||
impl PartialOrd for MinScored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// A multi-layer HNSW graph index over dense `Vec<f32>` embeddings.
|
||||
///
|
||||
/// IDs are the **insertion index** (`0..len`), returned by [`HnswIndex::search`]
|
||||
/// alongside the distance. The original vectors are retained (the graph needs
|
||||
/// them for distance computation at query time), so memory is
|
||||
/// `O(N·d) + O(N·M)` — the float vectors plus the adjacency lists.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HnswIndex {
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
dim: usize,
|
||||
/// Stored vectors, indexed by id.
|
||||
vectors: Vec<Vec<f32>>,
|
||||
/// `links[id][layer]` = neighbour ids of `id` on `layer`. A node of level
|
||||
/// `ℓ` has `ℓ+1` layers (`0..=ℓ`).
|
||||
links: Vec<Vec<Vec<u32>>>,
|
||||
/// Per-node top level.
|
||||
levels: Vec<usize>,
|
||||
/// Current entry point id (the highest-level node), or `None` if empty.
|
||||
entry: Option<u32>,
|
||||
/// Highest level currently present in the graph.
|
||||
top_level: usize,
|
||||
/// PRNG state for level assignment (advances per insert).
|
||||
rng_state: u64,
|
||||
}
|
||||
|
||||
impl HnswIndex {
|
||||
/// Create an empty index with the given metric and parameters.
|
||||
///
|
||||
/// `dim` is the expected embedding dimension. Inserts of a different length
|
||||
/// are accepted charitably (the metric compares over the shorter prefix), so
|
||||
/// a wrong-length vector degrades recall rather than panicking — but callers
|
||||
/// should keep dimension uniform.
|
||||
pub fn new(dim: usize, metric: Metric, params: HnswParams) -> Self {
|
||||
Self {
|
||||
metric,
|
||||
params,
|
||||
dim,
|
||||
vectors: Vec::new(),
|
||||
links: Vec::new(),
|
||||
levels: Vec::new(),
|
||||
entry: None,
|
||||
top_level: 0,
|
||||
rng_state: params.seed.wrapping_add(0x9E37_79B9_7F4A_7C15),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of indexed points.
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.vectors.len()
|
||||
}
|
||||
|
||||
/// True iff the index holds no points.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.vectors.is_empty()
|
||||
}
|
||||
|
||||
/// The metric this index ranks by.
|
||||
#[inline]
|
||||
pub fn metric(&self) -> Metric {
|
||||
self.metric
|
||||
}
|
||||
|
||||
/// The expected embedding dimension.
|
||||
#[inline]
|
||||
pub fn dim(&self) -> usize {
|
||||
self.dim
|
||||
}
|
||||
|
||||
/// The current entry-point id (highest-level node), or `None` if empty.
|
||||
/// Exposed so the quantized variant ([`crate::hnsw_quantized`]) can traverse
|
||||
/// the **same** graph with a different (quantized) score.
|
||||
#[inline]
|
||||
pub fn entry_point(&self) -> Option<u32> {
|
||||
self.entry
|
||||
}
|
||||
|
||||
/// The highest level currently present in the graph.
|
||||
#[inline]
|
||||
pub fn top_level(&self) -> usize {
|
||||
self.top_level
|
||||
}
|
||||
|
||||
/// The default query-time beam width (`ef_search`) from this index's params.
|
||||
#[inline]
|
||||
pub fn params_ef_search(&self) -> usize {
|
||||
self.params.ef_search
|
||||
}
|
||||
|
||||
/// Borrow the neighbour ids of `id` on `layer`. Returns an empty slice if the
|
||||
/// id is unknown or the node does not reach that layer — never panics. Used
|
||||
/// by the quantized variant to walk the shared graph.
|
||||
#[inline]
|
||||
pub fn neighbours(&self, id: u32, layer: usize) -> &[u32] {
|
||||
match self.links.get(id as usize).and_then(|l| l.get(layer)) {
|
||||
Some(v) => v.as_slice(),
|
||||
None => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// `m_max` for a layer: `2·M` on layer 0, `M` above. The base layer carries
|
||||
/// every node and needs higher degree to stay connected (the paper's
|
||||
/// asymmetric degree cap).
|
||||
#[inline]
|
||||
fn m_max(&self, layer: usize) -> usize {
|
||||
if layer == 0 {
|
||||
self.params.m * 2
|
||||
} else {
|
||||
self.params.m
|
||||
}
|
||||
}
|
||||
|
||||
/// Draw the next node's level from a geometric distribution with parameter
|
||||
/// `m_l = 1/ln(M)` — the paper's level generator — using the **seeded**
|
||||
/// SplitMix64 stream. `floor(−ln(U) · m_l)` with `U ∈ (0, 1]`.
|
||||
fn assign_level(&mut self) -> usize {
|
||||
let m = self.params.m.max(2) as f64;
|
||||
let m_l = 1.0 / m.ln();
|
||||
// Uniform in (0, 1] from the top 53 bits of a SplitMix64 word.
|
||||
let r = split_mix64(&mut self.rng_state);
|
||||
let u = (((r >> 11) as f64) + 1.0) / ((1u64 << 53) as f64 + 1.0);
|
||||
let level = (-(u.ln()) * m_l).floor();
|
||||
if level.is_finite() && level >= 0.0 {
|
||||
level as usize
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert `embedding` with the next sequential id. Returns the assigned id.
|
||||
///
|
||||
/// Builds the node's adjacency by searching the existing graph for its
|
||||
/// nearest neighbours at each of its layers and connecting via the
|
||||
/// neighbour-selection heuristic. The first insert becomes the entry point.
|
||||
pub fn insert(&mut self, embedding: &[f32]) -> u32 {
|
||||
let id = self.vectors.len() as u32;
|
||||
let vec = embedding.to_vec();
|
||||
let node_level = self.assign_level();
|
||||
|
||||
// Push the node into the arrays UP FRONT with empty per-layer link lists.
|
||||
// This is load-bearing: the bidirectional wiring below does
|
||||
// `self.links[nbr][l].push(id)`, after which a neighbour points at `id`;
|
||||
// a subsequent traversal step in the SAME insert can hop to that
|
||||
// neighbour and read `self.links[id]`. If `id`'s links did not exist yet
|
||||
// that read panics (the bug the recall gate caught). The new node has no
|
||||
// *incoming* edges until we add them, and empty outgoing lists, so it is
|
||||
// unreachable by the searches that run before its edges are wired —
|
||||
// pushing it early is safe and keeps every `self.links[*]` index valid.
|
||||
self.vectors.push(vec.clone());
|
||||
self.links.push(vec![Vec::new(); node_level + 1]);
|
||||
self.levels.push(node_level);
|
||||
|
||||
// First node: it is the entry point, no neighbours to connect.
|
||||
if self.entry.is_none() {
|
||||
self.entry = Some(id);
|
||||
self.top_level = node_level;
|
||||
return id;
|
||||
}
|
||||
|
||||
let entry = self.entry.unwrap();
|
||||
let mut ep = entry;
|
||||
|
||||
// Phase 1: greedy-descend from the top of the graph down to the layer
|
||||
// just above the node's own top level, refining the single entry point.
|
||||
let mut layer = self.top_level;
|
||||
while layer > node_level {
|
||||
ep = self.greedy_closest(&vec, ep, layer);
|
||||
if layer == 0 {
|
||||
break;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
// Phase 2: from min(node_level, top_level) down to 0, search for
|
||||
// ef_construction candidates, select neighbours, and wire bidirectional
|
||||
// edges (pruning the neighbour's list if it overflows m_max).
|
||||
let start = node_level.min(self.top_level);
|
||||
let mut layer = start as isize;
|
||||
while layer >= 0 {
|
||||
let l = layer as usize;
|
||||
let candidates =
|
||||
self.search_layer(&vec, &[ep], self.params.ef_construction.max(1), l);
|
||||
let selected = self.select_neighbours(&vec, &candidates, self.m_max(l));
|
||||
|
||||
// Connect node -> selected (write straight into the node's slot).
|
||||
self.links[id as usize][l] = selected.iter().map(|s| s.id).collect();
|
||||
|
||||
// Connect selected -> node (bidirectional), pruning if needed.
|
||||
for s in &selected {
|
||||
let nbr = s.id as usize;
|
||||
self.links[nbr][l].push(id);
|
||||
if self.links[nbr][l].len() > self.m_max(l) {
|
||||
self.prune_neighbours(nbr as u32, l);
|
||||
}
|
||||
}
|
||||
|
||||
// Move the entry for the next-lower layer to the closest candidate.
|
||||
if let Some(best) = candidates
|
||||
.iter()
|
||||
.min_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal))
|
||||
{
|
||||
ep = best.id;
|
||||
}
|
||||
layer -= 1;
|
||||
}
|
||||
|
||||
if node_level > self.top_level {
|
||||
self.top_level = node_level;
|
||||
self.entry = Some(id);
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Greedy single-best descent on one layer: hop to the neighbour closest to
|
||||
/// `query` until no neighbour improves. Iterative (bounded by the graph) —
|
||||
/// no recursion.
|
||||
fn greedy_closest(&self, query: &[f32], start: u32, layer: usize) -> u32 {
|
||||
let mut best = start;
|
||||
let mut best_d = self.metric.distance(query, &self.vectors[best as usize]);
|
||||
loop {
|
||||
let mut improved = false;
|
||||
for &nbr in &self.links[best as usize][layer] {
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
if d < best_d {
|
||||
best_d = d;
|
||||
best = nbr;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
if !improved {
|
||||
return best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Beam search on one layer (paper Algorithm 2): best-first expansion from
|
||||
/// `entry_points`, keeping the `ef` closest results. Returns the result set
|
||||
/// (unsorted; callers sort/truncate). Bounded by a visited set + the `ef`
|
||||
/// result heap — no recursion, no unbounded growth.
|
||||
fn search_layer(
|
||||
&self,
|
||||
query: &[f32],
|
||||
entry_points: &[u32],
|
||||
ef: usize,
|
||||
layer: usize,
|
||||
) -> Vec<Scored> {
|
||||
let mut visited: HashSet<u32> = HashSet::new();
|
||||
// `candidates`: min-heap (closest first) of nodes to expand.
|
||||
let mut candidates: BinaryHeap<MinScored> = BinaryHeap::new();
|
||||
// `results`: max-heap (farthest first) of the best-ef found so far, so
|
||||
// the top is the current worst and is cheap to evict.
|
||||
let mut results: BinaryHeap<Scored> = BinaryHeap::new();
|
||||
|
||||
for &ep in entry_points {
|
||||
if ep as usize >= self.vectors.len() {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[ep as usize]);
|
||||
let s = Scored { dist: d, id: ep };
|
||||
visited.insert(ep);
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
}
|
||||
// Cap results at ef from the start.
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
|
||||
while let Some(MinScored(cur)) = candidates.pop() {
|
||||
// Stop when the closest unexpanded candidate is farther than the
|
||||
// current worst result and the result set is already full.
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if cur.dist > worst && results.len() >= ef {
|
||||
break;
|
||||
}
|
||||
for &nbr in &self.links[cur.id as usize][layer] {
|
||||
if !visited.insert(nbr) {
|
||||
continue;
|
||||
}
|
||||
let d = self.metric.distance(query, &self.vectors[nbr as usize]);
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(f32::INFINITY);
|
||||
if results.len() < ef || d < worst {
|
||||
let s = Scored { dist: d, id: nbr };
|
||||
candidates.push(MinScored(s));
|
||||
results.push(s);
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_vec()
|
||||
}
|
||||
|
||||
/// Neighbour-selection heuristic (paper Algorithm 4): from `candidates`,
|
||||
/// greedily pick up to `m` that are **closer to the new point than to any
|
||||
/// already-picked neighbour**, giving diverse, navigable edges instead of a
|
||||
/// clump. Candidates are considered nearest-first.
|
||||
fn select_neighbours(&self, _base: &[f32], candidates: &[Scored], m: usize) -> Vec<Scored> {
|
||||
let mut sorted = candidates.to_vec();
|
||||
sorted.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
let mut selected: Vec<Scored> = Vec::with_capacity(m);
|
||||
for cand in sorted {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
// Keep `cand` only if it is closer to `base` than to every already
|
||||
// selected neighbour — the diversity condition.
|
||||
let cand_vec = &self.vectors[cand.id as usize];
|
||||
let mut keep = true;
|
||||
for sel in &selected {
|
||||
let d_cand_sel = self.metric.distance(cand_vec, &self.vectors[sel.id as usize]);
|
||||
if d_cand_sel < cand.dist {
|
||||
keep = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
selected.push(cand);
|
||||
}
|
||||
}
|
||||
// If the diversity filter left us short (sparse graph), backfill with the
|
||||
// remaining nearest candidates so the node is not under-connected.
|
||||
if selected.len() < m {
|
||||
let chosen: HashSet<u32> = selected.iter().map(|s| s.id).collect();
|
||||
let mut rest: Vec<Scored> = candidates
|
||||
.iter()
|
||||
.filter(|c| !chosen.contains(&c.id))
|
||||
.copied()
|
||||
.collect();
|
||||
rest.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
for c in rest {
|
||||
if selected.len() >= m {
|
||||
break;
|
||||
}
|
||||
selected.push(c);
|
||||
}
|
||||
}
|
||||
selected
|
||||
}
|
||||
|
||||
/// Re-prune a node's neighbour list on `layer` back down to `m_max` using
|
||||
/// the selection heuristic, after a bidirectional edge pushed it over cap.
|
||||
fn prune_neighbours(&mut self, id: u32, layer: usize) {
|
||||
let base = self.vectors[id as usize].clone();
|
||||
let current: Vec<Scored> = self.links[id as usize][layer]
|
||||
.iter()
|
||||
.map(|&nbr| Scored {
|
||||
dist: self.metric.distance(&base, &self.vectors[nbr as usize]),
|
||||
id: nbr,
|
||||
})
|
||||
.collect();
|
||||
let kept = self.select_neighbours(&base, ¤t, self.m_max(layer));
|
||||
self.links[id as usize][layer] = kept.iter().map(|s| s.id).collect();
|
||||
}
|
||||
|
||||
/// Search for the `k` nearest neighbours of `query`, using beam width `ef`
|
||||
/// (clamped to at least `k`). Returns up to `k` `(id, distance)` pairs sorted
|
||||
/// ascending by distance.
|
||||
///
|
||||
/// Degenerate cases return cleanly: empty index ⇒ empty vec; `k == 0` ⇒ empty
|
||||
/// vec; `k > len` ⇒ all points; a single node ⇒ that node. Never panics.
|
||||
pub fn search(&self, query: &[f32], k: usize, ef: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let entry = match self.entry {
|
||||
Some(e) => e,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
let ef = ef.max(k).max(1);
|
||||
|
||||
// Greedy-descend the upper layers to a good layer-0 entry point.
|
||||
let mut ep = entry;
|
||||
let mut layer = self.top_level;
|
||||
while layer > 0 {
|
||||
ep = self.greedy_closest(query, ep, layer);
|
||||
layer -= 1;
|
||||
}
|
||||
// Beam search on layer 0.
|
||||
let mut results = self.search_layer(query, &[ep], ef, 0);
|
||||
results.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap_or(Ordering::Equal));
|
||||
results.truncate(k);
|
||||
results.into_iter().map(|s| (s.id, s.dist)).collect()
|
||||
}
|
||||
|
||||
/// Search using the index's configured default `ef_search`.
|
||||
#[inline]
|
||||
pub fn search_default(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
self.search(query, k, self.params.ef_search)
|
||||
}
|
||||
|
||||
/// Borrow a stored vector by id (for the quantized variant / reranking).
|
||||
#[inline]
|
||||
pub fn vector(&self, id: u32) -> Option<&[f32]> {
|
||||
self.vectors.get(id as usize).map(|v| v.as_slice())
|
||||
}
|
||||
|
||||
/// Brute-force exact top-K linear scan over the stored vectors — the ANN
|
||||
/// **ground truth** and the linear-scan baseline the benchmark measures
|
||||
/// against. `O(N·d)` per query. Returns up to `k` `(id, distance)` ascending.
|
||||
pub fn brute_force(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut scored: Vec<(u32, f32)> = self
|
||||
.vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (i as u32, self.metric.distance(query, v)))
|
||||
.collect();
|
||||
scored.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
scored.truncate(k);
|
||||
scored
|
||||
}
|
||||
}
|
||||
|
||||
/// SplitMix64 step — the same deterministic PRNG used by [`crate::rotation`].
|
||||
/// Public-domain (Sebastiano Vigna). Dependency-free and reproducible.
|
||||
#[inline]
|
||||
pub(crate) fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// SplitMix64-driven uniform in [0,1) for building fixtures (mirrors
|
||||
/// `coverage.rs`'s style so the planted-cluster geometry matches).
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
let r = split_mix64(state);
|
||||
((r >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
|
||||
/// Build a planted-cluster fixture: `n` vectors of `dim`, in `clusters`
|
||||
/// Gaussian clusters. Returns the vectors. Deterministic from `seed`.
|
||||
fn planted(dim: usize, n: usize, clusters: usize, seed: u64) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..clusters)
|
||||
.map(|c| {
|
||||
let mut s = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let c = i % clusters;
|
||||
let mut s = seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..dim).map(|d| centres[c][d] + gauss(&mut s) * 0.35).collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build(vectors: &[Vec<f32>], metric: Metric, seed: u64) -> HnswIndex {
|
||||
let params = HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed,
|
||||
};
|
||||
let mut idx = HnswIndex::new(vectors[0].len(), metric, params);
|
||||
for v in vectors {
|
||||
idx.insert(v);
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
/// Recall@k of HNSW search vs brute-force ground truth, averaged over queries
|
||||
/// drawn from the same planted clusters.
|
||||
fn recall_at_k(
|
||||
idx: &HnswIndex,
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
clusters: usize,
|
||||
k: usize,
|
||||
ef: usize,
|
||||
n_queries: usize,
|
||||
seed: u64,
|
||||
) -> f64 {
|
||||
let centres_seed = seed; // reuse fixture seed for matching cluster geometry
|
||||
let mut total = 0.0f64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut s = centres_seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
// A query near cluster centre c: regenerate the centre then jitter.
|
||||
let mut cs = centres_seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
|
||||
let truth: HashSet<u32> = idx.brute_force(&qv, k).into_iter().map(|(id, _)| id).collect();
|
||||
let got = idx.search(&qv, k, ef);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / k as f64;
|
||||
let _ = vectors;
|
||||
}
|
||||
total / n_queries as f64
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_index_search_is_empty_no_panic() {
|
||||
let idx = HnswIndex::new(8, Metric::L2, HnswParams::default());
|
||||
assert!(idx.is_empty());
|
||||
assert!(idx.search(&[0.0; 8], 5, 16).is_empty());
|
||||
assert!(idx.brute_force(&[0.0; 8], 5).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_returns_itself() {
|
||||
let mut idx = HnswIndex::new(4, Metric::L2, HnswParams::default());
|
||||
let id = idx.insert(&[1.0, 2.0, 3.0, 4.0]);
|
||||
assert_eq!(id, 0);
|
||||
let r = idx.search(&[1.0, 2.0, 3.0, 4.0], 5, 16);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, 0);
|
||||
assert!(r[0].1 < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn k_zero_and_k_gt_n_no_panic() {
|
||||
let vectors = planted(16, 40, 4, 0xABCD);
|
||||
let idx = build(&vectors, Metric::L2, 0x1234);
|
||||
assert!(idx.search(&vectors[0], 0, 16).is_empty());
|
||||
// k > n returns all n.
|
||||
let r = idx.search(&vectors[0], 1000, 64);
|
||||
assert_eq!(r.len(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_query_no_panic() {
|
||||
let vectors = planted(16, 30, 3, 0x55);
|
||||
let idx = build(&vectors, Metric::Cosine, 0x66);
|
||||
// Short and long queries must not panic.
|
||||
assert!(!idx.search(&[1.0, 2.0, 3.0], 3, 16).is_empty());
|
||||
let long: Vec<f32> = (0..100).map(|i| i as f32).collect();
|
||||
assert!(!idx.search(&long, 3, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn self_query_ranks_self_first() {
|
||||
let vectors = planted(32, 200, 8, 0x77);
|
||||
let idx = build(&vectors, Metric::L2, 0x88);
|
||||
for &probe in &[0usize, 50, 137, 199] {
|
||||
let r = idx.search(&vectors[probe], 1, 64);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, probe as u32, "self-query should return the stored self");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hnsw_is_deterministic_for_seed() {
|
||||
// Same (seed, params, insertion order) ⇒ identical level assignment and
|
||||
// identical search output.
|
||||
let vectors = planted(24, 150, 6, 0x2222);
|
||||
let a = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
let b = build(&vectors, Metric::Cosine, 0xFEED);
|
||||
assert_eq!(a.levels, b.levels, "level assignment must be deterministic");
|
||||
let q = &vectors[42];
|
||||
assert_eq!(a.search(q, 10, 64), b.search(q, 10, 64));
|
||||
// A different seed (almost surely) changes the level structure.
|
||||
let c = build(&vectors, Metric::Cosine, 0x1357);
|
||||
assert_ne!(a.levels, c.levels, "different seed should change levels");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_l2() {
|
||||
// THE CORRECTNESS GATE (ADR-261): HNSW recall@10 vs brute-force must be
|
||||
// >= 0.95 at a reasonable ef. Low recall ⇒ a bug in the graph.
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0x9999;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::L2, 0xAAAA);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (L2) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_at_10_meets_correctness_gate_cosine() {
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0xBBBB;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let idx = build(&vectors, Metric::Cosine, 0xCCCC);
|
||||
let recall = recall_at_k(&idx, &vectors, dim, clusters, 10, 128, 64, seed);
|
||||
assert!(
|
||||
recall >= 0.95,
|
||||
"HNSW recall@10 (cosine) = {recall:.4} below the 0.95 correctness gate — graph bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn higher_ef_does_not_reduce_recall() {
|
||||
// Monotonicity sanity: more beam width should not hurt recall.
|
||||
let dim = 48;
|
||||
let vectors = planted(dim, 1000, 16, 0xD00D);
|
||||
let idx = build(&vectors, Metric::L2, 0xE00E);
|
||||
let lo = recall_at_k(&idx, &vectors, dim, 16, 10, 16, 48, 0xD00D);
|
||||
let hi = recall_at_k(&idx, &vectors, dim, 16, 10, 128, 48, 0xD00D);
|
||||
assert!(hi + 1e-9 >= lo, "recall dropped with larger ef: {lo:.3} -> {hi:.3}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_dim_no_panic() {
|
||||
// Degenerate zero-dimension index: inserts and searches must not panic.
|
||||
let mut idx = HnswIndex::new(0, Metric::Cosine, HnswParams::default());
|
||||
idx.insert(&[]);
|
||||
idx.insert(&[]);
|
||||
let r = idx.search(&[], 2, 16);
|
||||
assert_eq!(r.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
//! A **SymphonyQG-style quantized-traversal HNSW** — ADR-261 (multi-bit, §11).
|
||||
//!
|
||||
//! # The SymphonyQG bet (what we are testing)
|
||||
//!
|
||||
//! [SymphonyQG (SIGMOD 2025)](../../../../../docs/adr/ADR-261-ruvector-graph-ann-index.md)
|
||||
//! unifies **quantization with graph traversal**: instead of computing the full
|
||||
//! float distance at every node the beam search visits (the cost that dominates
|
||||
//! float HNSW — one `O(d)` float dot/diff per visited node), it scores traversal
|
||||
//! candidates with a **cheap quantized distance** and only computes the exact
|
||||
//! float distance for the *final* candidate set, which it **reranks**. The bet:
|
||||
//! the quantized score is cheap enough — and accurate enough to keep the beam on
|
||||
//! the right path — that you visit roughly as many nodes but pay far less per
|
||||
//! node, and recover the small recall loss with a final exact rerank. Source
|
||||
//! reports **3.5–17× QPS over HNSW at equal recall**.
|
||||
//!
|
||||
//! # Our implementation (honest scope)
|
||||
//!
|
||||
//! We are **not** reproducing SymphonyQG's exact system (their RaBitQ-fused codes,
|
||||
//! their SIMD layout, their refined graph). We build the **direction** of the
|
||||
//! claim from the pieces this crate already has, so the comparison is
|
||||
//! apples-to-apples on *our* hardware:
|
||||
//!
|
||||
//! - **Same graph** as the float [`crate::HnswIndex`] — identical structure,
|
||||
//! identical seed, identical level assignment. The *only* variable between the
|
||||
//! float and quantized search is **how a candidate is scored during traversal**,
|
||||
//! so any QPS/recall difference is attributable to the quantization, not to a
|
||||
//! different graph.
|
||||
//! - **Quantized score = `b`-bit code over the RaBitQ Pass-2 rotated coordinates**
|
||||
//! ([`crate::rotation`] + the multi-bit scalar quantizer mirrored from
|
||||
//! [ADR-156 §10](../../../../../docs/adr/ADR-156-ruvector-fusion-beyond-sota.md)'s
|
||||
//! `coverage::measure_multibit`). Each node stores a `b`-bit-per-dimension code
|
||||
//! over the padded rotation length `D = next_pow2(dim)`. During traversal we
|
||||
//! compare query-code vs node-code by the **L1 distance over the per-dim
|
||||
//! codes** — a few machine words of integer work, no per-dimension float work.
|
||||
//! For `b == 1` the codes are `{0, 1}` and the L1 distance is **exactly the
|
||||
//! 1-bit Hamming distance** of the original ADR-261 construction, so `b == 1`
|
||||
//! is fully backward-compatible.
|
||||
//! - **Exact float rerank** of the final beam: the top `rerank` candidates by
|
||||
//! code-L1 are re-scored with the true float metric and the best `k` returned.
|
||||
//!
|
||||
//! Higher `b` keeps the traversal beam on-path better than 1-bit (ADR-156 §10
|
||||
//! measured 1/2/3/4-bit strict-K coverage at ~46/54/67/74%), at a memory cost
|
||||
//! that scales linearly with `b` (bytes/node = `ceil(D·b/8)`). **Whether the
|
||||
//! extra bits net a QPS win at equal recall — and at what N a crossover with
|
||||
//! float HNSW appears, if any — is the measured question ADR-261 §11 answers.**
|
||||
//! We report the real number, win or lose, and do not tune to manufacture a
|
||||
//! speedup.
|
||||
//!
|
||||
//! # Determinism & robustness
|
||||
//!
|
||||
//! The graph seed drives everything (level assignment), so the quantized index
|
||||
//! is as reproducible as the float one. Empty/degenerate inputs are guarded
|
||||
//! exactly as in [`crate::hnsw`] — no panic on empty index, `k > n`, `k == 0`,
|
||||
//! single node, ragged query, or zero dim.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{BinaryHeap, HashSet};
|
||||
|
||||
use crate::hnsw::{HnswIndex, HnswParams, Metric};
|
||||
use crate::rotation::Rotation;
|
||||
|
||||
/// Symmetric clamp range for the uniform mid-rise scalar quantizer, in rotated-
|
||||
/// coordinate units. The normalized FHT (`1/√D`) puts AETHER-shape rotated
|
||||
/// coordinates roughly in `[-3, 3]`; out-of-range coords clamp to the end codes.
|
||||
/// This is the **same `RANGE = 3.0`** as ADR-156 §10's `coverage::measure_multibit`,
|
||||
/// so the multi-bit code here is the same scheme that module measured.
|
||||
const RANGE: f32 = 3.0;
|
||||
|
||||
/// A `b`-bit-per-dimension scalar code of a rotated embedding over the padded
|
||||
/// length `D`, compared by per-dim L1.
|
||||
///
|
||||
/// For `bits == 1` the per-dim code is `{0, 1}` (sign), and L1 over those codes
|
||||
/// is exactly POPCNT Hamming — so the 1-bit case is bit-for-bit the original
|
||||
/// ADR-261 construction. For `bits ∈ {2, 4}` the code is a uniform mid-rise
|
||||
/// quantizer with `2^bits` levels over `[-RANGE, RANGE]`.
|
||||
#[derive(Debug, Clone)]
|
||||
struct Code {
|
||||
/// Per-dimension codes (`0..2^bits`), one entry per padded dimension `D`.
|
||||
/// Kept unpacked as `u8` for branch-free L1; the *reported* memory cost is
|
||||
/// the packed footprint (`ceil(D·bits/8)`), since a production node would
|
||||
/// store the packed form. (We measure the packed bytes/node explicitly in
|
||||
/// [`QuantizedHnswIndex::bytes_per_node`].)
|
||||
codes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Code {
|
||||
/// L1 distance over the per-dimension codes — the multi-bit generalization
|
||||
/// of Hamming. At `bits == 1` (codes in `{0,1}`) this equals the popcount of
|
||||
/// the XOR, i.e. the 1-bit Hamming distance.
|
||||
#[inline]
|
||||
fn l1(&self, other: &Code) -> u32 {
|
||||
let n = self.codes.len().min(other.codes.len());
|
||||
let mut acc = 0u32;
|
||||
for i in 0..n {
|
||||
acc += (self.codes[i] as i32 - other.codes[i] as i32).unsigned_abs();
|
||||
}
|
||||
acc
|
||||
}
|
||||
}
|
||||
|
||||
/// Quantize the rotated coordinates of `embedding` to a `bits`-bit-per-dimension
|
||||
/// [`Code`] over the padded rotation length `D = rotation.padded_dim()`.
|
||||
///
|
||||
/// `bits == 1` reduces to sign-quantization (code `1` iff the rotated coord ≥ 0),
|
||||
/// preserving the original 1-bit construction; `bits ∈ {2, 4}` uses a uniform
|
||||
/// mid-rise quantizer with `2^bits` levels over `[-RANGE, RANGE]`, identical to
|
||||
/// ADR-156 §10's `measure_multibit`.
|
||||
fn encode(embedding: &[f32], rotation: &Rotation, bits: u32) -> Code {
|
||||
let rotated = rotation.apply_padded(embedding);
|
||||
let levels = 1u32 << bits; // 2^bits codes per dim
|
||||
let codes: Vec<u8> = rotated
|
||||
.iter()
|
||||
.map(|&x| {
|
||||
if bits == 1 {
|
||||
// Sign code: identical to the original 1-bit construction.
|
||||
u8::from(x >= 0.0)
|
||||
} else {
|
||||
let t = ((x + RANGE) / (2.0 * RANGE)).clamp(0.0, 1.0); // → [0,1]
|
||||
let code = (t * (levels - 1) as f32).round() as u32;
|
||||
code.min(levels - 1) as u8
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Code { codes }
|
||||
}
|
||||
|
||||
/// Packed bytes a node's `bits`-bit code occupies over padded length `D`:
|
||||
/// `ceil(D·bits/8)`. The memory cost reported by ADR-261 §11 (1-bit → `D/8`,
|
||||
/// 2-bit → `D/4`, 4-bit → `D/2`).
|
||||
#[inline]
|
||||
fn packed_bytes(padded_dim: usize, bits: u32) -> usize {
|
||||
(padded_dim * bits as usize).div_ceil(8)
|
||||
}
|
||||
|
||||
/// Min-heap node for the quantized beam (closest code-L1 at the top).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct HScored {
|
||||
/// Code-L1 distance (quantized score) — the traversal key.
|
||||
dist: u32,
|
||||
id: u32,
|
||||
}
|
||||
impl PartialEq for HScored {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.dist == other.dist && self.id == other.id
|
||||
}
|
||||
}
|
||||
impl Eq for HScored {}
|
||||
impl Ord for HScored {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.dist.cmp(&other.dist).then(self.id.cmp(&other.id))
|
||||
}
|
||||
}
|
||||
impl PartialOrd for HScored {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
/// Reversed wrapper for a min-heap (smallest code-L1 at the top).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct MinH(HScored);
|
||||
impl PartialEq for MinH {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.0 == other.0
|
||||
}
|
||||
}
|
||||
impl Eq for MinH {}
|
||||
impl Ord for MinH {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
other.0.cmp(&self.0)
|
||||
}
|
||||
}
|
||||
impl PartialOrd for MinH {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// A SymphonyQG-style HNSW: the same graph as [`HnswIndex`], traversed by a
|
||||
/// **cheap `b`-bit code-L1 score**, with a final **exact-float rerank**.
|
||||
///
|
||||
/// Built by inserting the same vectors in the same order with the same seed as
|
||||
/// a float [`HnswIndex`], so the two indices share identical graph structure and
|
||||
/// only differ in how the beam is scored. The shared [`Rotation`] (seed + dim)
|
||||
/// is the index/query frame for the `b`-bit codes. `bits ∈ {1, 2, 4}` selects
|
||||
/// the traversal-code resolution; `bits == 1` is the original 1-bit Hamming
|
||||
/// construction.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct QuantizedHnswIndex {
|
||||
/// The underlying graph (built with the float metric for exact rerank).
|
||||
graph: HnswIndex,
|
||||
/// Per-node `b`-bit codes, indexed by id (parallel to graph vectors).
|
||||
codes: Vec<Code>,
|
||||
/// The rotation frame shared by index and query codes.
|
||||
rotation: Rotation,
|
||||
/// Bits per dimension of the traversal code (`1`, `2`, or `4`).
|
||||
bits: u32,
|
||||
/// Number of final candidates to exact-float rerank (≥ k at query time).
|
||||
default_rerank: usize,
|
||||
}
|
||||
|
||||
impl QuantizedHnswIndex {
|
||||
/// Build a 1-bit quantized index (the original ADR-261 construction).
|
||||
///
|
||||
/// Equivalent to [`QuantizedHnswIndex::build_bits`] with `bits = 1`; kept as
|
||||
/// the backward-compatible entry point so existing callers and tests are
|
||||
/// unchanged.
|
||||
pub fn build(
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
rotation_seed: u64,
|
||||
default_rerank: usize,
|
||||
) -> Self {
|
||||
Self::build_bits(vectors, dim, metric, params, rotation_seed, 1, default_rerank)
|
||||
}
|
||||
|
||||
/// Build a `bits`-bit quantized index over `vectors`, mirroring a float
|
||||
/// [`HnswIndex`] built with the same `(dim, metric, params)` and insertion
|
||||
/// order. The `rotation_seed` fixes the code frame (index and query share it).
|
||||
///
|
||||
/// `bits` is clamped to `{1, 2, 4}` (the resolutions ADR-261 §11 sweeps): any
|
||||
/// other value is rounded up to the nearest of these so the constructor is
|
||||
/// total. `default_rerank` is how many top-code-L1 candidates get an exact
|
||||
/// float re-score before returning the best `k`; it is clamped to `≥ k` at
|
||||
/// query time. A larger rerank recovers more recall at more float cost — the
|
||||
/// knob that, alongside `ef`, sets the equal-recall operating point.
|
||||
pub fn build_bits(
|
||||
vectors: &[Vec<f32>],
|
||||
dim: usize,
|
||||
metric: Metric,
|
||||
params: HnswParams,
|
||||
rotation_seed: u64,
|
||||
bits: u32,
|
||||
default_rerank: usize,
|
||||
) -> Self {
|
||||
let bits = clamp_bits(bits);
|
||||
let rotation = Rotation::new(rotation_seed, dim);
|
||||
let mut graph = HnswIndex::new(dim, metric, params);
|
||||
let mut codes = Vec::with_capacity(vectors.len());
|
||||
for v in vectors {
|
||||
graph.insert(v);
|
||||
codes.push(encode(v, &rotation, bits));
|
||||
}
|
||||
Self {
|
||||
graph,
|
||||
codes,
|
||||
rotation,
|
||||
bits,
|
||||
default_rerank: default_rerank.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of indexed points.
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.graph.len()
|
||||
}
|
||||
|
||||
/// True iff empty.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.graph.is_empty()
|
||||
}
|
||||
|
||||
/// Borrow the underlying float graph (for shared-graph benchmark parity:
|
||||
/// the float-HNSW baseline runs on *this* graph so the only variable is
|
||||
/// scoring).
|
||||
#[inline]
|
||||
pub fn graph(&self) -> &HnswIndex {
|
||||
&self.graph
|
||||
}
|
||||
|
||||
/// The rerank width this index defaults to.
|
||||
#[inline]
|
||||
pub fn default_rerank(&self) -> usize {
|
||||
self.default_rerank
|
||||
}
|
||||
|
||||
/// Bits per dimension of the traversal code.
|
||||
#[inline]
|
||||
pub fn bits(&self) -> u32 {
|
||||
self.bits
|
||||
}
|
||||
|
||||
/// Packed memory footprint of one node's traversal code, in bytes:
|
||||
/// `ceil(D·bits/8)` where `D = next_pow2(dim)` is the padded rotation length.
|
||||
/// This is the per-node cost ADR-261 §11 reports for each `b`.
|
||||
#[inline]
|
||||
pub fn bytes_per_node(&self) -> usize {
|
||||
packed_bytes(self.rotation.padded_dim(), self.bits)
|
||||
}
|
||||
|
||||
/// SymphonyQG-style search: traverse the graph scoring candidates by the
|
||||
/// **`b`-bit code-L1**, collect a beam of `ef`, then **exact-float rerank**
|
||||
/// the top `rerank` (clamped ≥ k) and return the best `k` as `(id, float_dist)`.
|
||||
///
|
||||
/// Degenerate cases mirror [`HnswIndex::search`]: empty ⇒ empty; `k == 0` ⇒
|
||||
/// empty; `k > n` ⇒ all; never panics.
|
||||
pub fn search_quantized(
|
||||
&self,
|
||||
query: &[f32],
|
||||
k: usize,
|
||||
ef: usize,
|
||||
rerank: usize,
|
||||
) -> Vec<(u32, f32)> {
|
||||
if k == 0 || self.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let ef = ef.max(k).max(1);
|
||||
let rerank = rerank.max(k);
|
||||
let q_code = encode(query, &self.rotation, self.bits);
|
||||
|
||||
// Entry point: the graph's entry (highest-level node).
|
||||
let entry = match self.graph.entry_point() {
|
||||
Some(e) => e,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
// Greedy-descend upper layers by code-L1, then beam-search layer 0.
|
||||
let mut ep = entry;
|
||||
let mut layer = self.graph.top_level();
|
||||
while layer > 0 {
|
||||
ep = self.greedy_code(&q_code, ep, layer);
|
||||
layer -= 1;
|
||||
}
|
||||
let beam = self.beam_code(&q_code, ep, ef);
|
||||
|
||||
// Exact-float rerank of the top `rerank` code-L1 candidates.
|
||||
let mut cand: Vec<HScored> = beam;
|
||||
cand.sort_by_key(|c| c.dist);
|
||||
cand.truncate(rerank);
|
||||
let mut reranked: Vec<(u32, f32)> = cand
|
||||
.iter()
|
||||
.filter_map(|c| {
|
||||
self.graph
|
||||
.vector(c.id)
|
||||
.map(|v| (c.id, self.graph.metric().distance(query, v)))
|
||||
})
|
||||
.collect();
|
||||
reranked.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
reranked.truncate(k);
|
||||
reranked
|
||||
}
|
||||
|
||||
/// Search using the index's default `ef` (from graph params) and rerank.
|
||||
#[inline]
|
||||
pub fn search_default(&self, query: &[f32], k: usize) -> Vec<(u32, f32)> {
|
||||
self.search_quantized(query, k, self.graph.params_ef_search(), self.default_rerank)
|
||||
}
|
||||
|
||||
/// Greedy single-best descent on a layer scored by code-L1.
|
||||
fn greedy_code(&self, q_code: &Code, start: u32, layer: usize) -> u32 {
|
||||
let mut best = start;
|
||||
let mut best_d = self.codes[best as usize].l1(q_code);
|
||||
loop {
|
||||
let mut improved = false;
|
||||
for &nbr in self.graph.neighbours(best, layer) {
|
||||
let d = self.codes[nbr as usize].l1(q_code);
|
||||
if d < best_d {
|
||||
best_d = d;
|
||||
best = nbr;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
if !improved {
|
||||
return best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Beam search on layer 0 scored by code-L1. Returns the `ef` best-code nodes
|
||||
/// (unsorted). Iterative — bounded by the visited set + the ef beam.
|
||||
fn beam_code(&self, q_code: &Code, ep: u32, ef: usize) -> Vec<HScored> {
|
||||
let mut visited: HashSet<u32> = HashSet::new();
|
||||
let mut candidates: BinaryHeap<MinH> = BinaryHeap::new();
|
||||
let mut results: BinaryHeap<HScored> = BinaryHeap::new(); // max-heap: worst at top
|
||||
|
||||
let d0 = self.codes[ep as usize].l1(q_code);
|
||||
let s0 = HScored { dist: d0, id: ep };
|
||||
visited.insert(ep);
|
||||
candidates.push(MinH(s0));
|
||||
results.push(s0);
|
||||
|
||||
while let Some(MinH(cur)) = candidates.pop() {
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(u32::MAX);
|
||||
if cur.dist > worst && results.len() >= ef {
|
||||
break;
|
||||
}
|
||||
for &nbr in self.graph.neighbours(cur.id, 0) {
|
||||
if !visited.insert(nbr) {
|
||||
continue;
|
||||
}
|
||||
let d = self.codes[nbr as usize].l1(q_code);
|
||||
let worst = results.peek().map(|s| s.dist).unwrap_or(u32::MAX);
|
||||
if results.len() < ef || d < worst {
|
||||
let s = HScored { dist: d, id: nbr };
|
||||
candidates.push(MinH(s));
|
||||
results.push(s);
|
||||
while results.len() > ef {
|
||||
results.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_vec()
|
||||
}
|
||||
}
|
||||
|
||||
/// Clamp a requested bit-depth to the supported `{1, 2, 4}` set (round up to the
|
||||
/// nearest supported value; `0` → `1`, `3` → `4`, `> 4` → `4`).
|
||||
#[inline]
|
||||
fn clamp_bits(bits: u32) -> u32 {
|
||||
match bits {
|
||||
0 | 1 => 1,
|
||||
2 => 2,
|
||||
_ => 4,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn split_mix64(state: &mut u64) -> u64 {
|
||||
*state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = *state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn unif01(state: &mut u64) -> f32 {
|
||||
((split_mix64(state) >> 40) as f32) / ((1u64 << 24) as f32)
|
||||
}
|
||||
fn gauss(state: &mut u64) -> f32 {
|
||||
let u1 = unif01(state).max(1e-7);
|
||||
let u2 = unif01(state);
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
|
||||
}
|
||||
fn planted(dim: usize, n: usize, clusters: usize, seed: u64) -> Vec<Vec<f32>> {
|
||||
let centres: Vec<Vec<f32>> = (0..clusters)
|
||||
.map(|c| {
|
||||
let mut s = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
(0..dim).map(|_| gauss(&mut s) * 3.0).collect()
|
||||
})
|
||||
.collect();
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let c = i % clusters;
|
||||
let mut s = seed ^ (i as u64).wrapping_mul(0x9E37);
|
||||
(0..dim).map(|d| centres[c][d] + gauss(&mut s) * 0.35).collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
fn params(seed: u64) -> HnswParams {
|
||||
HnswParams {
|
||||
m: 16,
|
||||
ef_construction: 200,
|
||||
ef_search: 64,
|
||||
seed,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_quantized_search_is_empty_no_panic() {
|
||||
let idx = QuantizedHnswIndex::build(&[], 8, Metric::Cosine, params(1), 0x42, 16);
|
||||
assert!(idx.is_empty());
|
||||
assert!(idx.search_quantized(&[0.0; 8], 5, 16, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_quantized_returns_itself() {
|
||||
let v = vec![vec![1.0, 2.0, 3.0, 4.0]];
|
||||
let idx = QuantizedHnswIndex::build(&v, 4, Metric::L2, params(2), 0x7, 8);
|
||||
let r = idx.search_quantized(&v[0], 3, 16, 8);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn k_zero_and_k_gt_n_no_panic() {
|
||||
let vectors = planted(16, 40, 4, 0xABCD);
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 16, Metric::L2, params(3), 0x9, 32);
|
||||
assert!(idx.search_quantized(&vectors[0], 0, 16, 16).is_empty());
|
||||
let r = idx.search_quantized(&vectors[0], 1000, 64, 64);
|
||||
assert_eq!(r.len(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_query_no_panic() {
|
||||
let vectors = planted(16, 30, 3, 0x55);
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 16, Metric::Cosine, params(4), 0xB, 16);
|
||||
assert!(!idx.search_quantized(&[1.0, 2.0, 3.0], 3, 16, 16).is_empty());
|
||||
let long: Vec<f32> = (0..100).map(|i| i as f32).collect();
|
||||
assert!(!idx.search_quantized(&long, 3, 16, 16).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quantized_is_deterministic() {
|
||||
let vectors = planted(32, 300, 8, 0x2468);
|
||||
let a = QuantizedHnswIndex::build(&vectors, 32, Metric::Cosine, params(0xFEED), 0xC0DE, 32);
|
||||
let b = QuantizedHnswIndex::build(&vectors, 32, Metric::Cosine, params(0xFEED), 0xC0DE, 32);
|
||||
let q = &vectors[100];
|
||||
assert_eq!(
|
||||
a.search_quantized(q, 10, 64, 32),
|
||||
b.search_quantized(q, 10, 64, 32),
|
||||
"quantized search must be deterministic"
|
||||
);
|
||||
}
|
||||
|
||||
/// Recall@10 of quantized-HNSW vs brute-force ground truth, averaged over
|
||||
/// queries. With an exact-float rerank, recall should be high (the rerank
|
||||
/// repairs most of the 1-bit traversal's coarseness). This is the quantized
|
||||
/// variant's correctness gate.
|
||||
#[test]
|
||||
fn quantized_recall_at_10_is_high_with_rerank() {
|
||||
let dim = 64;
|
||||
let n = 2000;
|
||||
let clusters = 32;
|
||||
let seed = 0x9999;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
// Generous rerank so the exact float repairs the coarse Hamming beam.
|
||||
let idx = QuantizedHnswIndex::build(&vectors, dim, Metric::L2, params(0xAAAA), 0x5EED, 64);
|
||||
|
||||
let mut total = 0.0f64;
|
||||
let n_queries = 64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut cs = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let mut s = seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
let truth: HashSet<u32> = idx
|
||||
.graph()
|
||||
.brute_force(&qv, 10)
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
let got = idx.search_quantized(&qv, 10, 128, 64);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / 10.0;
|
||||
}
|
||||
let recall = total / n_queries as f64;
|
||||
// The 1-bit code is coarse, so we do not demand the float 0.95 gate here;
|
||||
// but with a 64-wide rerank over an ef=128 beam it must be clearly useful
|
||||
// (well above random). ADR-261 reports the exact number; this gate just
|
||||
// catches a broken traversal/rerank.
|
||||
assert!(
|
||||
recall >= 0.80,
|
||||
"quantized recall@10 = {recall:.4} too low — traversal or rerank bug"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_dim_no_panic() {
|
||||
let vectors = vec![vec![], vec![]];
|
||||
let idx = QuantizedHnswIndex::build(&vectors, 0, Metric::Cosine, params(5), 0x1, 4);
|
||||
let r = idx.search_quantized(&[], 2, 16, 4);
|
||||
assert_eq!(r.len(), 2);
|
||||
}
|
||||
|
||||
// ----- multi-bit (ADR-261 §11) -----
|
||||
|
||||
/// `bits == 1` via `build_bits` is byte-for-byte the legacy `build` 1-bit
|
||||
/// construction: same codes, same search output. Backward-compatibility pin.
|
||||
#[test]
|
||||
fn one_bit_build_bits_matches_legacy_build() {
|
||||
let vectors = planted(32, 400, 8, 0x1B17);
|
||||
let legacy = QuantizedHnswIndex::build(&vectors, 32, Metric::L2, params(0x5151), 0xC0DE, 40);
|
||||
let viabits =
|
||||
QuantizedHnswIndex::build_bits(&vectors, 32, Metric::L2, params(0x5151), 0xC0DE, 1, 40);
|
||||
assert_eq!(legacy.bits(), 1);
|
||||
assert_eq!(viabits.bits(), 1);
|
||||
let q = &vectors[123];
|
||||
assert_eq!(
|
||||
legacy.search_quantized(q, 10, 64, 40),
|
||||
viabits.search_quantized(q, 10, 64, 40),
|
||||
"build_bits(…,1,…) must equal legacy build(…)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Unsupported bit-depths round up to the supported `{1,2,4}` set so the
|
||||
/// constructor is total (no panic, predictable resolution).
|
||||
#[test]
|
||||
fn bits_are_clamped_to_supported_set() {
|
||||
let vectors = planted(16, 50, 4, 0xB175);
|
||||
for (req, exp) in [(0u32, 1u32), (1, 1), (2, 2), (3, 4), (4, 4), (7, 4)] {
|
||||
let idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
16,
|
||||
Metric::L2,
|
||||
params(0x9),
|
||||
0xB,
|
||||
req,
|
||||
16,
|
||||
);
|
||||
assert_eq!(idx.bits(), exp, "bits {req} should clamp to {exp}");
|
||||
// and it must still search without panic
|
||||
assert!(!idx.search_quantized(&vectors[0], 5, 32, 20).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes/node scales linearly with `bits`: for a power-of-two dim `D`,
|
||||
/// 1-bit → D/8, 2-bit → D/4, 4-bit → D/2.
|
||||
#[test]
|
||||
fn bytes_per_node_scales_with_bits() {
|
||||
let vectors = planted(128, 20, 4, 0xBEEF);
|
||||
let b1 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 1, 16);
|
||||
let b2 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 2, 16);
|
||||
let b4 = QuantizedHnswIndex::build_bits(&vectors, 128, Metric::L2, params(1), 0x5, 4, 16);
|
||||
assert_eq!(b1.bytes_per_node(), 16, "128-d 1-bit = 16 B/node");
|
||||
assert_eq!(b2.bytes_per_node(), 32, "128-d 2-bit = 32 B/node");
|
||||
assert_eq!(b4.bytes_per_node(), 64, "128-d 4-bit = 64 B/node");
|
||||
}
|
||||
|
||||
/// More bits must not *reduce* recall at a fixed (ef, rerank): the multi-bit
|
||||
/// code is a strictly finer angle proxy than 1-bit, so the traversal beam can
|
||||
/// only land on equal-or-better candidates for the rerank to repair. This is
|
||||
/// the core ADR-261 §11 hypothesis (multi-bit keeps the beam on-path better),
|
||||
/// pinned as a regression gate. We assert a small tolerance for ties.
|
||||
#[test]
|
||||
fn more_bits_does_not_reduce_recall() {
|
||||
let dim = 64;
|
||||
let n = 3000;
|
||||
let clusters = 32;
|
||||
let seed = 0x7A11;
|
||||
let vectors = planted(dim, n, clusters, seed);
|
||||
let recall_for = |bits: u32| -> f64 {
|
||||
let idx = QuantizedHnswIndex::build_bits(
|
||||
&vectors,
|
||||
dim,
|
||||
Metric::L2,
|
||||
params(0xA11A),
|
||||
0x5EED,
|
||||
bits,
|
||||
// Modest rerank so traversal quality — not a huge rerank pool —
|
||||
// is what drives the recall difference between bit depths.
|
||||
20,
|
||||
);
|
||||
let mut total = 0.0f64;
|
||||
let n_queries = 64;
|
||||
for q in 0..n_queries {
|
||||
let c = q % clusters;
|
||||
let mut cs = seed ^ (0xC0FFEE_u64.wrapping_mul(c as u64 + 1));
|
||||
let centre: Vec<f32> = (0..dim).map(|_| gauss(&mut cs) * 3.0).collect();
|
||||
let mut s = seed ^ 0xDEAD_0000 ^ (q as u64).wrapping_mul(0x2545_F491);
|
||||
let qv: Vec<f32> = (0..dim).map(|d| centre[d] + gauss(&mut s) * 0.35).collect();
|
||||
let truth: HashSet<u32> = idx
|
||||
.graph()
|
||||
.brute_force(&qv, 10)
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
let got = idx.search_quantized(&qv, 10, 64, 20);
|
||||
let hit = got.iter().filter(|(id, _)| truth.contains(id)).count();
|
||||
total += hit as f64 / 10.0;
|
||||
}
|
||||
total / n_queries as f64
|
||||
};
|
||||
let r1 = recall_for(1);
|
||||
let r2 = recall_for(2);
|
||||
let r4 = recall_for(4);
|
||||
// 2-bit and 4-bit must be at least as good as 1-bit (small tie tolerance).
|
||||
assert!(
|
||||
r2 + 0.02 >= r1,
|
||||
"2-bit recall {r2:.4} regressed vs 1-bit {r1:.4}"
|
||||
);
|
||||
assert!(
|
||||
r4 + 0.02 >= r1,
|
||||
"4-bit recall {r4:.4} regressed vs 1-bit {r1:.4}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -28,9 +28,12 @@
|
||||
|
||||
#[cfg(feature = "crv")]
|
||||
pub mod crv;
|
||||
pub mod ann_measure;
|
||||
pub mod coverage;
|
||||
pub mod estimator;
|
||||
pub mod event_log;
|
||||
pub mod hnsw;
|
||||
pub mod hnsw_quantized;
|
||||
pub mod mat;
|
||||
pub mod rotation;
|
||||
pub mod signal;
|
||||
@@ -41,6 +44,8 @@ pub use estimator::{
|
||||
DistanceEstimator, EstimatorBank, EstimatorQuery, EstimatorSketch, SideInfo,
|
||||
};
|
||||
pub use event_log::{NoveltyEvent, PrivacyEventLog};
|
||||
pub use hnsw::{HnswIndex, HnswParams, Metric};
|
||||
pub use hnsw_quantized::QuantizedHnswIndex;
|
||||
pub use rotation::Rotation;
|
||||
pub use sketch::{
|
||||
Sketch, SketchBank, SketchError, WireSketch, WireSketchError, WIRE_SKETCH_FORMAT_VERSION,
|
||||
|
||||
@@ -63,6 +63,13 @@ wifi-densepose-worldgraph = { version = "0.3.0", path = "../wifi-densepose-world
|
||||
wifi-densepose-bfld = { version = "0.3.1", path = "../wifi-densepose-bfld", default-features = false }
|
||||
wifi-densepose-geo = { version = "0.1.0", path = "../wifi-densepose-geo" }
|
||||
|
||||
# ADR-262 P3: live RuField surface. The thin anti-corruption bridge that turns
|
||||
# this server's governed sensing cycle into signed RuField `FieldEvent`s on
|
||||
# `/api/field` + `/ws/field`. It path-deps the standalone `vendor/rufield`
|
||||
# submodule (it is the single coupling point — ADR-262 §5.4) and pulls in no
|
||||
# RuView internal crate, so the dep surface added here is just the bridge.
|
||||
wifi-densepose-rufield = { version = "0.3.0", path = "../wifi-densepose-rufield" }
|
||||
|
||||
# midstream — real-time introspection / low-latency tap (ADR-099 D1).
|
||||
# Two crates only, on purpose: scheduler / neural-solver / strange-loop are
|
||||
# explicitly out of scope of ADR-099 (D5).
|
||||
|
||||
@@ -23,6 +23,10 @@ pub mod model_format;
|
||||
pub mod mqtt;
|
||||
pub mod path_safety;
|
||||
pub mod semantic;
|
||||
/// ADR-262 P3: the live RuField surface — turns the governed sensing cycle into
|
||||
/// signed RuField `FieldEvent`s on the additive `/api/field` + `/ws/field`
|
||||
/// endpoints, via the `wifi-densepose-rufield` anti-corruption bridge.
|
||||
pub mod rufield_surface;
|
||||
pub mod rvf_container;
|
||||
pub mod rvf_pipeline;
|
||||
pub mod sona;
|
||||
|
||||
@@ -26,7 +26,7 @@ mod vital_signs;
|
||||
|
||||
// Training pipeline modules (exposed via lib.rs)
|
||||
use wifi_densepose_sensing_server::{
|
||||
dataset, embedding, error_response, graph_transformer, trainer,
|
||||
dataset, embedding, error_response, graph_transformer, rufield_surface, trainer,
|
||||
};
|
||||
|
||||
use ruvector_mincut::{DynamicMinCut, MinCutBuilder};
|
||||
@@ -1093,6 +1093,14 @@ struct AppStateInner {
|
||||
pub(crate) dedup_factor: f64,
|
||||
/// Data directory for persisting runtime config (parent of `firmware_dir`).
|
||||
pub(crate) data_dir: std::path::PathBuf,
|
||||
/// ADR-262 P3: the live RuField surface. Holds the dedicated ed25519 signer
|
||||
/// + a bounded ring of recent signed `FieldEvent`s + the `/ws/field`
|
||||
/// broadcast topic. The governed sensing cycle calls `emit()` on it once per
|
||||
/// cycle (joining `SensingUpdate` features/classification/signal_field with
|
||||
/// the `TrustedOutput` trust class); `/api/field` + `/ws/field` read it.
|
||||
/// Held behind its own `Arc<RwLock<_>>` so the additive field router can
|
||||
/// take it as state without re-locking `AppStateInner`.
|
||||
field_surface: rufield_surface::FieldState,
|
||||
}
|
||||
|
||||
/// If no ESP32 frame arrives within this duration, source reverts to offline.
|
||||
@@ -4000,6 +4008,80 @@ fn derive_single_person_pose(
|
||||
/// the strongest peak so they remain co-located with real energy rather than at
|
||||
/// a fake origin; if the field has no peak above threshold the position stays at
|
||||
/// `[0,0,0]` and `motion_score` still reflects real motion power.
|
||||
/// ADR-262 P3: emit one signed RuField `FieldEvent` for this sensing cycle.
|
||||
///
|
||||
/// Joins the cycle's [`SensingUpdate`] (features / classification /
|
||||
/// signal_field) with the governed engine's trust state (`effective_class` /
|
||||
/// `demoted`, recorded on `engine_bridge` by `observe_cycle`) into a
|
||||
/// `SensingSnapshot`, then surfaces it via the P1 bridge on `/api/field` +
|
||||
/// `/ws/field`. The bridge maps privacy by information content and the surface
|
||||
/// applies the §10 network egress gate, so above-policy cycles never reach the
|
||||
/// wire.
|
||||
///
|
||||
/// **No phantom events:** an empty/no-presence cycle (`presence == false`)
|
||||
/// emits nothing — there is no person to describe, so no event is fabricated
|
||||
/// (ADR-262 §4 P3 / §6). Cycles before the governed engine has produced a trust
|
||||
/// class are likewise skipped (no class ⇒ nothing honest to stamp).
|
||||
///
|
||||
/// `identity_bound` is `false` on the live path: RuView's live cycle does not
|
||||
/// bind an enrolled identity to the surface yet (that is a per-room-calibration
|
||||
/// / AETHER concern, ADR-262 §8 Q4). This is conservative for egress — it only
|
||||
/// ever *lowers* a Derived cycle from P5 to P4, both of which are already held
|
||||
/// edge-local, so it cannot leak.
|
||||
fn emit_rufield_event(s: &AppStateInner, update: &SensingUpdate, node_id: u8) {
|
||||
// No-presence ⇒ no phantom event.
|
||||
if !update.classification.presence {
|
||||
return;
|
||||
}
|
||||
// Need a governed trust class before we can honestly stamp privacy.
|
||||
let Some(effective_class) = s.engine_bridge.effective_class() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let timestamp_ns = if update.timestamp.is_finite() && update.timestamp > 0.0 {
|
||||
(update.timestamp * 1_000_000_000.0) as u64
|
||||
} else {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as u64)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let snap = rufield_surface::build_snapshot(
|
||||
timestamp_ns,
|
||||
format!("esp32_node_{node_id}"),
|
||||
rufield_surface::SensingFeatures {
|
||||
mean_rssi: update.features.mean_rssi,
|
||||
variance: update.features.variance,
|
||||
motion_band_power: update.features.motion_band_power,
|
||||
breathing_band_power: update.features.breathing_band_power,
|
||||
dominant_freq_hz: update.features.dominant_freq_hz,
|
||||
change_points: update.features.change_points,
|
||||
spectral_power: update.features.spectral_power,
|
||||
},
|
||||
rufield_surface::SensingClass {
|
||||
motion_level: update.classification.motion_level.clone(),
|
||||
presence: update.classification.presence,
|
||||
confidence: update.classification.confidence,
|
||||
},
|
||||
Some(rufield_surface::SignalField {
|
||||
grid_size: update.signal_field.grid_size,
|
||||
values: update.signal_field.values.clone(),
|
||||
}),
|
||||
rufield_surface::ruview_class_from_bfld(effective_class),
|
||||
s.engine_bridge.demoted(),
|
||||
false, // identity_bound — see fn-doc (conservative, cannot leak).
|
||||
);
|
||||
|
||||
// `field_surface` is its own Arc<RwLock<_>>; `try_write` is non-blocking and
|
||||
// never deadlocks against the `s` guard (a different lock). The only other
|
||||
// touchers are the read-only `/api/field` / `/ws/field` handlers, so
|
||||
// contention is negligible; a rare miss just drops one cycle's event.
|
||||
if let Ok(mut fs) = s.field_surface.try_write() {
|
||||
fs.emit(&snap);
|
||||
}
|
||||
}
|
||||
|
||||
fn attach_field_positions(update: &mut SensingUpdate) {
|
||||
let Some(persons) = update.persons.as_mut() else {
|
||||
return;
|
||||
@@ -5990,6 +6072,18 @@ async fn udp_receiver_task(state: SharedState, udp_port: u16) {
|
||||
if let Ok(json) = serde_json::to_string(&update) {
|
||||
let _ = s.tx.send(json);
|
||||
}
|
||||
|
||||
// ── ADR-262 P3: emit a signed RuField FieldEvent ────────
|
||||
// Join this cycle's SensingUpdate (features / classification
|
||||
// / signal_field) with the governed engine's trust state
|
||||
// (effective_class / demoted, recorded by `observe_cycle`
|
||||
// above) into a `SensingSnapshot`, and surface it on
|
||||
// `/api/field` + `/ws/field` via the P1 bridge. Only cycles
|
||||
// whose mapped privacy class clears the §10 network egress
|
||||
// gate are surfaced (P1/P2); a `Derived → P4/P5` cycle is
|
||||
// held edge-local. `presence == false` ⇒ no phantom event.
|
||||
emit_rufield_event(&s, &update, node_id);
|
||||
|
||||
s.latest_update = Some(update);
|
||||
|
||||
// Evict stale nodes every 100 ticks to prevent memory leak.
|
||||
@@ -7322,6 +7416,13 @@ async fn main() {
|
||||
);
|
||||
}
|
||||
|
||||
// ADR-262 P3: build the live RuField surface (dedicated ed25519 signer from
|
||||
// WDP_RUFIELD_SIGNING_SEED, else a logged dev default). The same Arc is
|
||||
// stored in AppStateInner (so the sensing loop can `emit()` per cycle) and
|
||||
// cloned into the additive `/api/field` + `/ws/field` router below.
|
||||
let field_surface: rufield_surface::FieldState =
|
||||
Arc::new(RwLock::new(rufield_surface::FieldSurface::from_env()));
|
||||
|
||||
let state: SharedState = Arc::new(RwLock::new(AppStateInner {
|
||||
latest_update: None,
|
||||
rssi_history: VecDeque::new(),
|
||||
@@ -7424,6 +7525,7 @@ async fn main() {
|
||||
// ADR-044 §5.3: runtime-configurable dedup factor (persisted).
|
||||
dedup_factor: runtime_config.dedup_factor,
|
||||
data_dir: data_dir.clone(),
|
||||
field_surface: field_surface.clone(),
|
||||
}));
|
||||
|
||||
// Start background tasks from the resolved plan (issue #1004).
|
||||
@@ -7497,11 +7599,15 @@ async fn main() {
|
||||
let ws_app = Router::new()
|
||||
.route("/ws/sensing", get(ws_sensing_handler))
|
||||
.route("/health", get(health))
|
||||
.with_state(ws_state)
|
||||
// ADR-262 P3: additive `/ws/field` (+ `/api/field`) on the WS port too,
|
||||
// so a client on :8765 can stream signed RuField FieldEvents alongside
|
||||
// `/ws/sensing`. Merged with its own FieldState (different state type).
|
||||
.merge(rufield_surface::router(field_surface.clone()))
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
host_allowlist.clone(),
|
||||
wifi_densepose_sensing_server::host_validation::require_allowed_host,
|
||||
))
|
||||
.with_state(ws_state);
|
||||
));
|
||||
|
||||
let ws_addr = SocketAddr::from((bind_ip, args.ws_port));
|
||||
let ws_listener = tokio::net::TcpListener::bind(ws_addr)
|
||||
@@ -7615,15 +7721,24 @@ async fn main() {
|
||||
bearer_auth_state.clone(),
|
||||
wifi_densepose_sensing_server::bearer_auth::require_bearer,
|
||||
))
|
||||
.with_state(state.clone())
|
||||
// ADR-262 P3: additive RuField surface (`/api/field` + `/ws/field`).
|
||||
// Merged AFTER `.with_state` (so http_app is already `Router<()>` and
|
||||
// can absorb the field router's own `FieldState`). These routes sit
|
||||
// OUTSIDE `/api/v1/*` so they are not bearer-gated, but the
|
||||
// host-validation layer below still applies (it is added last, so it
|
||||
// runs first, over the whole merged router). The surface's own §10
|
||||
// egress gate is what keeps above-policy classes off the wire.
|
||||
.merge(rufield_surface::router(field_surface.clone()))
|
||||
// DNS-rebinding defense: applied last so it runs first on the request
|
||||
// path (axum layers run outermost-in). Rejects requests whose `Host`
|
||||
// header is not in the allowlist before any handler — including
|
||||
// `/health` and `/ws/*` — observes the body.
|
||||
// `/health`, `/ws/*`, and the merged `/api/field` + `/ws/field` —
|
||||
// observes the body.
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
host_allowlist.clone(),
|
||||
wifi_densepose_sensing_server::host_validation::require_allowed_host,
|
||||
))
|
||||
.with_state(state.clone());
|
||||
));
|
||||
|
||||
let http_addr = SocketAddr::from((bind_ip, args.http_port));
|
||||
let http_listener = tokio::net::TcpListener::bind(http_addr)
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
//! ADR-262 **P3** — the live RuField surface.
|
||||
//!
|
||||
//! This is the data-path wiring that turns RuView's governed sensing cycle into
|
||||
//! signed RuField [`FieldEvent`]s on two **additive** network endpoints:
|
||||
//!
|
||||
//! - `GET /api/field` — the most recent surfaced `FieldEvent`(s) as JSON;
|
||||
//! - `GET /ws/field` — a WebSocket that streams each cycle's `FieldEvent`
|
||||
//! (mirrors the `/ws/sensing` broadcast-subscribe pattern).
|
||||
//!
|
||||
//! It is purely additive: `/ws/sensing` and every existing endpoint are
|
||||
//! unchanged. The conversion itself lives entirely in the P1
|
||||
//! [`wifi_densepose_rufield`] anti-corruption bridge (ADR-262 §5.4 — the single
|
||||
//! coupling point); this module only (a) holds the dedicated signer + a bounded
|
||||
//! ring buffer of recent events in server state, (b) builds a
|
||||
//! [`SensingSnapshot`] from the **same real data** the cycle already produced
|
||||
//! (`SensingUpdate` features/classification/signal_field joined with the
|
||||
//! governed-engine [`TrustedOutput`] trust state at `main.rs:~5886`/`:~5938`),
|
||||
//! and (c) applies the §10 network egress gate so above-policy classes never
|
||||
//! reach the wire.
|
||||
//!
|
||||
//! ## Honesty (ADR-262 §0 / §6)
|
||||
//!
|
||||
//! This wires **real** RuView sensing into RuField events on a live endpoint,
|
||||
//! but: (a) it is the **single-link CSI** sensing with its existing caveats —
|
||||
//! there is **no validated room-coordinate accuracy** (`field_localize` says so;
|
||||
//! positions are "strongest field peak", not triangulation); (b) the signing
|
||||
//! key is a **dedicated dev/sensing key** pending the ADR-262 §8 Q1 ownership
|
||||
//! decision (reusing the `cog-ha-matter` Ed25519 key is the **deferred P2**
|
||||
//! call — P3 deliberately uses a standalone key so it does not pre-empt that);
|
||||
//! (c) **no accuracy is claimed.** The win is narrowly: "RuView's live sensing
|
||||
//! now speaks RuField on `/ws/field`."
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
State,
|
||||
},
|
||||
response::{IntoResponse, Json},
|
||||
};
|
||||
use tokio::sync::{broadcast, RwLock};
|
||||
|
||||
// Re-export the bridge input types `main.rs` needs to build a snapshot, so the
|
||||
// server-side call site depends only on `rufield_surface` (the server seam).
|
||||
pub use wifi_densepose_rufield::{
|
||||
network_egress_allowed, snapshot_to_field_event, FieldEvent, RuViewPrivacyClass,
|
||||
SensingClass, SensingFeatures, SensingSnapshot, Signer, SignalField,
|
||||
};
|
||||
|
||||
/// How many recent surfaced `FieldEvent`s the ring buffer retains. Small and
|
||||
/// bounded — this is a live tap, not a store (ADR-262 §4 P3 "small bounded ring
|
||||
/// buffer of recent events").
|
||||
pub const FIELD_RING_CAPACITY: usize = 64;
|
||||
|
||||
/// Broadcast channel depth for `/ws/field`. Matches the `/ws/sensing` `tx`
|
||||
/// channel size (256) so a slow field client drops messages rather than
|
||||
/// stalling the sensing loop.
|
||||
pub const FIELD_BROADCAST_CAPACITY: usize = 256;
|
||||
|
||||
/// Environment variable carrying the 32-byte hex/raw signing seed for the
|
||||
/// dedicated RuField sensing signer. When unset, a deterministic dev default is
|
||||
/// used (with a logged warning). See [`FieldSurface::from_env`].
|
||||
pub const SIGNING_SEED_ENV: &str = "WDP_RUFIELD_SIGNING_SEED";
|
||||
|
||||
/// Deterministic dev signing seed used when [`SIGNING_SEED_ENV`] is unset. This
|
||||
/// is a **dev/sensing key**, intentionally standalone (ADR-262 §8 Q1 — the
|
||||
/// `cog-ha-matter` key reuse is the deferred P2 decision, not pre-empted here).
|
||||
const DEV_SIGNING_SEED: &[u8; 32] = b"adr262-ruview-rufield-dev-seed!!";
|
||||
|
||||
/// The live RuField surface state held in `AppStateInner` (ADR-262 P3).
|
||||
///
|
||||
/// Owns the **dedicated** ed25519 [`Signer`], a bounded ring buffer of the most
|
||||
/// recent network-surfaced events, and the `/ws/field` broadcast sender.
|
||||
pub struct FieldSurface {
|
||||
signer: Signer,
|
||||
/// Bounded ring of recent **network-surfaced** events (most recent last).
|
||||
recent: VecDeque<FieldEvent>,
|
||||
/// Broadcast topic for `/ws/field` (JSON-serialized `FieldEvent`s).
|
||||
tx: broadcast::Sender<String>,
|
||||
/// True when the dev default seed is in use (drives a one-time warning and
|
||||
/// is surfaced in `/api/field` metadata so operators can see they are on a
|
||||
/// dev key).
|
||||
using_dev_key: bool,
|
||||
}
|
||||
|
||||
impl FieldSurface {
|
||||
/// Build a surface with an explicit 32-byte seed (deterministic signer).
|
||||
#[must_use]
|
||||
pub fn from_seed(seed: &[u8; 32], using_dev_key: bool) -> Self {
|
||||
let (tx, _rx) = broadcast::channel(FIELD_BROADCAST_CAPACITY);
|
||||
Self {
|
||||
signer: Signer::from_seed(seed),
|
||||
recent: VecDeque::with_capacity(FIELD_RING_CAPACITY),
|
||||
tx,
|
||||
using_dev_key,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a surface from the environment (ADR-262 §4 P3 / open-question 1).
|
||||
///
|
||||
/// Reads [`SIGNING_SEED_ENV`] as either a 64-char hex string or a raw 32+
|
||||
/// byte UTF-8 value (first 32 bytes used). When unset/invalid it falls back
|
||||
/// to the deterministic [`DEV_SIGNING_SEED`] and logs a `WARN` — the key is
|
||||
/// a standalone **dev/sensing** key, NOT the deferred-P2 `cog-ha-matter`
|
||||
/// key.
|
||||
#[must_use]
|
||||
pub fn from_env() -> Self {
|
||||
match std::env::var(SIGNING_SEED_ENV).ok().and_then(|v| parse_seed(&v)) {
|
||||
Some(seed) => {
|
||||
tracing::info!(
|
||||
"ADR-262 P3: RuField surface using signing seed from {SIGNING_SEED_ENV} \
|
||||
(dedicated sensing key)"
|
||||
);
|
||||
Self::from_seed(&seed, false)
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"ADR-262 P3: {SIGNING_SEED_ENV} unset/invalid — RuField surface using the \
|
||||
DETERMINISTIC DEV signing key. This is a dev/sensing key pending the \
|
||||
ADR-262 §8 Q1 (P2) key-ownership decision; set {SIGNING_SEED_ENV} (64-hex \
|
||||
or 32-byte value) for a real deployment."
|
||||
);
|
||||
Self::from_seed(DEV_SIGNING_SEED, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The public key of the dedicated signer (hex), so consumers can verify
|
||||
/// receipts without the private seed.
|
||||
#[must_use]
|
||||
pub fn signer_pubkey_hex(&self) -> String {
|
||||
self.signer.public_hex()
|
||||
}
|
||||
|
||||
/// Whether the dev default key is in use.
|
||||
#[must_use]
|
||||
pub fn using_dev_key(&self) -> bool {
|
||||
self.using_dev_key
|
||||
}
|
||||
|
||||
/// A `/ws/field` subscription.
|
||||
#[must_use]
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<String> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
|
||||
/// The most recent surfaced events, oldest→newest.
|
||||
#[must_use]
|
||||
pub fn recent(&self) -> Vec<FieldEvent> {
|
||||
self.recent.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Convert one cycle's [`SensingSnapshot`] into a signed [`FieldEvent`],
|
||||
/// apply the §10 network egress gate, and — **iff** the event may leave the
|
||||
/// box — push it into the ring + broadcast it on `/ws/field`.
|
||||
///
|
||||
/// Returns `Some(event)` when an event was surfaced, `None` when the cycle
|
||||
/// was held edge-local (above network policy — e.g. a `Derived → P4/P5`
|
||||
/// cycle) or carried no presence. Two structural guarantees live here, so
|
||||
/// they hold regardless of caller:
|
||||
///
|
||||
/// - **no phantom events** — a no-presence cycle (`presence == false`)
|
||||
/// surfaces nothing (ADR-262 §4 P3 / §6); there is no person to describe.
|
||||
/// - **privacy-safety pin** — above-policy classes (P0, P3–P5) are never
|
||||
/// placed on the network surface; only egress-safe P1/P2 events leave.
|
||||
pub fn emit(&mut self, snap: &SensingSnapshot) -> Option<FieldEvent> {
|
||||
// No-presence ⇒ no phantom event (fabricating one would be dishonest).
|
||||
if !snap.classification.presence {
|
||||
return None;
|
||||
}
|
||||
|
||||
let event = snapshot_to_field_event(snap, &self.signer);
|
||||
|
||||
// §10 network egress gate (ADR-262 §4 P3): only P1/P2 leave the box by
|
||||
// default; P0 raw and P3/P4/P5 (above the default P2 ceiling, or
|
||||
// identity/biometric) are held edge-local. A `Derived` cycle is P4/P5
|
||||
// ⇒ never surfaced as a low-privacy network event.
|
||||
if !network_egress_allowed(event.observation.privacy_class, snap.identity_bound) {
|
||||
tracing::trace!(
|
||||
privacy_class = ?event.observation.privacy_class,
|
||||
"ADR-262 P3: cycle held edge-local (above network policy), not surfaced on /api/field"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
if self.recent.len() == FIELD_RING_CAPACITY {
|
||||
self.recent.pop_front();
|
||||
}
|
||||
self.recent.push_back(event.clone());
|
||||
|
||||
if let Ok(json) = serde_json::to_string(&event) {
|
||||
let _ = self.tx.send(json);
|
||||
}
|
||||
Some(event)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse [`SIGNING_SEED_ENV`] as 64-char hex or a raw 32+ byte UTF-8 value.
|
||||
fn parse_seed(v: &str) -> Option<[u8; 32]> {
|
||||
let v = v.trim();
|
||||
// 64 hex chars → 32 bytes.
|
||||
if v.len() == 64 && v.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
let mut out = [0u8; 32];
|
||||
for (i, chunk) in v.as_bytes().chunks(2).enumerate() {
|
||||
let hi = (chunk[0] as char).to_digit(16)?;
|
||||
let lo = (chunk[1] as char).to_digit(16)?;
|
||||
out[i] = ((hi << 4) | lo) as u8;
|
||||
}
|
||||
return Some(out);
|
||||
}
|
||||
// Otherwise: first 32 bytes of the raw value (must be at least 32 long so a
|
||||
// short/typo'd value fails closed to the dev key rather than a weak key).
|
||||
let bytes = v.as_bytes();
|
||||
if bytes.len() >= 32 {
|
||||
let mut out = [0u8; 32];
|
||||
out.copy_from_slice(&bytes[..32]);
|
||||
return Some(out);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Build a [`SensingSnapshot`] from the real per-cycle values (ADR-262 P3 §4.2).
|
||||
///
|
||||
/// This is the join the ADR mandates: `SensingUpdate` features / classification
|
||||
/// / signal-field **plus** the governed engine's `effective_class` / `demoted`
|
||||
/// / `identity_bound` trust state. All inputs are the same real data the cycle
|
||||
/// already computed — nothing is fabricated. `signal_field` is passed through as
|
||||
/// the honest "strongest field peak" readout (no calibrated coordinates).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[must_use]
|
||||
pub fn build_snapshot(
|
||||
timestamp_ns: u64,
|
||||
node_id: String,
|
||||
features: SensingFeatures,
|
||||
classification: SensingClass,
|
||||
signal_field: Option<SignalField>,
|
||||
trust_class: RuViewPrivacyClass,
|
||||
demoted: bool,
|
||||
identity_bound: bool,
|
||||
) -> SensingSnapshot {
|
||||
SensingSnapshot {
|
||||
timestamp_ns,
|
||||
features,
|
||||
classification,
|
||||
signal_field,
|
||||
trust_class,
|
||||
demoted,
|
||||
identity_bound,
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map RuView's live governed-engine `bfld::PrivacyClass` (the `effective_class`
|
||||
/// on `TrustedOutput`) onto the bridge's [`RuViewPrivacyClass`] input.
|
||||
///
|
||||
/// This is a **lossless, same-meaning** re-encoding of the four byte-level
|
||||
/// classes — both enums are `Raw/Derived/Anonymous/Restricted` in the same
|
||||
/// order. It exists only so `main.rs` can pass the engine's class into the
|
||||
/// bridge without the bridge depending on `wifi-densepose-bfld` (keeping it an
|
||||
/// anti-corruption layer, ADR-262 §5.4). The information-content privacy
|
||||
/// mapping (the §3.3 correctness item) happens *inside* the bridge.
|
||||
#[must_use]
|
||||
pub fn ruview_class_from_bfld(class: wifi_densepose_bfld::PrivacyClass) -> RuViewPrivacyClass {
|
||||
use wifi_densepose_bfld::PrivacyClass as B;
|
||||
match class {
|
||||
B::Raw => RuViewPrivacyClass::Raw,
|
||||
B::Derived => RuViewPrivacyClass::Derived,
|
||||
B::Anonymous => RuViewPrivacyClass::Anonymous,
|
||||
B::Restricted => RuViewPrivacyClass::Restricted,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Handlers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Shared state for the field surface handlers. Generic over the lock guard so
|
||||
/// the module can be tested in isolation with a tiny state (ADR-262 P3 test
|
||||
/// gate) and wired into the full `AppStateInner` in `main.rs` via an adapter.
|
||||
pub type FieldState = Arc<RwLock<FieldSurface>>;
|
||||
|
||||
/// `GET /api/field` — the most recent network-surfaced `FieldEvent`s as JSON,
|
||||
/// plus surface metadata (the signer pubkey + whether a dev key is in use).
|
||||
///
|
||||
/// When no event has been surfaced yet (empty room / above-policy cycles only)
|
||||
/// the `events` array is empty — an **explicit empty payload**, never a
|
||||
/// fabricated event (ADR-262 §4 P3 / §6 honesty).
|
||||
pub async fn api_field(State(state): State<FieldState>) -> Json<serde_json::Value> {
|
||||
let s = state.read().await;
|
||||
Json(serde_json::json!({
|
||||
"spec": "rufield",
|
||||
"endpoint": "/api/field",
|
||||
"signer_pubkey_hex": s.signer_pubkey_hex(),
|
||||
"dev_signing_key": s.using_dev_key(),
|
||||
"events": s.recent(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// `GET /ws/field` — upgrade to a WebSocket that streams each surfaced
|
||||
/// `FieldEvent` (JSON) as the sensing loop emits it. Mirrors `/ws/sensing`:
|
||||
/// subscribe to the broadcast topic and forward.
|
||||
pub async fn ws_field(ws: WebSocketUpgrade, State(state): State<FieldState>) -> impl IntoResponse {
|
||||
let rx = {
|
||||
let s = state.read().await;
|
||||
s.subscribe()
|
||||
};
|
||||
ws.on_upgrade(move |socket| handle_ws_field_client(socket, rx))
|
||||
}
|
||||
|
||||
async fn handle_ws_field_client(mut socket: WebSocket, mut rx: broadcast::Receiver<String>) {
|
||||
// Forward broadcast events; exit on client close or fatal lag.
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(json) => {
|
||||
if socket.send(Message::Text(json)).await.is_err() {
|
||||
break; // client gone
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||
// Slow client missed events — keep going from the latest.
|
||||
continue;
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the additive field-surface router. Mounted into the main HTTP router
|
||||
/// in `main.rs`; also used standalone by the integration tests (ADR-262 P3
|
||||
/// gate, `tower::oneshot`).
|
||||
#[must_use]
|
||||
pub fn router(state: FieldState) -> axum::Router {
|
||||
use axum::routing::get;
|
||||
axum::Router::new()
|
||||
.route("/api/field", get(api_field))
|
||||
.route("/ws/field", get(ws_field))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use wifi_densepose_rufield::{is_fusable, PrivacyClass};
|
||||
|
||||
fn features() -> SensingFeatures {
|
||||
SensingFeatures {
|
||||
mean_rssi: -55.0,
|
||||
variance: 0.4,
|
||||
motion_band_power: 2.0,
|
||||
breathing_band_power: 0.3,
|
||||
dominant_freq_hz: 0.25,
|
||||
change_points: 1,
|
||||
spectral_power: 3.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn present_class() -> SensingClass {
|
||||
SensingClass {
|
||||
motion_level: "low".into(),
|
||||
presence: true,
|
||||
confidence: 0.82,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_seed_hex_and_raw_and_short() {
|
||||
// 64 hex chars → 32 bytes.
|
||||
let hex = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff";
|
||||
let parsed = parse_seed(hex).expect("valid hex seed");
|
||||
assert_eq!(parsed[0], 0x00);
|
||||
assert_eq!(parsed[31], 0xff);
|
||||
// Raw 32-byte value.
|
||||
assert!(parse_seed("0123456789abcdef0123456789abcdef").is_some());
|
||||
// Too short → fail closed (None → dev key).
|
||||
assert!(parse_seed("short").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anonymous_cycle_surfaces_fusable_event() {
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Anonymous, // → P2, network-allowed
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let ev = surface.emit(&snap).expect("anonymous P2 cycle is surfaced");
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert!(is_fusable(&ev), "live event must be ed25519-signed & fusable");
|
||||
assert_eq!(surface.recent().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_cycle_never_surfaces_low_privacy() {
|
||||
// The privacy-safety pin: a Derived (identity) cycle maps to P4/P5 and
|
||||
// is held edge-local — it must NEVER appear on the network surface.
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
for identity_bound in [false, true] {
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Derived,
|
||||
false,
|
||||
identity_bound,
|
||||
);
|
||||
assert!(
|
||||
surface.emit(&snap).is_none(),
|
||||
"Derived cycle (identity_bound={identity_bound}) must be held edge-local"
|
||||
);
|
||||
}
|
||||
assert!(surface.recent().is_empty(), "no Derived event may reach the surface");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ring_buffer_is_bounded() {
|
||||
let mut surface = FieldSurface::from_seed(DEV_SIGNING_SEED, true);
|
||||
for i in 0..(FIELD_RING_CAPACITY + 10) {
|
||||
let snap = build_snapshot(
|
||||
1_791_986_400_000_000_000 + i as u64,
|
||||
"esp32_room_01".into(),
|
||||
features(),
|
||||
present_class(),
|
||||
None,
|
||||
RuViewPrivacyClass::Anonymous,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
surface.emit(&snap);
|
||||
}
|
||||
assert_eq!(surface.recent().len(), FIELD_RING_CAPACITY);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! ADR-262 **P3** acceptance gate — the live RuField surface.
|
||||
//!
|
||||
//! In-process integration test (mirrors the `/ws/sensing` / #1050 oneshot
|
||||
//! style with `tower::ServiceExt::oneshot`): drives synthetic sensing cycles
|
||||
//! through the real `FieldSurface` + the real `/api/field` router, and asserts:
|
||||
//!
|
||||
//! 1. an injected `Anonymous` (occupancy) cycle surfaces a **well-formed signed
|
||||
//! `FieldEvent`** — `Modality::WifiCsi`, privacy class consistent with the
|
||||
//! trust (P2, never P1), `is_fusable` (ed25519 receipt verifies), real
|
||||
//! timestamp;
|
||||
//! 2. an empty / no-presence cycle produces **no phantom event** (explicit
|
||||
//! empty payload);
|
||||
//! 3. the **privacy-safety pin** — an injected `Derived` (identity) trust state
|
||||
//! never surfaces as a low-privacy event on `/api/field` (held edge-local).
|
||||
//!
|
||||
//! These gates are plumbing + privacy-safety, NOT accuracy (ADR-262 §0 / §6).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use tokio::sync::RwLock;
|
||||
use tower::ServiceExt; // `oneshot`
|
||||
|
||||
use wifi_densepose_rufield::{is_fusable, verify_event, FieldEvent, Modality, PrivacyClass};
|
||||
use wifi_densepose_sensing_server::rufield_surface::{
|
||||
self, FieldState, FieldSurface, RuViewPrivacyClass, SensingClass, SensingFeatures, SignalField,
|
||||
};
|
||||
|
||||
/// A fixed dev seed for deterministic, signed events under test.
|
||||
const TEST_SEED: &[u8; 32] = b"adr262-p3-integration-test-seed!";
|
||||
|
||||
fn features() -> SensingFeatures {
|
||||
SensingFeatures {
|
||||
mean_rssi: -55.0,
|
||||
variance: 0.4,
|
||||
motion_band_power: 2.0,
|
||||
breathing_band_power: 0.3,
|
||||
dominant_freq_hz: 0.25,
|
||||
change_points: 1,
|
||||
spectral_power: 3.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn class(presence: bool) -> SensingClass {
|
||||
SensingClass {
|
||||
motion_level: if presence { "low".into() } else { "none".into() },
|
||||
presence,
|
||||
confidence: if presence { 0.82 } else { 0.05 },
|
||||
}
|
||||
}
|
||||
|
||||
/// A small 2×1×2 signal field with a clear peak, so the bridge derives a real
|
||||
/// (non-fabricated) position from the strongest cell.
|
||||
fn signal_field() -> SignalField {
|
||||
SignalField {
|
||||
grid_size: [2, 1, 2],
|
||||
values: vec![0.1, 0.2, 0.9, 0.3], // peak at index 2
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a `FieldState` + the real `/api/field` + `/ws/field` router over it.
|
||||
fn surface_router() -> (FieldState, axum::Router) {
|
||||
let state: FieldState = Arc::new(RwLock::new(FieldSurface::from_seed(TEST_SEED, true)));
|
||||
let app = rufield_surface::router(state.clone());
|
||||
(state, app)
|
||||
}
|
||||
|
||||
/// Drive one cycle into the surface (the in-process equivalent of the live
|
||||
/// sensing loop calling `emit()` per cycle).
|
||||
async fn inject(state: &FieldState, trust: RuViewPrivacyClass, presence: bool, identity_bound: bool) {
|
||||
let snap = rufield_surface::build_snapshot(
|
||||
1_791_986_400_000_000_000,
|
||||
"esp32_node_7".into(),
|
||||
features(),
|
||||
class(presence),
|
||||
Some(signal_field()),
|
||||
trust,
|
||||
false, // demoted
|
||||
identity_bound,
|
||||
);
|
||||
state.write().await.emit(&snap);
|
||||
}
|
||||
|
||||
/// `GET /api/field` and parse the `events` array.
|
||||
async fn get_field_events(app: &axum::Router) -> Vec<FieldEvent> {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/field")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "/api/field must return 200");
|
||||
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap();
|
||||
let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(v["spec"], "rufield");
|
||||
serde_json::from_value(v["events"].clone()).expect("events array deserializes to FieldEvents")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_anonymous_cycle_surfaces_wellformed_signed_event() {
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert_eq!(events.len(), 1, "one occupancy cycle ⇒ exactly one surfaced event");
|
||||
let ev = &events[0];
|
||||
|
||||
// Well-formed: WiFi-CSI modality, real timestamp.
|
||||
assert_eq!(ev.tensor.modality, Modality::WifiCsi);
|
||||
assert_eq!(ev.timestamp_ns, 1_791_986_400_000_000_000);
|
||||
assert!(ev.timestamp_ns > 0, "real (non-zero) timestamp");
|
||||
|
||||
// Privacy consistent with the injected trust: Anonymous → P2, NEVER P1.
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert_ne!(ev.observation.privacy_class, PrivacyClass::P1);
|
||||
|
||||
// Signed + fusable: the ed25519 receipt verifies (real, non-synthetic).
|
||||
assert!(!ev.provenance.synthetic, "live event is non-synthetic");
|
||||
assert!(verify_event(ev).is_ok(), "ed25519 signature must verify");
|
||||
assert!(is_fusable(ev), "verified receipt ⇒ fusable");
|
||||
|
||||
// Real position derived from the signal-field peak (not fabricated).
|
||||
assert!(ev.observation.range_m.is_some(), "field peak ⇒ a real range readout");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_empty_cycle_produces_no_phantom_event() {
|
||||
let (state, app) = surface_router();
|
||||
// A no-presence cycle: nothing to describe.
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, false, false).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert!(
|
||||
events.is_empty(),
|
||||
"no-presence cycle must surface no phantom event (explicit empty payload)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_derived_trust_never_surfaces_low_privacy() {
|
||||
// The privacy-safety pin (ADR-262 §3.3 / §6): a Derived (identity) trust
|
||||
// state maps to P4/P5 and is held edge-local — it must NEVER appear on the
|
||||
// network surface, and certainly never as a low-privacy (P1/P2) event.
|
||||
for identity_bound in [false, true] {
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, identity_bound).await;
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert!(
|
||||
events.is_empty(),
|
||||
"Derived cycle (identity_bound={identity_bound}) must not surface on /api/field"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gate_mixed_stream_surfaces_only_egress_safe_events() {
|
||||
// Determinism / privacy-safety over a stream: Anonymous cycles surface,
|
||||
// interleaved Derived cycles are dropped — the surface only ever carries
|
||||
// egress-safe (P1/P2) events.
|
||||
let (state, app) = surface_router();
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await; // P2 → surfaced
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, false).await; // P4 → dropped
|
||||
inject(&state, RuViewPrivacyClass::Anonymous, true, false).await; // P2 → surfaced
|
||||
inject(&state, RuViewPrivacyClass::Derived, true, true).await; // P5 → dropped
|
||||
|
||||
let events = get_field_events(&app).await;
|
||||
assert_eq!(events.len(), 2, "only the two Anonymous cycles surface");
|
||||
for ev in &events {
|
||||
assert_eq!(ev.observation.privacy_class, PrivacyClass::P2);
|
||||
assert!(is_fusable(ev));
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
Submodule vendor/rufield updated: c6abe92746...ba66e2e0a6
Reference in New Issue
Block a user