diff --git a/harness/wifi-densepose-sar/.claude-plugin/plugin.json b/harness/wifi-densepose-sar/.claude-plugin/plugin.json new file mode 100644 index 00000000..8f7587b0 --- /dev/null +++ b/harness/wifi-densepose-sar/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "name": "wifi-densepose-sar-harness", + "version": "0.1.0", + "description": "Harness for wifi-densepose-sar", + "author": { + "displayName": "Generated by metaharness", + "url": "https://www.npmjs.com/package/metaharness" + }, + "license": "MIT", + "categories": [ + "agent-harness", + "metaharness-scaffold", + "Engineering", + "software-engineering" + ], + "tags": [ + "metaharness", + "agent-harness", + "vertical:coding", + "software-engineering" + ], + "homepage": "https://github.com/ruvnet/agent-harness-generator" +} diff --git a/harness/wifi-densepose-sar/.claude/commands/doctor.md b/harness/wifi-densepose-sar/.claude/commands/doctor.md new file mode 100644 index 00000000..7fa0c94f --- /dev/null +++ b/harness/wifi-densepose-sar/.claude/commands/doctor.md @@ -0,0 +1,12 @@ +--- +description: "Health-check the harness: kernel load, MCP wiring, memory backend, host adapter." +--- + +Run a full health check and print a PASS/FAIL table. + +1. Kernel loads and `kernelInfo().version` matches package.json. +2. The MCP server starts and lists its tools. +3. The memory backend is reachable. +4. The configured host adapter is present. + +Exit non-zero if any check fails. diff --git a/harness/wifi-densepose-sar/.claude/commands/review-diff.md b/harness/wifi-densepose-sar/.claude/commands/review-diff.md new file mode 100644 index 00000000..5ed67e90 --- /dev/null +++ b/harness/wifi-densepose-sar/.claude/commands/review-diff.md @@ -0,0 +1,10 @@ +--- +description: "Review the current working diff for correctness, security, and reuse." +--- + +Review the current git diff. + +1. `git diff` to read the change. +2. Report only high-confidence findings as `file:line — issue — fix`. +3. Separate bugs from nits. +4. End with APPROVE or REQUEST-CHANGES and a one-line reason. diff --git a/harness/wifi-densepose-sar/.claude/settings.json b/harness/wifi-densepose-sar/.claude/settings.json new file mode 100644 index 00000000..b853f25b --- /dev/null +++ b/harness/wifi-densepose-sar/.claude/settings.json @@ -0,0 +1,40 @@ +{ + "permissions": { + "allow": [ + "Bash(npx wifi-densepose-sar-harness*)", + "mcp__wifi-densepose-sar-harness__*", + "mcp__code_index__*", + "Bash(npm test*)", + "Bash(npm run*)", + "Bash(git diff*)", + "Bash(git status*)", + "Bash(git log*)" + ], + "deny": [ + "Read(./.env)", + "Read(./.env.*)", + "Bash(git push*)", + "Bash(rm -rf*)" + ] + }, + "mcpServers": { + "wifi-densepose-sar-harness": { + "command": "npx", + "args": [ + "-y", + "wifi-densepose-sar-harness@latest", + "mcp", + "start" + ] + }, + "code_index": { + "command": "npx", + "args": [ + "-y", + "wifi-densepose-sar-harness@latest", + "mcp", + "index" + ] + } + } +} diff --git a/harness/wifi-densepose-sar/.claude/skills/evolve/SKILL.md b/harness/wifi-densepose-sar/.claude/skills/evolve/SKILL.md new file mode 100644 index 00000000..d09db2c1 --- /dev/null +++ b/harness/wifi-densepose-sar/.claude/skills/evolve/SKILL.md @@ -0,0 +1,53 @@ +--- +name: evolve +description: "Evolve this harness with Darwin Mode — frozen model, evolving harness (real, sandboxed, safety-gated)." +--- + +# evolve — Darwin Mode self-improvement + +`wifi-densepose-sar-harness` ships with **Darwin Mode** (`@metaharness/darwin`, ADR-070…146): the model +is frozen; the *harness* evolves. Each generation mutates ONE of the 7 surface files +(planner, contextBuilder, reviewer, retry/tool/memory/score policy), sandboxes each +child, scores it, and keeps only variants that *measurably* improve — building an +archive of successful descendants. + +## Run it + +```bash +npm run evolve # real substrate: runs your test command per variant (deterministic mutator — no API key, no network) +npm run evolve:dry # mock substrate: fast, fully offline, no test execution +``` + +Or directly: + +```bash +npx metaharness-darwin evolve . --sandbox real --generations 3 --children 4 +``` + +## Safety (secure by default) + +- **Deterministic mutator** is the default — **no network, no API key, air-gapped**. +- Every mutation passes the `validateGeneratedCode` gate: no new imports, network, + filesystem, shell, env access, or dependencies — pure refactor/tuning only. +- Mutations run in a **sandbox**; only variants that pass your tests are archived. +- Nothing is promoted without measured improvement (guard against Goodharting). + +See `@metaharness/darwin` for selection strategies (`--selection`, `--crossover`, +`--curriculum`), statistical gates (`--fdr`, `--bench`), and the real-LLM mutator (library API). + +## What the benchmarks taught us (measured, full SWE-bench Lite 300) + +Defaults worth carrying into how you evolve and run this harness (full evidence + CIs in +`@metaharness/darwin`'s `LEARNINGS.md` / `bench/results/RESULTS.md`): + +1. **Closed-loop repair is the #1 lever (~2×).** Feeding test/compiler failure back and retrying took + resolve-rate 7.7% → 15.3% on the *same cheap model*. Iterate against ground truth, don't single-shot. +2. **Cheap-first + cost-aware routing.** Track **$/resolve**, not just resolve-rate; a cheap model + resolved 31× cheaper per fix than a frontier one. Reserve frontier for *measured* capability gaps. +3. **Tier the models (Barbarian & Scholar).** Cheap sweep + frontier on *only the residual* = 33.3% + at ~6× lower cost than running frontier everywhere. +4. **Put the output-format contract in a system message + example**, and size prompts to the model's + real context window — this alone took a weak local model from 0% to ~50% valid output. +5. **Only trust batch evaluation of the final artifact** — in-loop counters drift 1.5–5×. +6. **The harness multiplies the model; it can't rescue one below the task's reasoning floor.** Pick + the smallest model *above* the floor, then let evolution do the rest. diff --git a/harness/wifi-densepose-sar/.claude/skills/plan-change/SKILL.md b/harness/wifi-densepose-sar/.claude/skills/plan-change/SKILL.md new file mode 100644 index 00000000..22fb03f5 --- /dev/null +++ b/harness/wifi-densepose-sar/.claude/skills/plan-change/SKILL.md @@ -0,0 +1,15 @@ +--- +name: plan-change +description: "Turn a feature request into a minimal, file-level implementation plan before any code." +--- + +# plan-change + +Produce an implementation plan for a requested change. + +1. Restate the goal in one sentence. +2. List the files to touch and why. +3. Name the smallest interface that satisfies it. +4. Flag anything that ripples beyond three files or widens a permission. + +Hand the plan to the implementer; do not write code in this step. diff --git a/harness/wifi-densepose-sar/.harness/manifest.json b/harness/wifi-densepose-sar/.harness/manifest.json new file mode 100644 index 00000000..8cd694aa --- /dev/null +++ b/harness/wifi-densepose-sar/.harness/manifest.json @@ -0,0 +1,39 @@ +{ + "schema": 1, + "generator": "0.1.0", + "template": "vertical:coding", + "template_version": "0.0.0", + "vars": { + "name": "wifi-densepose-sar-harness", + "description": "Harness for wifi-densepose-sar", + "host": "claude-code" + }, + "hosts": [ + "claude-code" + ], + "files": { + ".claude/commands/doctor.md": "2f1475fa0ed34729cb2ed9d6cecf29e99781da0a56afef133361e3ff0a17bb52", + ".claude/commands/review-diff.md": "2ab52f01487bfe67335f4de5193d7a6fb0f72f646411114613d8fa5da071ef2b", + ".claude/settings.json": "ea983de8f425d313ee42848a7da58ce98e05713a06dc2f45a5119f421a311e07", + ".claude/skills/plan-change/SKILL.md": "84e1c44ca264b999ebf80cd8fa0e5ebf554275bbd8023b1530ad053a44482e30", + ".claude-plugin/plugin.json": "f716a379f3e077b47dae884b5ca0d4a077e9a1ab1f091afa8f0f031ead1008e5", + "bin/cli.js": "78bd27074b3ce22bff63729995032c7b262a414ebfc8d75345fc82b40a4a84ee", + "CLAUDE.md": "4e9e11558e124605fd1a5de4be116c2b8d9dc15a3bc7c8f6789a7eec23dac19d", + "package.json": "a9f103fc452b5497a41333d2e80c834de07972e951be1701f25f98a5acaaea9d", + "README.md": "0efc949a11c20a8179261887d44afac60fb6e28af1ee0cf13451bd665b1cfbd4", + "src/agents/architect.ts": "b146bb8ad7729d7738f6c07d7770fecb6a13cf7683019c8b8afaec7c13806515", + "src/agents/implementer.ts": "55898d303ae87574348821ef2c270eceeec733278bb94041d767308b3a509291", + "src/agents/reviewer.ts": "ef5ab428ea799be7caeb05e761723122c52ab35d9ef4a18b618647ff72549ad8", + "src/agents/test-writer.ts": "f966a4d3a97a02b98606aac104f6fcbcf294c0186c15d7a809aa7555b20c0b5f", + "src/init.ts": "e19b1dbe6e4c3b7282a102bf068c20630e8cb0323d8abefa477d0f2dbbc68ce2", + "tsconfig.json": "8b4e730a1aa39162ac574455d7a98e1881f5313ca80ffe503b9652dcf0c76b9d", + "vitest.config.ts": "e9e94875611ab1cd602c1f6923902c2dab7953a75ac022962973639d1937587a", + "__tests__/smoke.test.ts": "e38dfc1389b8b419841f2c6513854dbb291e957be0c1c325c507dc58ef3d8c5e", + "LICENSE": "0580604803a2c38a1e4fe2c86d5e2f6d213cc3317cd5bc050691e484c523aedc", + ".claude/skills/evolve/SKILL.md": "05965b34edd9bbea83391ae20c5c22bf2653396e7b54557cfcf395fd7d11294a" + }, + "generated_at": "2026-07-30T22:39:05.579Z", + "meta": { + "surface": "cli" + } +} \ No newline at end of file diff --git a/harness/wifi-densepose-sar/.harness/manifest.sha256 b/harness/wifi-densepose-sar/.harness/manifest.sha256 new file mode 100644 index 00000000..20b397e6 --- /dev/null +++ b/harness/wifi-densepose-sar/.harness/manifest.sha256 @@ -0,0 +1 @@ +b5ac57b4ae092fda710fef597519f3f623ebd9bb40bee9c7e010d0ad42200241 diff --git a/harness/wifi-densepose-sar/CLAUDE.md b/harness/wifi-densepose-sar/CLAUDE.md new file mode 100644 index 00000000..c3e4dd98 --- /dev/null +++ b/harness/wifi-densepose-sar/CLAUDE.md @@ -0,0 +1,43 @@ +# wifi-densepose-sar-harness + +Harness for wifi-densepose-sar + +> Advanced Coding harness · domain: `software-engineering`. Generated with [create-agent-harness](https://github.com/ruvnet/agent-harness-generator). + +## Behavioral rules + +- Use the harness's MCP tools (`mcp__wifi-densepose-sar-harness__*`) for orchestration +- Memory and routing are handled by the kernel — you don't need to learn them +- Defer destructive operations to the user + +## Agents + +| Agent | Tier | Role | +|---|---|---| +| `architect` | opus | Designs the change before code is written. | +| `implementer` | sonnet | Writes code that matches the surrounding style. | +| `reviewer` | opus | Hunts correctness bugs in the diff. | +| `test-writer` | sonnet | Adds the missing tests for the change. | +## Skills + +- `/plan-change` — Turn a feature request into a minimal, file-level implementation plan before any code. +- `/evolve` — Run Darwin Mode (`npm run evolve` / `evolve:dry`) to self-mutate the harness's own operating policy and keep only measurable improvements. + +## Commands + +- `doctor` — Health-check the harness: kernel load, MCP wiring, memory backend, host adapter. +- `review-diff` — Review the current working diff for correctness, security, and reuse. +- `route ` — cost-optimal model routing via `@metaharness/router` (needs `npm run build` first). +- `flywheel [generations]` — run the SYNTHETIC self-improvement demo via `@metaharness/flywheel` (needs `npm run build` first). + +## Architecture + +This harness uses [@metaharness/kernel](https://www.npmjs.com/package/@metaharness/kernel) — a Rust-compiled WASM module with a NAPI-RS native fallback — so the same code runs identically on every platform. + +### Darwin, router, flywheel + +Three complementary self-improvement/cost pieces, all real npm dependencies (not aspirational): + +- **Darwin Mode** (`@metaharness/darwin`, devDependency) — `npm run evolve` (real sandbox) / `npm run evolve:dry` (mock sandbox) mutates the harness's own config and keeps only changes that measurably improve it. Wired by the scaffold itself. +- **Router** (`@metaharness/router`) — `src/router.ts` wires a real `Router` with a `qualityBar: 0.8` cost-optimal policy over two example model tiers. Its labelled examples are illustrative/seed data (see the file's honesty note), not measured eval-log observations — the routing *mechanism* is real and tested (`__tests__/router.test.ts`), the specific decisions it makes today are not yet backed by real data. +- **Flywheel** (`@metaharness/flywheel`) — `src/flywheel.ts` wires the real `runFlywheelGenerations` promotion loop (propose → evaluate → gate → promote, Ed25519-signed, independently replayable) with a SYNTHETIC proposer/evaluator (`dataSource: 'SYNTHETIC'`, no model call). It proves the wiring end-to-end (`__tests__/flywheel.test.ts` checks a real lift curve and a passing `verifyReplayBundle`); a LIVE run needs a real Proposer (model call) and Evaluator (real coding-task holdout/anchor suites) supplied by the operator — see the file's comments for exactly what those seams are. diff --git a/harness/wifi-densepose-sar/LICENSE b/harness/wifi-densepose-sar/LICENSE new file mode 100644 index 00000000..b1f6d54c --- /dev/null +++ b/harness/wifi-densepose-sar/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 wifi-densepose-sar-harness authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/harness/wifi-densepose-sar/README.md b/harness/wifi-densepose-sar/README.md new file mode 100644 index 00000000..72e3f61e --- /dev/null +++ b/harness/wifi-densepose-sar/README.md @@ -0,0 +1,42 @@ +# wifi-densepose-sar-harness + +Harness for wifi-densepose-sar + +> **Advanced Coding** — Architect → implement → review → test, with a code-index MCP and push-guarded git perms. +> +> Generated with [`create-agent-harness`](https://github.com/ruvnet/agent-harness-generator). Multi-host scaffolding with a kernel that resolves native → wasm → js (js backend in the published beta; see `harness doctor`). + +## Install + +```bash +npm install -g wifi-densepose-sar-harness +wifi-densepose-sar-harness init +wifi-densepose-sar-harness doctor +``` + +## Agents + +| Agent | Role | +|---|---| +| `architect` | Designs the change before code is written. | +| `implementer` | Writes code that matches the surrounding style. | +| `reviewer` | Hunts correctness bugs in the diff. | +| `test-writer` | Adds the missing tests for the change. | + +This harness ships with the **claude-code** adapter. + +## Darwin, router, flywheel + +- `npm run evolve` / `evolve:dry` — Darwin Mode self-mutation of the harness's own config (`@metaharness/darwin`). +- `npm run route -- ` (after `npm run build`) — cost-optimal model routing via `@metaharness/router`. +- `npm run flywheel:dry` — the SYNTHETIC `@metaharness/flywheel` self-improvement demo (propose → evaluate → gate → promote, signed + independently replayable). + +See `CLAUDE.md`'s "Darwin, router, flywheel" section and the comments at the top of `src/router.ts` / `src/flywheel.ts` for what's real wiring vs. illustrative/synthetic data. + +## Known gaps + +- `.harness/manifest.json` / `manifest.sha256` reflect the initial `metaharness analyze --scaffold` output and were not regenerated after adding `src/router.ts`, `src/flywheel.ts`, and their tests — this scaffold has no `manifest:update` script (unlike `harness/homecore/`). Treat the manifest as historical provenance for the scaffold step, not a current file-integrity check. + +## License + +MIT diff --git a/harness/wifi-densepose-sar/__tests__/flywheel.test.ts b/harness/wifi-densepose-sar/__tests__/flywheel.test.ts new file mode 100644 index 00000000..1f6d181f --- /dev/null +++ b/harness/wifi-densepose-sar/__tests__/flywheel.test.ts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT +import { describe, it, expect } from 'vitest'; +import { runSarFlywheelDemo, verifySarFlywheelDemo, SAR_ROOT_POLICY } from '../src/flywheel.js'; + +describe('runSarFlywheelDemo — @metaharness/flywheel wiring (SYNTHETIC data)', () => { + it('runs the configured number of generations and stamps SYNTHETIC provenance', async () => { + const result = await runSarFlywheelDemo(3); + expect(result.generationsRun).toBeGreaterThan(0); + expect(result.generationsRun).toBeLessThanOrEqual(3); + expect(result.replayBundle.data_source).toBe('SYNTHETIC'); + }); + + it('produces a non-empty lift curve rooted at the initial policy', async () => { + const result = await runSarFlywheelDemo(3); + expect(result.liftCurve.length).toBeGreaterThan(0); + expect(result.liftCurve[0].generation).toBe(0); + }); + + it('the final policy still carries every root lever', async () => { + const result = await runSarFlywheelDemo(2); + for (const key of Object.keys(SAR_ROOT_POLICY)) { + expect(result.finalPolicy).toHaveProperty(key); + } + }); + + it('the replay bundle independently verifies (no trust in the producer)', async () => { + const result = await runSarFlywheelDemo(3); + const verdict = verifySarFlywheelDemo(result); + expect(verdict.pass).toBe(true); + }); + + it('is deterministic in structure across repeated runs (same generations requested)', async () => { + const a = await runSarFlywheelDemo(2); + const b = await runSarFlywheelDemo(2); + expect(a.generationsRun).toBe(b.generationsRun); + }); +}); diff --git a/harness/wifi-densepose-sar/__tests__/router.test.ts b/harness/wifi-densepose-sar/__tests__/router.test.ts new file mode 100644 index 00000000..d11febe1 --- /dev/null +++ b/harness/wifi-densepose-sar/__tests__/router.test.ts @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT +import { describe, it, expect } from 'vitest'; +import { sarTaskRouter, routeSarQuery, SAR_ROUTER_CANDIDATES } from '../src/router.js'; + +describe('sarTaskRouter — @metaharness/router wiring', () => { + it('has both a cheap and a frontier candidate', () => { + const ids = SAR_ROUTER_CANDIDATES.map((c) => c.id); + expect(ids).toContain('cheap-tier'); + expect(ids).toContain('frontier-tier'); + }); + + it('routes a physics-explanation-shaped query to the cheap tier', () => { + const pick = routeSarQuery([1, 0, 0, 0]); + expect(pick.id).toBe('cheap-tier'); + expect(pick.metBar).toBe(true); + }); + + it('routes a code-review-shaped query to the frontier tier (cheap tier misses the quality bar)', () => { + const pick = routeSarQuery([0, 1, 0, 0]); + expect(pick.id).toBe('frontier-tier'); + }); + + it('always returns a candidate with a nonnegative predicted quality and a positive cost', () => { + for (const q of [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]] as const) { + const pick = routeSarQuery(q); + expect(pick.predictedQuality).toBeGreaterThanOrEqual(0); + expect(pick.costPerMTok).toBeGreaterThan(0); + } + }); + + it('sarTaskRouter.route and routeSarQuery agree (same underlying router)', () => { + const a = sarTaskRouter.route([1, 0, 0, 0]); + const b = routeSarQuery([1, 0, 0, 0]); + expect(a).toEqual(b); + }); +}); diff --git a/harness/wifi-densepose-sar/__tests__/smoke.test.ts b/harness/wifi-densepose-sar/__tests__/smoke.test.ts new file mode 100644 index 00000000..90000767 --- /dev/null +++ b/harness/wifi-densepose-sar/__tests__/smoke.test.ts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT +// Generated by metaharness — a real smoke test for wifi-densepose-sar-harness. +// +// This is NOT a placeholder: it boots the actual kernel + host adapter the +// harness depends on, so `npm test` fails loudly if @metaharness/kernel or +// @metaharness/host-claude-code is missing, broken, or version-skewed. It is the +// fastest signal that `npm install` produced a runnable harness. + +import { describe, it, expect } from 'vitest'; +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; +import { run } from '../bin/cli.js'; + +describe('wifi-densepose-sar-harness — install smoke test', () => { + it('loads the kernel and reports a version + a known backend', async () => { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + expect(typeof info.version).toBe('string'); + expect(info.version.length).toBeGreaterThan(0); + expect(['native', 'wasm', 'js']).toContain(kernel.backend); + }); + + it('resolves the host adapter with a name', () => { + expect(typeof adapter.name).toBe('string'); + expect(adapter.name.length).toBeGreaterThan(0); + }); + + it('the CLI doctor command succeeds (exit 0)', async () => { + const code = await run(['doctor']); + expect(code).toBe(0); + }); + + it('an unknown CLI command exits non-zero', async () => { + const code = await run(['definitely-not-a-command']); + expect(code).not.toBe(0); + }); +}); diff --git a/harness/wifi-densepose-sar/bin/cli.js b/harness/wifi-densepose-sar/bin/cli.js new file mode 100644 index 00000000..4ade8360 --- /dev/null +++ b/harness/wifi-densepose-sar/bin/cli.js @@ -0,0 +1,156 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +// Generated by metaharness — the `wifi-densepose-sar-harness` CLI entry point. +// +// This is plain ESM JavaScript on purpose: it runs as-is via `npx wifi-densepose-sar-harness` +// with NO build step. `npm run build` (tsc) is only needed if you extend the +// TypeScript in src/. The published package ships this file directly (see the +// "bin" + "files" fields in package.json), so `npx wifi-densepose-sar-harness` works the moment +// `npm install` has resolved @metaharness/kernel + @metaharness/host-claude-code. + +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; + +const HARNESS_NAME = 'wifi-densepose-sar-harness'; + +/** `wifi-densepose-sar-harness init` — boot the kernel + host adapter and report status. */ +async function init() { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + console.log(`${HARNESS_NAME} — kernel ${info.version} (${kernel.backend})`); + console.log(`Host adapter: ${adapter.name}`); + console.log(`Run \`${HARNESS_NAME} doctor\` to verify the install.`); + return 0; +} + +/** + * `wifi-densepose-sar-harness route ` — route a 4-axis + * task embedding to the cost-optimal model tier via @metaharness/router. + * Needs `npm run build` first (src/router.ts is TypeScript; this command + * imports the compiled dist/ output so the base CLI stays build-free). + */ +async function route(args) { + const embedding = args.map(Number); + if (embedding.length !== 4 || embedding.some((n) => Number.isNaN(n))) { + console.error('Usage: wifi-densepose-sar-harness route (four 0..1 numbers)'); + return 2; + } + let routeSarQuery; + try { + ({ routeSarQuery } = await import('../dist/router.js')); + } catch (err) { + console.error(`route: dist/router.js not found — run \`npm run build\` first. (${err.message})`); + return 1; + } + const pick = routeSarQuery(embedding); + console.log(`route -> ${pick.id} (predicted quality ${pick.predictedQuality.toFixed(3)}, $${pick.costPerMTok}/MTok, met bar: ${pick.metBar})`); + return 0; +} + +/** + * `wifi-densepose-sar-harness flywheel [generations]` — run the SYNTHETIC + * @metaharness/flywheel demo (see src/flywheel.ts) and print the lift curve + * + an independent replay-bundle verification. Needs `npm run build` first. + */ +async function flywheel(args) { + const generations = args[0] ? Number(args[0]) : 3; + if (Number.isNaN(generations) || generations < 1) { + console.error('Usage: wifi-densepose-sar-harness flywheel [generations>=1]'); + return 2; + } + let runSarFlywheelDemo, verifySarFlywheelDemo; + try { + ({ runSarFlywheelDemo, verifySarFlywheelDemo } = await import('../dist/flywheel.js')); + } catch (err) { + console.error(`flywheel: dist/flywheel.js not found — run \`npm run build\` first. (${err.message})`); + return 1; + } + console.log(`Running ${generations}-generation flywheel demo (dataSource: SYNTHETIC — see src/flywheel.ts)...`); + const result = await runSarFlywheelDemo(generations); + for (const point of result.liftCurve) { + console.log(` gen ${point.generation}: primary=${point.primary.toFixed(3)} delta=${point.delta.toFixed(3)} anchor=${point.anchor ?? 'n/a'}`); + } + const verdict = verifySarFlywheelDemo(result); + console.log(`generations run: ${result.generationsRun} · promotions: ${result.promotions.length} · replay verified: ${verdict.pass}`); + return verdict.pass ? 0 : 1; +} + +/** `wifi-densepose-sar-harness doctor` — verify the install end-to-end (kernel + host resolve). */ +async function doctor() { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + const checks = [ + ['kernel loads', !!kernel], + ['kernel reports a version', typeof info.version === 'string' && info.version.length > 0], + ['kernel backend is native|wasm|js', ['native', 'wasm', 'js'].includes(kernel.backend)], + ['host adapter has a name', typeof adapter?.name === 'string' && adapter.name.length > 0], + ]; + let ok = true; + for (const [label, pass] of checks) { + console.log(`${pass ? 'PASS' : 'FAIL'} ${label}`); + if (!pass) ok = false; + } + console.log( + ok + ? `\n${HARNESS_NAME}: all checks passed (kernel ${info.version}, ${kernel.backend} backend, host ${adapter.name})` + : `\n${HARNESS_NAME}: doctor found problems`, + ); + return ok ? 0 : 1; +} + +/** + * Dispatch one CLI invocation. Exported (not just run on import) so a test can + * drive it without spawning a subprocess. Returns the intended exit code. + */ +export async function run(argv) { + const cmd = argv[0] ?? 'init'; + switch (cmd) { + case 'init': + return init(); + case 'doctor': + return doctor(); + case 'route': + return route(argv.slice(1)); + case 'flywheel': + return flywheel(argv.slice(1)); + case '--version': + case '-v': { + const kernel = await loadKernel(); + console.log(kernel.version()); + return 0; + } + case '--help': + case '-h': + console.log(`Usage: ${HARNESS_NAME} \n\n init boot the kernel + host adapter (default)\n doctor verify the install end-to-end\n route route a 4-axis task embedding to a cost-optimal model tier (needs \`npm run build\`)\n flywheel run the SYNTHETIC self-improvement demo loop (needs \`npm run build\`)\n --version print the kernel version`); + return 0; + default: + console.error(`Unknown command: ${cmd}. Try \`${HARNESS_NAME} --help\`.`); + return 2; + } +} + +// CLI guard: execute only when invoked directly (not when imported by a test). +// npm's bin shims pass a NON-normalized argv[1] (e.g. ".../.bin/..//bin/cli.js" +// on Windows) and may differ in case, so realpath BOTH sides before comparing — +// a naive string === misses the npx/shim path and the CLI silently no-ops. +import { fileURLToPath } from 'node:url'; +import { realpathSync } from 'node:fs'; +import { argv } from 'node:process'; +const invokedDirectly = (() => { + if (!argv[1]) return false; + try { + const a = realpathSync(argv[1]); + const b = realpathSync(fileURLToPath(import.meta.url)); + return process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b; + } catch { + return false; + } +})(); +if (invokedDirectly) { + run(argv.slice(2)) + .then((code) => process.exit(code)) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/harness/wifi-densepose-sar/package.json b/harness/wifi-densepose-sar/package.json new file mode 100644 index 00000000..02e8062f --- /dev/null +++ b/harness/wifi-densepose-sar/package.json @@ -0,0 +1,48 @@ +{ + "name": "wifi-densepose-sar-harness", + "version": "0.1.0", + "description": "Harness for wifi-densepose-sar", + "license": "MIT", + "type": "module", + "bin": { + "wifi-densepose-sar-harness": "bin/cli.js" + }, + "files": [ + "bin/**", + "dist/**", + "src/**", + "tsconfig.json", + ".claude/**", + "CLAUDE.md", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsc", + "test": "vitest run", + "init": "node ./bin/cli.js init", + "doctor": "node ./bin/cli.js doctor", + "evolve": "metaharness-darwin evolve . --sandbox real --generations 3 --children 4", + "evolve:dry": "metaharness-darwin evolve . --sandbox mock --generations 2 --children 3", + "route": "node ./bin/cli.js route", + "flywheel:dry": "npm run build && node ./bin/cli.js flywheel 3" + }, + "dependencies": { + "@metaharness/kernel": "^0.1.0", + "@metaharness/host-claude-code": "^0.1.0", + "@metaharness/router": "^0.3.2", + "@metaharness/flywheel": "^0.1.7" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.4.0", + "vitest": "^3.0.0", + "@metaharness/darwin": "^0.2.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/harness/wifi-densepose-sar/src/agents/architect.ts b/harness/wifi-densepose-sar/src/agents/architect.ts new file mode 100644 index 00000000..e33ec63f --- /dev/null +++ b/harness/wifi-densepose-sar/src/agents/architect.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Architect agent — Designs the change before code is written. + +export const SYSTEM_PROMPT = `You are the architect. Before any code is written you produce the smallest design that satisfies the request: the files to touch, the interfaces to add, and the trade-offs. You never write the implementation — you hand a crisp plan to the implementer. Prefer reuse over new abstractions; call out any change that ripples beyond three files. You operate inside the wifi-densepose-sar-harness harness; defer destructive actions to the user.`; + +export const NAME = 'architect'; +export const TIER = 'opus' as const; diff --git a/harness/wifi-densepose-sar/src/agents/implementer.ts b/harness/wifi-densepose-sar/src/agents/implementer.ts new file mode 100644 index 00000000..35f0f117 --- /dev/null +++ b/harness/wifi-densepose-sar/src/agents/implementer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Implementer agent — Writes code that matches the surrounding style. + +export const SYSTEM_PROMPT = `You implement the architect's plan. Match the existing code's naming, comment density, and idioms — your diff should read like the person who wrote the file kept writing. Make the minimal change; do not refactor unrelated code. Leave the tests to the test-writer unless asked. You operate inside the wifi-densepose-sar-harness harness; defer destructive actions to the user.`; + +export const NAME = 'implementer'; +export const TIER = 'sonnet' as const; diff --git a/harness/wifi-densepose-sar/src/agents/reviewer.ts b/harness/wifi-densepose-sar/src/agents/reviewer.ts new file mode 100644 index 00000000..d7581110 --- /dev/null +++ b/harness/wifi-densepose-sar/src/agents/reviewer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Reviewer agent — Hunts correctness bugs in the diff. + +export const SYSTEM_PROMPT = `You review diffs for correctness, security, and reuse. Report only high-confidence findings, each with a file:line and a concrete fix. Distinguish a bug (will break) from a nit (style). Never approve a change that widens a permission, swallows an error, or ships a secret. You operate inside the wifi-densepose-sar-harness harness; defer destructive actions to the user.`; + +export const NAME = 'reviewer'; +export const TIER = 'opus' as const; diff --git a/harness/wifi-densepose-sar/src/agents/test-writer.ts b/harness/wifi-densepose-sar/src/agents/test-writer.ts new file mode 100644 index 00000000..a6a76959 --- /dev/null +++ b/harness/wifi-densepose-sar/src/agents/test-writer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Test Writer agent — Adds the missing tests for the change. + +export const SYSTEM_PROMPT = `You write the tests the change needs: the happy path, the boundary, and the one failure mode most likely to regress. Mirror the project's existing test style and runner. A test that cannot fail is worse than no test — assert behaviour, not implementation. You operate inside the wifi-densepose-sar-harness harness; defer destructive actions to the user.`; + +export const NAME = 'test-writer'; +export const TIER = 'sonnet' as const; diff --git a/harness/wifi-densepose-sar/src/flywheel.ts b/harness/wifi-densepose-sar/src/flywheel.ts new file mode 100644 index 00000000..e0027c48 --- /dev/null +++ b/harness/wifi-densepose-sar/src/flywheel.ts @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT +// +// The wifi-densepose-sar harness's self-improvement loop, via +// @metaharness/flywheel: run -> measure -> mutate -> verify -> promote, +// with a frozen, conjunctive promotion gate and a signed, replayable lineage +// (see @metaharness/darwin's `npm run evolve` for the harness-wide mutation +// entry point this formalizes the promotion loop for). +// +// HONESTY NOTE (load-bearing): `runSarFlywheelDemo()` below wires the real +// @metaharness/flywheel API end-to-end, but its Proposer and Evaluator are +// SYNTHETIC stand-ins — a deterministic string mutation and a deterministic +// scoring function over that string, with NO model call and NO real +// benchmark suite. It exists to prove the wiring works (see +// __tests__/flywheel.test.ts: a non-empty lift curve, a verifiable replay +// bundle) and to give a `dataSource: 'SYNTHETIC'`-stamped demo, exactly as +// the flywheel's own design requires callers to be honest about data +// provenance. A LIVE run needs the operator to supply: +// - a real Proposer: an actual model call that improves one policy lever +// (e.g. the review-diff checklist, the architect's planning prompt); +// - a real Evaluator: scores that policy against real coding-task +// holdout/anchor suites (e.g. "did review-diff catch the seeded bug"). +// Neither exists in this repo — wiring them is a live-API-key decision for +// whoever operates this harness, not something to fake here. + +import { + runFlywheelGenerations, + meetsPromotionRule, + makeSigner, + verifyReplayBundle, + type Policy, + type PolicyGenome, + type Proposer, + type Evaluator, + type Suite, + type FlywheelResult, +} from '@metaharness/flywheel'; + +/** The gen-0 operating policy for the SAR harness's review agents. Opaque + * string levers — the flywheel never interprets their meaning, only the + * Evaluator does. */ +export const SAR_ROOT_POLICY: Policy = { + reviewDepth: 'standard-checklist', + architectPlanning: 'single-pass', +}; + +/** + * SYNTHETIC proposer: deterministically lengthens/varies the target lever's + * value rather than calling a model. Stands in for a real model call that + * would draft an improved lever value. + */ +const syntheticProposer: Proposer = async (base: PolicyGenome, target: string) => { + const current = base.policy[target] ?? ''; + return `${current}+g${base.generation + 1}`; +}; + +/** + * SYNTHETIC evaluator: scores a policy purely as a function of its own + * string content (longer, more "refined"-looking levers score marginally + * higher on quality and lower on no-op rate, both with a floor/ceiling) — + * a deterministic stand-in for actually running the harness's agents + * against a real coding-task suite. `noopRate` must move (not stay + * constant) for anything to ever promote: the default gate's clause 2 + * requires it to strictly improve generation over generation (ADR-226's + * "the executor policy is the part that mattered" finding, encoded as a + * hard requirement) — a constant noopRate, even a "good" one, gates every + * candidate out forever. + */ +const syntheticEvaluator: Evaluator = async (policy: Policy, _suite: Suite) => { + const totalLength = Object.values(policy).reduce((s, v) => s + v.length, 0); + const primary = Math.min(0.5 + totalLength / 200, 0.98); + const noopRate = Math.max(0.3 - totalLength / 300, 0.02); + return { + primary, + noopRate, + costPerWin: 1 / primary, + regressed: false, + }; +}; + +const SAR_HOLDOUT: Suite = { + id: 'sar-harness-holdout-synthetic', + items: ['seeded-review-task-1', 'seeded-review-task-2', 'seeded-review-task-3'], +}; + +const SAR_ANCHOR: Suite = { + id: 'sar-harness-anchor-synthetic', + items: ['frozen-regression-task-1'], +}; + +/** + * Run a small, fully SYNTHETIC flywheel demo end-to-end: propose, evaluate, + * gate, and (when it clears the gate) promote a few generations of mutated + * policy, returning the real @metaharness/flywheel result — a genuine lift + * curve and a signed, independently replayable bundle, just built from + * synthetic (not live) evidence. + */ +export async function runSarFlywheelDemo(maxGenerations = 3): Promise { + return runFlywheelGenerations({ + rootPolicy: SAR_ROOT_POLICY, + proposer: syntheticProposer, + evaluator: syntheticEvaluator, + promotionRule: meetsPromotionRule, + holdout: SAR_HOLDOUT, + anchor: SAR_ANCHOR, + maxGenerations, + signer: makeSigner(), + dataSource: 'SYNTHETIC', + }); +} + +/** Independently verify a flywheel demo's replay bundle (no trust in the producer). */ +export function verifySarFlywheelDemo(result: FlywheelResult) { + return verifyReplayBundle(result.replayBundle); +} diff --git a/harness/wifi-densepose-sar/src/init.ts b/harness/wifi-densepose-sar/src/init.ts new file mode 100644 index 00000000..4d4629eb --- /dev/null +++ b/harness/wifi-densepose-sar/src/init.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT +// Generated by create-agent-harness — your harness's `wifi-densepose-sar-harness init` entry. + +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; + +const HARNESS_NAME = 'wifi-densepose-sar-harness'; + +async function main(): Promise { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + console.log(`${HARNESS_NAME} — kernel ${info.version} (${kernel.backend})`); + console.log(`Host adapter: ${adapter.name}`); + console.log(`Run \`${HARNESS_NAME} doctor\` to verify the install.`); + return 0; +} + +main().then(c => process.exit(c)).catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/harness/wifi-densepose-sar/src/router.ts b/harness/wifi-densepose-sar/src/router.ts new file mode 100644 index 00000000..a35f4c5e --- /dev/null +++ b/harness/wifi-densepose-sar/src/router.ts @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT +// +// Cost-optimal task routing for the wifi-densepose-sar harness, via +// @metaharness/router (ADR-040's DRACO Phase-2 finding, productized): route +// each agent query to the cheapest model predicted to clear a quality bar, +// instead of sending every query to the frontier tier by default. +// +// HONESTY NOTE: the candidate `examples` below are SEED/ILLUSTRATIVE data — +// four hand-picked (embedding, quality) points per candidate, not measured +// eval-log observations. They exist so `sarTaskRouter` is a real, runnable +// k-NN router out of the box (see __tests__/router.test.ts), not so its +// routing decisions should be trusted for production cost savings. Replace +// `SAR_ROUTER_CANDIDATES[*].examples` with real (query embedding → quality +// achieved) rows from your own eval logs before relying on this for +// production routing — see @metaharness/router's README ("the more +// examples, the closer it gets to the per-query oracle"). + +import { Router, type RouterCandidate } from '@metaharness/router'; + +/** + * A 4-axis feature embedding for a harness query, used only to pick a + * nearby labelled example — NOT a real text embedding. Axes (each 0..1): + * [0] physicsExplanation — "explain the range-resolution formula"-shaped + * [1] codeReview — "review this diff for correctness"-shaped + * [2] numericalDebugging — "why did this reconstruction test fail"-shaped + * [3] docWriting — "write/update the tutorial"-shaped + * A caller with a real embedding model should project onto whatever + * dimensionality that model produces instead — the router only needs + * consistent vectors, not these specific four axes. + */ +export type SarTaskEmbedding = readonly [number, number, number, number]; + +export const SAR_ROUTER_CANDIDATES: RouterCandidate[] = [ + { + id: 'cheap-tier', + costPerMTok: 1, + examples: [ + { embedding: [1, 0, 0, 0], quality: 0.9 }, // physics explanations: cheap tier does fine + { embedding: [0, 0, 0, 1], quality: 0.85 }, // doc writing: cheap tier does fine + { embedding: [0, 1, 0, 0], quality: 0.55 }, // code review: cheap tier is weak + { embedding: [0, 0, 1, 0], quality: 0.5 }, // numerical debugging: cheap tier is weak + ], + }, + { + id: 'frontier-tier', + costPerMTok: 15, + examples: [ + { embedding: [1, 0, 0, 0], quality: 0.95 }, + { embedding: [0, 0, 0, 1], quality: 0.93 }, + { embedding: [0, 1, 0, 0], quality: 0.92 }, // code review: frontier tier needed + { embedding: [0, 0, 1, 0], quality: 0.9 }, // numerical debugging: frontier tier needed + ], + }, +]; + +/** + * Cost-optimal router for the harness's four query shapes above. `qualityBar` + * of 0.8 matches the router README's worked example: return the cheapest + * candidate predicted to clear 80% quality, or the best-predicted candidate + * if none do. + */ +export const sarTaskRouter = new Router({ + qualityBar: 0.8, + candidates: SAR_ROUTER_CANDIDATES, + // k=1: each candidate has only 4 (orthogonal, one-hot) examples covering + // the 4 task axes. The router's default k=5 would average ALL of a + // candidate's examples regardless of query similarity once a candidate + // has <=5 examples, collapsing every query to the same prediction. k=1 + // makes it pick the single nearest labelled task type, which is what + // this small illustrative dataset is shaped for. + k: 1, +}); + +/** Route one query embedding to the cost-optimal model tier. */ +export function routeSarQuery(queryEmbedding: SarTaskEmbedding) { + return sarTaskRouter.route([...queryEmbedding]); +} diff --git a/harness/wifi-densepose-sar/tsconfig.json b/harness/wifi-densepose-sar/tsconfig.json new file mode 100644 index 00000000..4f908fa4 --- /dev/null +++ b/harness/wifi-densepose-sar/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "__tests__"] +} diff --git a/harness/wifi-densepose-sar/vitest.config.ts b/harness/wifi-densepose-sar/vitest.config.ts new file mode 100644 index 00000000..ccfd4319 --- /dev/null +++ b/harness/wifi-densepose-sar/vitest.config.ts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MIT +// Generated by metaharness. Strips the `#!/usr/bin/env node` shebang from +// importable entrypoints (e.g. bin/cli.js) before Vite parses them — Vite/esbuild +// (used internally by Vitest) does NOT strip shebangs, so importing a shebanged +// module throws `SyntaxError: Invalid or unexpected token`. See issue #44. +// Has no effect on direct CLI execution or `npm run doctor` (those bypass Vite). +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + plugins: [ + { + name: 'strip-shebang', + enforce: 'pre', + transform(code: string) { + if (code.startsWith('#!')) { + // Replace with a blank line so source line numbers stay aligned. + return { code: code.replace(/^#![^\n]*/, ''), map: null }; + } + return null; + }, + }, + ], +});