mirror of
https://github.com/ruvnet/RuView
synced 2026-08-03 19:21:42 +00:00
fix(npm): address 10 verified review findings in harness + rvagent before 0.2.0 publish
harness/ruview (@ruvnet/ruview): - guardrails: digit gate now sees numbers inside code spans; F1-style metric tokens followed by ':' or a nearby number are no longer scrubbed (fail-open regressions in the honesty gate) - mcp-server: tools/call requests serialize through a FIFO promise chain (hardware/mutating tools never overlap) while ping/tools/list stay immediate; stdin close drains in-flight responses before exit - tools: which() no longer memoizes negative lookups tools/ruview-mcp (@ruvnet/rvagent): - index: realpath invoked-directly guard — library import no longer connects a stdio transport to the consumer's process - http-transport: explicit allowedOrigins is exact-match only (localhost any-port convenience applies only with no configured allowlist); session map gains maxSessions=64 + 5min idle TTL sweep - train-count: job records persist the child pid and reconcile stale 'running' status after a server restart (exit-code marker or dead pid) - config: cog binary candidates ordered by process.arch .github/workflows/ruview-npm-release.yml: port the full ADR-265 D1 gate (version-literal check, unpacked-size budget, tarball-install smoke test) from npm-packages.yml so the publish path enforces what the header claims. Tests: harness 30→36, rvagent 99→112, all passing. Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
@@ -99,3 +99,50 @@ test('MCP server answers ping while a long tools/call is in flight (ADR-263 O2)'
|
||||
rmSync(repo, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('tools/call executions are serialized — two slow calls run sequentially', { skip: !which('python') && !which('python3') ? 'python not on PATH' : false }, async () => {
|
||||
// Two verify.py that each sleep 0.8 s. Serialized ⇒ ~1.6 s+; concurrent ⇒ ~0.8 s.
|
||||
const repo = mkdtempSync(join(tmpdir(), 'ruview-mcp-serial-'));
|
||||
const proofDir = join(repo, 'archive', 'v1', 'data', 'proof');
|
||||
mkdirSync(proofDir, { recursive: true });
|
||||
writeFileSync(join(proofDir, 'verify.py'), 'import time\ntime.sleep(0.8)\nprint("VERDICT: PASS")\n');
|
||||
|
||||
const s = startServer();
|
||||
try {
|
||||
s.send({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} });
|
||||
await s.next(1);
|
||||
|
||||
const t0 = Date.now();
|
||||
const a = s.next(20);
|
||||
const b = s.next(21);
|
||||
s.send({ jsonrpc: '2.0', id: 20, method: 'tools/call', params: { name: 'ruview_verify', arguments: { repo } } });
|
||||
s.send({ jsonrpc: '2.0', id: 21, method: 'tools/call', params: { name: 'ruview_verify', arguments: { repo } } });
|
||||
const [ra, rb] = await Promise.all([a, b]);
|
||||
const elapsed = Date.now() - t0;
|
||||
|
||||
assert.equal(JSON.parse(ra.result.content[0].text).verdict, 'PASS');
|
||||
assert.equal(JSON.parse(rb.result.content[0].text).verdict, 'PASS');
|
||||
assert.ok(elapsed > 1400, `two 0.8 s tool calls finished in ${elapsed} ms — they overlapped instead of serializing`);
|
||||
} finally {
|
||||
s.close();
|
||||
rmSync(repo, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('stdin close flushes an in-flight tools/call response before exit', async () => {
|
||||
const child = spawn(process.execPath, [CLI, 'mcp', 'start'], { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
let out = '';
|
||||
child.stdout.on('data', (d) => { out += d; });
|
||||
const exited = new Promise((res) => child.on('exit', res));
|
||||
|
||||
// Write a tools/call then immediately close stdin. The old fire-and-forget
|
||||
// dispatch raced rl 'close' → process.exit and could drop this response.
|
||||
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 42, method: 'tools/call', params: { name: 'ruview_onboard', arguments: {} } }) + '\n');
|
||||
child.stdin.end();
|
||||
|
||||
await exited;
|
||||
const msgs = out.trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
||||
const resp = msgs.find((m) => m.id === 42);
|
||||
assert.ok(resp, 'the in-flight tools/call response must be flushed to stdout before exit');
|
||||
assert.equal(resp.result.isError, false);
|
||||
});
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readdirSync, readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { readdirSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { join, dirname, delimiter } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { claimCheck, summarize } from '../src/guardrails.js';
|
||||
import { TOOLS, TOOL_ALIASES, runTool, listTools, findRepoRoot, run, which } from '../src/tools.js';
|
||||
@@ -58,6 +59,33 @@ test('guardrail still catches real short-token metric claims', () => {
|
||||
assert.equal(claimCheck('IoU 0.75 across rooms.').ok, false);
|
||||
});
|
||||
|
||||
// Digits hidden in a code span still make a claim — scrubbing must not blind the
|
||||
// number gate to `0.95` (regression: code-span number bypassed the gate).
|
||||
test('guardrail flags an accuracy number stated inside a code span', () => {
|
||||
const r = claimCheck('Count accuracy reached `0.95` in our tests.');
|
||||
assert.equal(r.ok, false, JSON.stringify(r.findings));
|
||||
assert.ok(r.findings.some((f) => /not tagged/i.test(f.reason)));
|
||||
});
|
||||
|
||||
// A MEASURED claim whose only number hides in a code span must still reach the
|
||||
// missing-reproducer check (regression: the scrubbed gate short-circuited it).
|
||||
// Bare metric prose with no number at all (e.g. the README rule text) stays a pass.
|
||||
test('guardrail flags a MEASURED code-span number with no reproducer', () => {
|
||||
const r = claimCheck('Detection accuracy `0.97` on the set (MEASURED).');
|
||||
assert.equal(r.ok, false, JSON.stringify(r.findings));
|
||||
assert.ok(r.findings.some((f) => /no reproducer/i.test(f.reason)));
|
||||
assert.equal(claimCheck('Every accuracy number must be MEASURED against a baseline.').ok, true);
|
||||
});
|
||||
|
||||
// F1-score phrasings ("F1: 0.91", "F1 reaches 0.91") were scrubbed as option
|
||||
// labels and slipped through; option refs alone must still not false-positive.
|
||||
test('guardrail catches F1-score claims but not bare option refs (ADR-263 F11)', () => {
|
||||
assert.equal(claimCheck('F1: 0.91 on the held-out set.').ok, false, 'F1: value is a metric claim');
|
||||
assert.equal(claimCheck('F1 reaches 0.91 on the held-out set.').ok, false, 'F1 with a nearby number is a claim');
|
||||
assert.equal(claimCheck('Options O1–O9 are tracked in ADR-263 O2.').ok, true, 'option labels are not metrics');
|
||||
assert.equal(claimCheck('ADR-263 O2 lands the exports fix.').ok, true);
|
||||
});
|
||||
|
||||
test('summarize gives PASS/finding text', () => {
|
||||
assert.match(summarize(claimCheck('nothing here')), /PASS/);
|
||||
assert.match(summarize(claimCheck('100% accuracy')), /finding/);
|
||||
@@ -156,10 +184,29 @@ test('run() bounds captured output instead of dying on big streams (ADR-263 O4)'
|
||||
assert.ok(r.stdout.includes('TAIL_MARKER'), 'tail must keep the end of the stream');
|
||||
});
|
||||
|
||||
test('which() finds node and memoizes misses', () => {
|
||||
test('which() finds node and re-probes misses (hits are cached)', () => {
|
||||
assert.ok(which('node'), 'node must be on PATH in the test env');
|
||||
assert.equal(which('definitely-not-a-binary-xyz'), null);
|
||||
assert.equal(which('definitely-not-a-binary-xyz'), null); // cached path
|
||||
assert.equal(which('definitely-not-a-binary-xyz'), null); // re-probed, still absent
|
||||
});
|
||||
|
||||
// ADR-263 O8: a miss must not be cached — an operator who installs a tool
|
||||
// mid-session (e.g. python after a python_missing failure) must be found next call.
|
||||
test('which() re-probes after a miss so a newly-installed tool is found', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'ruview-which-'));
|
||||
const name = 'ruview-probe-xyz';
|
||||
const isWin = process.platform === 'win32';
|
||||
const bin = join(dir, isWin ? `${name}.cmd` : name);
|
||||
const prevPath = process.env.PATH;
|
||||
try {
|
||||
assert.equal(which(name), null, 'not on PATH yet → miss');
|
||||
writeFileSync(bin, isWin ? '@echo off\n' : '#!/bin/sh\n', { mode: 0o755 });
|
||||
process.env.PATH = dir + delimiter + prevPath;
|
||||
assert.ok(which(name), 'installed mid-session → the miss must not have been cached');
|
||||
} finally {
|
||||
process.env.PATH = prevPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('CLI run(): claim-check exits non-zero on a bad claim', async () => {
|
||||
|
||||
Reference in New Issue
Block a user