fix(scripts): improve usability across all ADR-061 QEMU testing scripts

- Add --help/-h flags to all 4 shell scripts with usage, env vars, examples
- Add prerequisite checks with install hints (apt/brew/pip) for missing tools
- Standardize exit codes (0=PASS, 1=WARN, 2=FAIL, 3=FATAL) across all scripts
- Standardize MESH_TIMEOUT to QEMU_TIMEOUT with backward compatibility
- Add SKIP_BUILD precheck for missing flash image in qemu-esp32s3-test.sh
- Add argparse to validate_qemu_output.py (was using raw sys.argv)
- Improve error messages in generate_nvs_matrix.py with NVS tool install hints
- Add socket connection warnings in inject_fault.py connect_monitor()
- Add example output epilog to check_health.py --help
- Add glossary (14 terms) and quick-start section to ADR-061
- Add GDB debugging walkthrough to ADR-061 Layer 4
- Fix stat portability in CI workflow (stat -c%s -> portable file_size())
- Add -type f to find commands in CI workflow

Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv
2026-03-14 11:19:39 -04:00
parent fb2d1afb0c
commit 1dbea4e9fb
11 changed files with 457 additions and 51 deletions
+7
View File
@@ -256,6 +256,13 @@ def run_health_checks(
def main():
parser = argparse.ArgumentParser(
description="QEMU Post-Fault Health Checker — ADR-061 Layer 9",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=(
"Example output:\n"
" [HEALTHY] t=30s frames=150 (5.0 fps) crashes=0 heap_err=0 wdt=0 reboots=0\n"
" \n"
" VERDICT: Firmware is healthy. No critical issues detected."
),
)
parser.add_argument(
"--log", required=True,
+15 -6
View File
@@ -314,12 +314,21 @@ def generate_nvs_binary(csv_content: str, size: int) -> bytes:
return f.read()
# Last resort: try as a module
subprocess.check_call([
sys.executable, "-m", "nvs_partition_gen", "generate",
csv_path, bin_path, hex(size)
])
with open(bin_path, "rb") as f:
return f.read()
try:
subprocess.check_call([
sys.executable, "-m", "nvs_partition_gen", "generate",
csv_path, bin_path, hex(size)
])
with open(bin_path, "rb") as f:
return f.read()
except (subprocess.CalledProcessError, FileNotFoundError):
print("ERROR: NVS partition generator tool not found.", file=sys.stderr)
print("Install: pip install esp-idf-nvs-partition-gen", file=sys.stderr)
print("Or set IDF_PATH to your ESP-IDF installation", file=sys.stderr)
raise RuntimeError(
"NVS partition generator not available. "
"Install: pip install esp-idf-nvs-partition-gen"
)
finally:
for p in (csv_path, bin_path):
+5 -1
View File
@@ -46,8 +46,12 @@ def connect_monitor(sock_path: str, timeout: float = CMD_TIMEOUT) -> socket.sock
banner = s.recv(RECV_BUFSIZE).decode("utf-8", errors="replace")
if banner:
pass # Consume silently
else:
print(f"WARNING: Connected to {sock_path} but received no banner data. "
f"QEMU monitor may not be ready.", file=sys.stderr)
except socket.timeout:
pass # No banner is OK
print(f"WARNING: Connected to {sock_path} but timed out waiting for banner "
f"after {timeout}s. QEMU monitor may be unresponsive.", file=sys.stderr)
return s
+60 -5
View File
@@ -20,10 +20,52 @@
# FAULT_WAIT - Seconds to wait after fault injection (default: 5)
#
# Exit codes:
# 0 All faults handled gracefully
# 1 Some faults caused degraded state
# 2 Some faults caused failures
# 3 Fatal — firmware crashed or QEMU died
# 0 PASS — all checks passed
# 1 WARN — non-critical checks failed
# 2 FAIL — critical checks failed
# 3 FATAL — build error, crash, or infrastructure failure
# ── Help ──────────────────────────────────────────────────────────────
usage() {
cat <<'HELP'
Usage: qemu-chaos-test.sh [OPTIONS]
Launch firmware under QEMU and inject a series of faults to verify the
firmware's resilience. Each fault is injected via the QEMU monitor socket,
followed by a recovery window and health check.
Fault types:
wifi_kill Pause/resume VM to simulate WiFi reconnect
ring_flood Inject 1000 rapid mock frames (ring buffer stress)
heap_pressure Write to heap metadata to simulate low memory
timer_starvation Pause VM for 500ms to starve FreeRTOS timers
corrupt_frame Inject a CSI frame with bad magic bytes
nvs_corrupt Write garbage to NVS flash region
Options:
-h, --help Show this help message and exit
Environment variables:
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
QEMU_TIMEOUT Boot timeout in seconds (default: 15)
FLASH_IMAGE Path to merged flash image (default: build/qemu_flash.bin)
FAULT_WAIT Seconds to wait after injection (default: 5)
Examples:
./qemu-chaos-test.sh
QEMU_TIMEOUT=30 FAULT_WAIT=10 ./qemu-chaos-test.sh
FLASH_IMAGE=/path/to/image.bin ./qemu-chaos-test.sh
Exit codes:
0 PASS — all checks passed
1 WARN — non-critical checks failed
2 FAIL — critical checks failed
3 FATAL — build error, crash, or infrastructure failure
HELP
exit 0
}
case "${1:-}" in -h|--help) usage ;; esac
set -euo pipefail
@@ -160,16 +202,29 @@ echo ""
if ! command -v "$QEMU_BIN" &>/dev/null; then
echo "ERROR: QEMU binary not found: $QEMU_BIN"
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
echo " Install: brew install qemu # macOS"
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
exit 3
fi
if ! command -v socat &>/dev/null; then
echo "ERROR: socat not found. Install socat for QEMU monitor communication."
echo "ERROR: socat not found (needed for QEMU monitor communication)."
echo " Install: sudo apt install socat # Debian/Ubuntu"
echo " Install: brew install socat # macOS"
exit 3
fi
if ! command -v python3 &>/dev/null; then
echo "ERROR: python3 not found (needed for fault injection scripts)."
echo " Install: sudo apt install python3 # Debian/Ubuntu"
echo " Install: brew install python # macOS"
exit 3
fi
if [ ! -f "$FLASH_IMAGE" ]; then
echo "ERROR: Flash image not found: $FLASH_IMAGE"
echo "Run qemu-esp32s3-test.sh first to build the flash image."
exit 3
fi
+63 -6
View File
@@ -12,10 +12,44 @@
# NVS_BIN - Path to a pre-built NVS binary to inject (optional)
#
# Exit codes:
# 0 All checks passed
# 1 Warnings (non-critical checks failed)
# 2 Errors (critical checks failed)
# 3 Fatal (crash detected or build failure)
# 0 PASS — all checks passed
# 1 WARN — non-critical checks failed
# 2 FAIL — critical checks failed
# 3 FATAL — build error, crash, or infrastructure failure
# ── Help ──────────────────────────────────────────────────────────────
usage() {
cat <<'HELP'
Usage: qemu-esp32s3-test.sh [OPTIONS]
Build ESP32-S3 firmware with mock CSI, merge binaries into a single flash
image, run under QEMU with a timeout, and validate the UART output.
Options:
-h, --help Show this help message and exit
Environment variables:
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
QEMU_TIMEOUT Timeout in seconds (default: 60)
SKIP_BUILD Set to "1" to skip idf.py build (default: unset)
NVS_BIN Path to pre-built NVS binary (optional)
QEMU_NET Set to "0" to disable networking (default: 1)
Examples:
./qemu-esp32s3-test.sh
SKIP_BUILD=1 ./qemu-esp32s3-test.sh
QEMU_PATH=/opt/qemu/bin/qemu-system-xtensa QEMU_TIMEOUT=120 ./qemu-esp32s3-test.sh
Exit codes:
0 PASS — all checks passed
1 WARN — non-critical checks failed
2 FAIL — critical checks failed
3 FATAL — build error, crash, or infrastructure failure
HELP
exit 0
}
case "${1:-}" in -h|--help) usage ;; esac
set -euo pipefail
@@ -35,10 +69,33 @@ echo "QEMU binary: $QEMU_BIN"
echo "Timeout: ${TIMEOUT_SEC}s"
echo ""
# Verify QEMU is available
# ── Prerequisite checks ───────────────────────────────────────────────
if ! command -v "$QEMU_BIN" &>/dev/null; then
echo "ERROR: QEMU binary not found: $QEMU_BIN"
echo "Set QEMU_PATH to the qemu-system-xtensa binary."
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
echo " Install: brew install qemu # macOS"
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
exit 3
fi
if ! command -v python3 &>/dev/null; then
echo "ERROR: python3 not found."
echo " Install: sudo apt install python3 # Debian/Ubuntu"
echo " Install: brew install python # macOS"
exit 3
fi
if ! python3 -m esptool version &>/dev/null 2>&1; then
echo "ERROR: esptool not found (needed to merge flash binaries)."
echo " Install: pip install esptool"
exit 3
fi
# ── SKIP_BUILD precheck ──────────────────────────────────────────────
if [ "${SKIP_BUILD:-}" = "1" ] && [ ! -f "$BUILD_DIR/esp32-csi-node.bin" ]; then
echo "ERROR: SKIP_BUILD=1 but flash image not found: $BUILD_DIR/esp32-csi-node.bin"
echo "Build the firmware first: ./qemu-esp32s3-test.sh (without SKIP_BUILD)"
echo "Or unset SKIP_BUILD to build automatically."
exit 3
fi
+76 -10
View File
@@ -24,10 +24,52 @@
# - Rust workspace with wifi-densepose-hardware crate (aggregator binary)
#
# Exit codes:
# 0 All checks passed
# 1 Warnings (non-critical checks failed)
# 2 Errors (critical checks failed)
# 3 Fatal (build failure, crash, or infrastructure error)
# 0 PASS — all checks passed
# 1 WARN — non-critical checks failed
# 2 FAIL — critical checks failed
# 3 FATAL — build error, crash, or infrastructure failure
# ── Help ──────────────────────────────────────────────────────────────
usage() {
cat <<'HELP'
Usage: sudo ./qemu-mesh-test.sh [OPTIONS] [N_NODES]
Spawn N ESP32-S3 QEMU instances connected via a Linux bridge, each with
unique NVS provisioning (node ID, TDM slot), and a Rust aggregator that
collects frames from all nodes.
NOTE: Requires root/sudo for TAP/bridge creation.
Options:
-h, --help Show this help message and exit
Positional:
N_NODES Number of mesh nodes (default: 3, minimum: 2)
Environment variables:
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
QEMU_TIMEOUT Timeout in seconds (default: 45)
MESH_TIMEOUT Alias for QEMU_TIMEOUT (deprecated)(default: 45)
SKIP_BUILD Set to "1" to skip idf.py build (default: unset)
BRIDGE_NAME Bridge interface name (default: qemu-br0)
BRIDGE_SUBNET Bridge IP/mask (default: 10.0.0.1/24)
AGGREGATOR_PORT UDP port for aggregator (default: 5005)
Examples:
sudo ./qemu-mesh-test.sh
sudo QEMU_TIMEOUT=90 ./qemu-mesh-test.sh 5
sudo SKIP_BUILD=1 ./qemu-mesh-test.sh 4
Exit codes:
0 PASS — all checks passed
1 WARN — non-critical checks failed
2 FAIL — critical checks failed
3 FATAL — build error, crash, or infrastructure failure
HELP
exit 0
}
case "${1:-}" in -h|--help) usage ;; esac
set -euo pipefail
@@ -48,7 +90,7 @@ VALIDATE_SCRIPT="$SCRIPT_DIR/validate_mesh_test.py"
# ---------------------------------------------------------------------------
N_NODES="${1:-3}"
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
MESH_TIMEOUT="${MESH_TIMEOUT:-45}"
TIMEOUT="${QEMU_TIMEOUT:-${MESH_TIMEOUT:-45}}"
BRIDGE="${BRIDGE_NAME:-qemu-br0}"
BRIDGE_IP="${BRIDGE_SUBNET:-10.0.0.1/24}"
AGG_PORT="${AGGREGATOR_PORT:-5005}"
@@ -59,7 +101,7 @@ echo "Nodes: $N_NODES"
echo "Bridge: $BRIDGE ($BRIDGE_IP)"
echo "Aggregator: 0.0.0.0:$AGG_PORT"
echo "QEMU binary: $QEMU_BIN"
echo "Timeout: ${MESH_TIMEOUT}s"
echo "Timeout: ${TIMEOUT}s"
echo ""
# ---------------------------------------------------------------------------
@@ -72,12 +114,22 @@ fi
if ! command -v "$QEMU_BIN" &>/dev/null; then
echo "ERROR: QEMU binary not found: $QEMU_BIN"
echo "Set QEMU_PATH to the qemu-system-xtensa binary."
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
echo " Install: brew install qemu # macOS"
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
exit 3
fi
if ! command -v python3 &>/dev/null; then
echo "ERROR: python3 not found."
echo " Install: sudo apt install python3 # Debian/Ubuntu"
echo " Install: brew install python # macOS"
exit 3
fi
if ! command -v ip &>/dev/null; then
echo "ERROR: 'ip' command not found. Install iproute2."
echo "ERROR: 'ip' command not found."
echo " Install: sudo apt install iproute2 # Debian/Ubuntu"
exit 3
fi
@@ -85,6 +137,20 @@ if ! command -v brctl &>/dev/null && ! ip link help bridge &>/dev/null 2>&1; the
echo "WARNING: bridge-utils not found; will use 'ip link' for bridge creation."
fi
if command -v socat &>/dev/null; then
true # optional, available
else
echo "NOTE: socat not found (optional, used for advanced monitor communication)."
echo " Install: sudo apt install socat # Debian/Ubuntu"
echo " Install: brew install socat # macOS"
fi
if ! command -v cargo &>/dev/null; then
echo "ERROR: cargo not found (needed to build the Rust aggregator)."
echo " Install: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh"
exit 3
fi
if [ "$(id -u)" -ne 0 ]; then
echo "ERROR: This script must be run as root (for TAP/bridge creation)."
echo "Usage: sudo $0 [N_NODES]"
@@ -288,13 +354,13 @@ for i in $(seq 0 $((N_NODES - 1))); do
done
echo ""
echo "All nodes launched. Waiting ${MESH_TIMEOUT}s for mesh simulation..."
echo "All nodes launched. Waiting ${TIMEOUT}s for mesh simulation..."
echo ""
# ---------------------------------------------------------------------------
# Wait for timeout
# ---------------------------------------------------------------------------
sleep "$MESH_TIMEOUT"
sleep "$TIMEOUT"
echo "Timeout reached. Stopping all processes..."
+51 -6
View File
@@ -16,10 +16,44 @@
# SKIP_SNAPSHOT - Set to "1" to run without snapshots (baseline timing)
#
# Exit codes:
# 0 All tests passed
# 1 Some tests had warnings
# 2 Some tests failed
# 3 Fatal error (QEMU failed to start, crash detected)
# 0 PASS — all checks passed
# 1 WARN — non-critical checks failed
# 2 FAIL — critical checks failed
# 3 FATAL — build error, crash, or infrastructure failure
# ── Help ──────────────────────────────────────────────────────────────
usage() {
cat <<'HELP'
Usage: qemu-snapshot-test.sh [OPTIONS]
Use QEMU VM snapshots to accelerate repeated test runs. Snapshots the VM
state after boot and after the first CSI frame, then restores from the
snapshot for each individual test (~2s vs ~15s per test).
Options:
-h, --help Show this help message and exit
Environment variables:
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
QEMU_TIMEOUT Per-test timeout in seconds (default: 10)
FLASH_IMAGE Path to merged flash image (default: build/qemu_flash.bin)
SKIP_SNAPSHOT Set to "1" to run without snapshots (baseline timing)
Examples:
./qemu-snapshot-test.sh
QEMU_TIMEOUT=20 ./qemu-snapshot-test.sh
FLASH_IMAGE=/path/to/image.bin ./qemu-snapshot-test.sh
Exit codes:
0 PASS — all checks passed
1 WARN — non-critical checks failed
2 FAIL — critical checks failed
3 FATAL — build error, crash, or infrastructure failure
HELP
exit 0
}
case "${1:-}" in -h|--help) usage ;; esac
set -euo pipefail
@@ -165,12 +199,23 @@ echo ""
if ! command -v "$QEMU_BIN" &>/dev/null; then
echo "ERROR: QEMU binary not found: $QEMU_BIN"
echo "Set QEMU_PATH to the qemu-system-xtensa binary."
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
echo " Install: brew install qemu # macOS"
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
exit 3
fi
if ! command -v qemu-img &>/dev/null; then
echo "ERROR: qemu-img not found (needed for snapshot disk management)."
echo " Install: sudo apt install qemu-utils # Debian/Ubuntu"
echo " Install: brew install qemu # macOS"
exit 3
fi
if ! command -v socat &>/dev/null; then
echo "ERROR: socat not found. Install socat for QEMU monitor communication."
echo "ERROR: socat not found (needed for QEMU monitor communication)."
echo " Install: sudo apt install socat # Debian/Ubuntu"
echo " Install: brew install socat # macOS"
exit 3
fi
+19 -7
View File
@@ -196,12 +196,18 @@ def validate_mesh(
# Load aggregator results if available
results: Optional[dict] = None
if results_path and results_path.exists():
try:
results = json.loads(results_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
report.add("Results JSON", Severity.ERROR,
f"Failed to parse results: {exc}")
if results_path:
if not results_path.exists():
print(f"WARNING: Aggregator results file not found: {results_path}",
file=sys.stderr)
report.add("Results JSON", Severity.WARN,
f"Results file not found: {results_path}")
else:
try:
results = json.loads(results_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
report.add("Results JSON", Severity.ERROR,
f"Failed to parse results: {exc}")
# Load per-node logs
node_logs: Dict[int, str] = {}
@@ -449,8 +455,14 @@ def validate_mesh(
def main():
parser = argparse.ArgumentParser(
description="Validate multi-node mesh QEMU test output (ADR-061 Layer 3)",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=(
"Examples:\n"
" python3 validate_mesh_test.py --nodes 3 --results mesh_results.json\n"
" python3 validate_mesh_test.py --nodes 4 --log node0.log --log node1.log"
),
)
parser.add_argument("results", nargs="?", default=None,
parser.add_argument("--results", default=None,
help="Path to mesh_test_results.json from aggregator")
parser.add_argument("--nodes", "-n", type=int, required=True,
help="Expected number of mesh nodes")
+12 -4
View File
@@ -16,6 +16,7 @@ Exit codes:
3 Fatal (crash or corruption detected)
"""
import argparse
import re
import sys
from dataclasses import dataclass, field
@@ -364,11 +365,18 @@ def validate_log(log_text: str) -> ValidationReport:
def main():
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <log_file>", file=sys.stderr)
sys.exit(3)
parser = argparse.ArgumentParser(
description="Validate QEMU ESP32-S3 UART output (ADR-061)",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="Example: python3 validate_qemu_output.py build/qemu_output.log",
)
parser.add_argument(
"log_file",
help="Path to QEMU UART log file",
)
args = parser.parse_args()
log_path = Path(sys.argv[1])
log_path = Path(args.log_file)
if not log_path.exists():
print(f"ERROR: Log file not found: {log_path}", file=sys.stderr)
sys.exit(3)