mirror of
https://github.com/ruvnet/RuView
synced 2026-08-10 20:31:42 +00:00
feat(firmware): complete ADR-061 QEMU testing platform (all 9 layers)
Fix 9 bugs (LFSR bias, MAC filter init, scenario loop, NVS boundary values), add 7 new files completing Layers 3 (mesh), 4 (GDB), 5 (coverage), 8 (snapshots), 9 (chaos testing), expand CI with fuzz and NVS validation jobs, update README with full platform overview. Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv
Executable
+341
@@ -0,0 +1,341 @@
|
||||
#!/bin/bash
|
||||
# QEMU Chaos / Fault Injection Test Runner — ADR-061 Layer 9
|
||||
#
|
||||
# Launches firmware under QEMU and injects a series of faults to verify
|
||||
# the firmware's resilience. Each fault is injected via the QEMU monitor
|
||||
# socket (or GDB stub), followed by a recovery window and health check.
|
||||
#
|
||||
# Fault types:
|
||||
# 1. wifi_kill — Pause/resume VM to simulate WiFi reconnect
|
||||
# 2. ring_flood — Inject 1000 rapid mock frames (ring buffer stress)
|
||||
# 3. heap_pressure — Write to heap metadata to simulate low memory
|
||||
# 4. timer_starvation — Pause VM for 500ms to starve FreeRTOS timers
|
||||
# 5. corrupt_frame — Inject a CSI frame with bad magic bytes
|
||||
# 6. nvs_corrupt — Write garbage to NVS flash region
|
||||
#
|
||||
# Environment variables:
|
||||
# QEMU_PATH - Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
# QEMU_TIMEOUT - Boot timeout in seconds (default: 15)
|
||||
# FLASH_IMAGE - Path to merged flash image (default: build/qemu_flash.bin)
|
||||
# FAULT_WAIT - Seconds to wait after fault injection (default: 5)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 All faults handled gracefully
|
||||
# 1 Some faults caused degraded state
|
||||
# 2 Some faults caused failures
|
||||
# 3 Fatal — firmware crashed or QEMU died
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
BUILD_DIR="$FIRMWARE_DIR/build"
|
||||
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
|
||||
FLASH_IMAGE="${FLASH_IMAGE:-$BUILD_DIR/qemu_flash.bin}"
|
||||
BOOT_TIMEOUT="${QEMU_TIMEOUT:-15}"
|
||||
FAULT_WAIT="${FAULT_WAIT:-5}"
|
||||
MONITOR_SOCK="$BUILD_DIR/qemu-chaos.sock"
|
||||
LOG_DIR="$BUILD_DIR/chaos-tests"
|
||||
UART_LOG="$LOG_DIR/qemu_uart.log"
|
||||
QEMU_PID=""
|
||||
|
||||
# Fault definitions
|
||||
FAULTS=("wifi_kill" "ring_flood" "heap_pressure" "timer_starvation" "corrupt_frame" "nvs_corrupt")
|
||||
declare -a FAULT_RESULTS=()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Cleanup
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "[cleanup] Shutting down QEMU and removing socket..."
|
||||
if [ -n "$QEMU_PID" ] && kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
kill "$QEMU_PID" 2>/dev/null || true
|
||||
wait "$QEMU_PID" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$MONITOR_SOCK"
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
monitor_cmd() {
|
||||
local cmd="$1"
|
||||
local timeout="${2:-5}"
|
||||
echo "$cmd" | socat - "UNIX-CONNECT:$MONITOR_SOCK,connect-timeout=$timeout" 2>/dev/null
|
||||
}
|
||||
|
||||
log_line_count() {
|
||||
wc -l < "$UART_LOG" 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
wait_for_boot() {
|
||||
local elapsed=0
|
||||
while [ "$elapsed" -lt "$BOOT_TIMEOUT" ]; do
|
||||
if [ -f "$UART_LOG" ] && grep -qE "app_main|main_task|ESP32-S3|mock_csi" "$UART_LOG" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Fault injection functions
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
inject_wifi_kill() {
|
||||
# Simulate WiFi disconnect/reconnect by pausing and resuming the VM.
|
||||
# The firmware should handle the time gap gracefully.
|
||||
echo " [inject] Pausing VM for 2s (simulating WiFi disconnect)..."
|
||||
monitor_cmd "stop"
|
||||
sleep 2
|
||||
echo " [inject] Resuming VM (simulating WiFi reconnect)..."
|
||||
monitor_cmd "cont"
|
||||
}
|
||||
|
||||
inject_ring_flood() {
|
||||
# Send 1000 rapid mock frames by triggering scenario 7 repeatedly.
|
||||
# This stresses the ring buffer and tests backpressure handling.
|
||||
echo " [inject] Flooding ring buffer with 1000 rapid frame triggers..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault ring_flood
|
||||
}
|
||||
|
||||
inject_heap_pressure() {
|
||||
# Use monitor to simulate memory pressure by writing to heap tracking
|
||||
# regions. The firmware's heap checks should detect and handle this.
|
||||
echo " [inject] Simulating heap pressure via memory write..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault heap_exhaust
|
||||
}
|
||||
|
||||
inject_timer_starvation() {
|
||||
# Pause execution for 500ms to starve FreeRTOS timer callbacks.
|
||||
# Tests watchdog recovery and timer resilience.
|
||||
echo " [inject] Starving timers (500ms pause)..."
|
||||
monitor_cmd "stop"
|
||||
sleep 0.5
|
||||
monitor_cmd "cont"
|
||||
}
|
||||
|
||||
inject_corrupt_frame() {
|
||||
# Inject a CSI frame with bad magic bytes via monitor memory write.
|
||||
# The frame parser should reject it without crashing.
|
||||
echo " [inject] Injecting corrupt CSI frame (bad magic)..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault corrupt_frame
|
||||
}
|
||||
|
||||
inject_nvs_corrupt() {
|
||||
# Write garbage to the NVS flash region (offset 0x9000).
|
||||
# The firmware should detect NVS corruption and fall back to defaults.
|
||||
echo " [inject] Corrupting NVS flash region..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault nvs_corrupt
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Pre-flight checks
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "=== QEMU Chaos Test Runner — ADR-061 Layer 9 ==="
|
||||
echo "QEMU binary: $QEMU_BIN"
|
||||
echo "Flash image: $FLASH_IMAGE"
|
||||
echo "Boot timeout: ${BOOT_TIMEOUT}s"
|
||||
echo "Fault wait: ${FAULT_WAIT}s"
|
||||
echo "Faults: ${FAULTS[*]}"
|
||||
echo ""
|
||||
|
||||
if ! command -v "$QEMU_BIN" &>/dev/null; then
|
||||
echo "ERROR: QEMU binary not found: $QEMU_BIN"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v socat &>/dev/null; then
|
||||
echo "ERROR: socat not found. Install socat for QEMU monitor communication."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ ! -f "$FLASH_IMAGE" ]; then
|
||||
echo "ERROR: Flash image not found: $FLASH_IMAGE"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Launch QEMU
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Launching QEMU ──"
|
||||
echo ""
|
||||
|
||||
rm -f "$MONITOR_SOCK"
|
||||
> "$UART_LOG"
|
||||
|
||||
QEMU_ARGS=(
|
||||
-machine esp32s3
|
||||
-nographic
|
||||
-drive "file=$FLASH_IMAGE,if=mtd,format=raw"
|
||||
-serial "file:$UART_LOG"
|
||||
-no-reboot
|
||||
-monitor "unix:$MONITOR_SOCK,server,nowait"
|
||||
)
|
||||
|
||||
"$QEMU_BIN" "${QEMU_ARGS[@]}" &
|
||||
QEMU_PID=$!
|
||||
echo "[qemu] PID=$QEMU_PID"
|
||||
|
||||
# Wait for monitor socket
|
||||
waited=0
|
||||
while [ ! -S "$MONITOR_SOCK" ] && [ "$waited" -lt 10 ]; do
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
|
||||
if [ ! -S "$MONITOR_SOCK" ]; then
|
||||
echo "ERROR: QEMU monitor socket did not appear after 10s"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Wait for boot
|
||||
echo "[boot] Waiting for firmware boot (up to ${BOOT_TIMEOUT}s)..."
|
||||
if wait_for_boot; then
|
||||
echo "[boot] Firmware booted successfully."
|
||||
else
|
||||
echo "[boot] No boot indicator found (continuing anyway)."
|
||||
fi
|
||||
|
||||
# Let firmware stabilize for a few seconds
|
||||
echo "[boot] Stabilizing (3s)..."
|
||||
sleep 3
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Fault injection loop
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Fault Injection ──"
|
||||
echo ""
|
||||
|
||||
MAX_EXIT=0
|
||||
|
||||
for fault in "${FAULTS[@]}"; do
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Fault: $fault"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Record log position before injection
|
||||
pre_lines=$(log_line_count)
|
||||
|
||||
# Check QEMU is still alive
|
||||
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
echo " ERROR: QEMU process died before fault injection"
|
||||
FAULT_RESULTS+=("${fault}:3")
|
||||
MAX_EXIT=3
|
||||
break
|
||||
fi
|
||||
|
||||
# Inject the fault
|
||||
case "$fault" in
|
||||
wifi_kill) inject_wifi_kill ;;
|
||||
ring_flood) inject_ring_flood ;;
|
||||
heap_pressure) inject_heap_pressure ;;
|
||||
timer_starvation) inject_timer_starvation ;;
|
||||
corrupt_frame) inject_corrupt_frame ;;
|
||||
nvs_corrupt) inject_nvs_corrupt ;;
|
||||
*)
|
||||
echo " ERROR: Unknown fault type: $fault"
|
||||
FAULT_RESULTS+=("${fault}:2")
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Wait for firmware to respond/recover
|
||||
echo " [recovery] Waiting ${FAULT_WAIT}s for recovery..."
|
||||
sleep "$FAULT_WAIT"
|
||||
|
||||
# Extract post-fault log segment
|
||||
post_lines=$(log_line_count)
|
||||
new_lines=$((post_lines - pre_lines))
|
||||
fault_log="$LOG_DIR/fault_${fault}.log"
|
||||
|
||||
if [ "$new_lines" -gt 0 ]; then
|
||||
tail -n "$new_lines" "$UART_LOG" > "$fault_log"
|
||||
else
|
||||
# Grab last 50 lines as context
|
||||
tail -n 50 "$UART_LOG" > "$fault_log"
|
||||
fi
|
||||
|
||||
echo " [check] Captured $new_lines new log lines"
|
||||
|
||||
# Health check
|
||||
fault_exit=0
|
||||
python3 "$SCRIPT_DIR/check_health.py" \
|
||||
--log "$fault_log" \
|
||||
--after-fault "$fault" || fault_exit=$?
|
||||
|
||||
case "$fault_exit" in
|
||||
0) echo " [result] HEALTHY — firmware recovered gracefully" ;;
|
||||
1) echo " [result] DEGRADED — firmware running but with issues" ;;
|
||||
*) echo " [result] UNHEALTHY — firmware in bad state" ;;
|
||||
esac
|
||||
|
||||
FAULT_RESULTS+=("${fault}:${fault_exit}")
|
||||
if [ "$fault_exit" -gt "$MAX_EXIT" ]; then
|
||||
MAX_EXIT=$fault_exit
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Summary
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Chaos Test Results ──"
|
||||
echo ""
|
||||
|
||||
PASS=0
|
||||
DEGRADED=0
|
||||
FAIL=0
|
||||
|
||||
for result in "${FAULT_RESULTS[@]}"; do
|
||||
name="${result%%:*}"
|
||||
code="${result##*:}"
|
||||
case "$code" in
|
||||
0) echo " [PASS] $name"; PASS=$((PASS + 1)) ;;
|
||||
1) echo " [DEGRADED] $name"; DEGRADED=$((DEGRADED + 1)) ;;
|
||||
*) echo " [FAIL] $name"; FAIL=$((FAIL + 1)) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " $PASS passed, $DEGRADED degraded, $FAIL failed out of ${#FAULTS[@]} faults"
|
||||
echo ""
|
||||
|
||||
# Check if QEMU survived all faults
|
||||
if kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
echo " QEMU process survived all fault injections."
|
||||
else
|
||||
echo " WARNING: QEMU process died during fault injection."
|
||||
if [ "$MAX_EXIT" -lt 3 ]; then
|
||||
MAX_EXIT=3
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Chaos Test Complete (exit code: $MAX_EXIT) ==="
|
||||
exit "$MAX_EXIT"
|
||||
Reference in New Issue
Block a user