mirror of
https://github.com/ruvnet/RuView
synced 2026-07-25 17:51:48 +00:00
feat: QEMU ESP32-S3 testing platform + swarm configurator (ADR-061/062) (#260)
9-layer QEMU testing platform (ADR-061) and YAML-driven swarm configurator (ADR-062) for ESP32-S3 firmware testing without hardware. 12 commits, 56 files, +9,500 lines. Tested on Windows with Espressif QEMU 9.0.0 — firmware boots, mock CSI generates frames, 14/16 validation checks pass. 39 bugs found and fixed across 2 deep code reviews. Closes #259 Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
rUv
Executable
+290
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
QEMU Post-Fault Health Checker — ADR-061 Layer 9
|
||||
|
||||
Reads a log segment captured after a fault injection and checks whether
|
||||
the firmware is still healthy. Used by qemu-chaos-test.sh after each
|
||||
fault in the chaos testing loop.
|
||||
|
||||
Health checks:
|
||||
1. No crash patterns (Guru Meditation, assert, panic, abort)
|
||||
2. No heap errors (OOM, heap corruption, alloc failure)
|
||||
3. No stack overflow (FreeRTOS stack overflow hook)
|
||||
4. Firmware still producing frames (CSI frame activity)
|
||||
|
||||
Exit codes:
|
||||
0 HEALTHY — all checks pass
|
||||
1 DEGRADED — no crash, but missing expected activity
|
||||
2 UNHEALTHY — crash, heap error, or stack overflow detected
|
||||
|
||||
Usage:
|
||||
python3 check_health.py --log /path/to/fault_segment.log --after-fault wifi_kill
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
|
||||
# ANSI colors
|
||||
USE_COLOR = sys.stdout.isatty()
|
||||
|
||||
|
||||
def color(text: str, code: str) -> str:
|
||||
if not USE_COLOR:
|
||||
return text
|
||||
return f"\033[{code}m{text}\033[0m"
|
||||
|
||||
|
||||
def green(t: str) -> str:
|
||||
return color(t, "32")
|
||||
|
||||
|
||||
def yellow(t: str) -> str:
|
||||
return color(t, "33")
|
||||
|
||||
|
||||
def red(t: str) -> str:
|
||||
return color(t, "1;31")
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheck:
|
||||
name: str
|
||||
passed: bool
|
||||
message: str
|
||||
severity: int # 0=pass, 1=degraded, 2=unhealthy
|
||||
|
||||
|
||||
def check_no_crash(lines: List[str]) -> HealthCheck:
|
||||
"""Check for crash indicators in the log."""
|
||||
crash_patterns = [
|
||||
r"Guru Meditation",
|
||||
r"assert failed",
|
||||
r"abort\(\)",
|
||||
r"panic",
|
||||
r"LoadProhibited",
|
||||
r"StoreProhibited",
|
||||
r"InstrFetchProhibited",
|
||||
r"IllegalInstruction",
|
||||
r"Unhandled debug exception",
|
||||
r"Fatal exception",
|
||||
]
|
||||
|
||||
for line in lines:
|
||||
for pat in crash_patterns:
|
||||
if re.search(pat, line):
|
||||
return HealthCheck(
|
||||
name="No crash",
|
||||
passed=False,
|
||||
message=f"Crash detected: {line.strip()[:120]}",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
return HealthCheck(
|
||||
name="No crash",
|
||||
passed=True,
|
||||
message="No crash indicators found",
|
||||
severity=0,
|
||||
)
|
||||
|
||||
|
||||
def check_no_heap_errors(lines: List[str]) -> HealthCheck:
|
||||
"""Check for heap/memory errors."""
|
||||
heap_patterns = [
|
||||
r"HEAP_ERROR",
|
||||
r"out of memory",
|
||||
r"heap_caps_alloc.*failed",
|
||||
r"malloc.*fail",
|
||||
r"heap corruption",
|
||||
r"CORRUPT HEAP",
|
||||
r"multi_heap",
|
||||
r"heap_lock",
|
||||
]
|
||||
|
||||
for line in lines:
|
||||
for pat in heap_patterns:
|
||||
if re.search(pat, line, re.IGNORECASE):
|
||||
return HealthCheck(
|
||||
name="No heap errors",
|
||||
passed=False,
|
||||
message=f"Heap error: {line.strip()[:120]}",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
return HealthCheck(
|
||||
name="No heap errors",
|
||||
passed=True,
|
||||
message="No heap errors found",
|
||||
severity=0,
|
||||
)
|
||||
|
||||
|
||||
def check_no_stack_overflow(lines: List[str]) -> HealthCheck:
|
||||
"""Check for FreeRTOS stack overflow."""
|
||||
stack_patterns = [
|
||||
r"[Ss]tack overflow",
|
||||
r"stack_overflow",
|
||||
r"vApplicationStackOverflowHook",
|
||||
r"stack smashing",
|
||||
]
|
||||
|
||||
for line in lines:
|
||||
for pat in stack_patterns:
|
||||
if re.search(pat, line):
|
||||
return HealthCheck(
|
||||
name="No stack overflow",
|
||||
passed=False,
|
||||
message=f"Stack overflow: {line.strip()[:120]}",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
return HealthCheck(
|
||||
name="No stack overflow",
|
||||
passed=True,
|
||||
message="No stack overflow detected",
|
||||
severity=0,
|
||||
)
|
||||
|
||||
|
||||
def check_frame_activity(lines: List[str]) -> HealthCheck:
|
||||
"""Check that the firmware is still producing CSI frames."""
|
||||
frame_patterns = [
|
||||
r"frame",
|
||||
r"CSI",
|
||||
r"mock_csi",
|
||||
r"iq_data",
|
||||
r"subcarrier",
|
||||
r"csi_collector",
|
||||
r"enqueue",
|
||||
r"presence",
|
||||
r"vitals",
|
||||
r"breathing",
|
||||
]
|
||||
|
||||
activity_lines = 0
|
||||
for line in lines:
|
||||
for pat in frame_patterns:
|
||||
if re.search(pat, line, re.IGNORECASE):
|
||||
activity_lines += 1
|
||||
break
|
||||
|
||||
if activity_lines > 0:
|
||||
return HealthCheck(
|
||||
name="Frame activity",
|
||||
passed=True,
|
||||
message=f"Firmware producing output ({activity_lines} activity lines)",
|
||||
severity=0,
|
||||
)
|
||||
else:
|
||||
return HealthCheck(
|
||||
name="Frame activity",
|
||||
passed=False,
|
||||
message="No frame/CSI activity detected after fault",
|
||||
severity=1, # Degraded, not fatal
|
||||
)
|
||||
|
||||
|
||||
def run_health_checks(
|
||||
log_path: Path,
|
||||
fault_name: str,
|
||||
tail_lines: int = 200,
|
||||
) -> int:
|
||||
"""Run all health checks and report results.
|
||||
|
||||
Returns:
|
||||
0 = healthy, 1 = degraded, 2 = unhealthy
|
||||
"""
|
||||
if not log_path.exists():
|
||||
print(f" ERROR: Log file not found: {log_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
all_lines = text.splitlines()
|
||||
|
||||
# Use last N lines (most recent, after fault injection)
|
||||
lines = all_lines[-tail_lines:] if len(all_lines) > tail_lines else all_lines
|
||||
|
||||
if not lines:
|
||||
print(f" WARNING: Log file is empty (fault may have killed output)")
|
||||
# Empty log after fault is degraded, not necessarily unhealthy
|
||||
return 1
|
||||
|
||||
print(f" Health check after fault: {fault_name}")
|
||||
print(f" Log lines analyzed: {len(lines)} (of {len(all_lines)} total)")
|
||||
print()
|
||||
|
||||
# Run checks
|
||||
checks = [
|
||||
check_no_crash(lines),
|
||||
check_no_heap_errors(lines),
|
||||
check_no_stack_overflow(lines),
|
||||
check_frame_activity(lines),
|
||||
]
|
||||
|
||||
max_severity = 0
|
||||
for check in checks:
|
||||
if check.passed:
|
||||
icon = green("PASS")
|
||||
elif check.severity == 1:
|
||||
icon = yellow("WARN")
|
||||
else:
|
||||
icon = red("FAIL")
|
||||
|
||||
print(f" [{icon}] {check.name}: {check.message}")
|
||||
max_severity = max(max_severity, check.severity)
|
||||
|
||||
print()
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for c in checks if c.passed)
|
||||
total = len(checks)
|
||||
|
||||
if max_severity == 0:
|
||||
print(f" {green(f'HEALTHY')} — {passed}/{total} checks passed")
|
||||
elif max_severity == 1:
|
||||
print(f" {yellow(f'DEGRADED')} — {passed}/{total} checks passed")
|
||||
else:
|
||||
print(f" {red(f'UNHEALTHY')} — {passed}/{total} checks passed")
|
||||
|
||||
return max_severity
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="QEMU Post-Fault Health Checker — ADR-061 Layer 9",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"Example output:\n"
|
||||
" [HEALTHY] t=30s frames=150 (5.0 fps) crashes=0 heap_err=0 wdt=0 reboots=0\n"
|
||||
" \n"
|
||||
" VERDICT: Firmware is healthy. No critical issues detected."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log", required=True,
|
||||
help="Path to the log file (or log segment) to check",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--after-fault", required=True,
|
||||
help="Name of the fault that was injected (for reporting)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tail", type=int, default=200,
|
||||
help="Number of lines from end of log to analyze (default: 200)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
exit_code = run_health_checks(
|
||||
log_path=Path(args.log),
|
||||
fault_name=args.after_fault,
|
||||
tail_lines=args.tail,
|
||||
)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,430 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
NVS Test Matrix Generator (ADR-061)
|
||||
|
||||
Generates NVS partition binaries for 14 test configurations using the
|
||||
provision.py script's CSV builder and NVS binary generator. Each binary
|
||||
can be injected into a QEMU flash image at offset 0x9000 for automated
|
||||
firmware testing under different NVS configurations.
|
||||
|
||||
Usage:
|
||||
python3 generate_nvs_matrix.py --output-dir build/nvs_matrix
|
||||
|
||||
# Generate only specific configs:
|
||||
python3 generate_nvs_matrix.py --output-dir build/nvs_matrix --only default,full-adr060
|
||||
|
||||
Requirements:
|
||||
- esp_idf_nvs_partition_gen (pip install) or ESP-IDF nvs_partition_gen.py
|
||||
- Python 3.8+
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
# NVS partition size must match partitions_display.csv: 0x6000 = 24576 bytes
|
||||
NVS_PARTITION_SIZE = 0x6000
|
||||
|
||||
|
||||
@dataclass
|
||||
class NvsEntry:
|
||||
"""A single NVS key-value entry."""
|
||||
key: str
|
||||
type: str # "data" or "namespace"
|
||||
encoding: str # "string", "u8", "u16", "u32", "hex2bin", ""
|
||||
value: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class NvsConfig:
|
||||
"""A named NVS configuration with a list of entries."""
|
||||
name: str
|
||||
description: str
|
||||
entries: List[NvsEntry] = field(default_factory=list)
|
||||
|
||||
def to_csv(self) -> str:
|
||||
"""Generate NVS CSV content."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf)
|
||||
writer.writerow(["key", "type", "encoding", "value"])
|
||||
writer.writerow(["csi_cfg", "namespace", "", ""])
|
||||
for entry in self.entries:
|
||||
writer.writerow([entry.key, entry.type, entry.encoding, entry.value])
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def define_configs() -> List[NvsConfig]:
|
||||
"""Define all 14 NVS test configurations."""
|
||||
configs = []
|
||||
|
||||
# 1. default - no NVS entries (firmware uses Kconfig defaults)
|
||||
configs.append(NvsConfig(
|
||||
name="default",
|
||||
description="No NVS entries; firmware uses Kconfig defaults",
|
||||
entries=[],
|
||||
))
|
||||
|
||||
# 2. wifi-only - just WiFi credentials
|
||||
configs.append(NvsConfig(
|
||||
name="wifi-only",
|
||||
description="WiFi SSID and password only",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
],
|
||||
))
|
||||
|
||||
# 3. full-adr060 - channel override + MAC filter
|
||||
configs.append(NvsConfig(
|
||||
name="full-adr060",
|
||||
description="ADR-060: channel override + MAC filter + full config",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("target_port", "data", "u16", "5005"),
|
||||
NvsEntry("node_id", "data", "u8", "1"),
|
||||
NvsEntry("csi_channel", "data", "u8", "6"),
|
||||
NvsEntry("filter_mac", "data", "hex2bin", "aabbccddeeff"),
|
||||
],
|
||||
))
|
||||
|
||||
# 4. edge-tier0 - raw passthrough (no DSP)
|
||||
configs.append(NvsConfig(
|
||||
name="edge-tier0",
|
||||
description="Edge tier 0: raw CSI passthrough, no on-device DSP",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "0"),
|
||||
],
|
||||
))
|
||||
|
||||
# 5. edge-tier1 - basic presence/motion detection
|
||||
configs.append(NvsConfig(
|
||||
name="edge-tier1",
|
||||
description="Edge tier 1: basic presence and motion detection",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "1"),
|
||||
NvsEntry("pres_thresh", "data", "u16", "50"),
|
||||
],
|
||||
))
|
||||
|
||||
# 6. edge-tier2-custom - full pipeline with custom thresholds
|
||||
configs.append(NvsConfig(
|
||||
name="edge-tier2-custom",
|
||||
description="Edge tier 2: full pipeline with custom thresholds",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "2"),
|
||||
NvsEntry("pres_thresh", "data", "u16", "100"),
|
||||
NvsEntry("fall_thresh", "data", "u16", "3000"),
|
||||
NvsEntry("vital_win", "data", "u16", "256"),
|
||||
NvsEntry("vital_int", "data", "u16", "500"),
|
||||
NvsEntry("subk_count", "data", "u8", "16"),
|
||||
],
|
||||
))
|
||||
|
||||
# 7. tdm-3node - TDM mesh with 3 nodes (slot 0)
|
||||
configs.append(NvsConfig(
|
||||
name="tdm-3node",
|
||||
description="TDM mesh: 3-node schedule, this node is slot 0",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("node_id", "data", "u8", "0"),
|
||||
NvsEntry("tdm_slot", "data", "u8", "0"),
|
||||
NvsEntry("tdm_nodes", "data", "u8", "3"),
|
||||
],
|
||||
))
|
||||
|
||||
# 8. wasm-signed - WASM runtime with signature verification
|
||||
configs.append(NvsConfig(
|
||||
name="wasm-signed",
|
||||
description="WASM runtime enabled with Ed25519 signature verification",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "2"),
|
||||
# wasm_verify=1 + a 32-byte dummy Ed25519 pubkey
|
||||
NvsEntry("wasm_verify", "data", "u8", "1"),
|
||||
NvsEntry("wasm_pubkey", "data", "hex2bin",
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
|
||||
],
|
||||
))
|
||||
|
||||
# 9. wasm-unsigned - WASM runtime without signature verification
|
||||
configs.append(NvsConfig(
|
||||
name="wasm-unsigned",
|
||||
description="WASM runtime with signature verification disabled",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "2"),
|
||||
NvsEntry("wasm_verify", "data", "u8", "0"),
|
||||
NvsEntry("wasm_max", "data", "u8", "2"),
|
||||
],
|
||||
))
|
||||
|
||||
# 10. 5ghz-channel - 5 GHz channel override
|
||||
configs.append(NvsConfig(
|
||||
name="5ghz-channel",
|
||||
description="ADR-060: 5 GHz channel 36 override",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork5G"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("csi_channel", "data", "u8", "36"),
|
||||
],
|
||||
))
|
||||
|
||||
# 11. boundary-max - maximum VALID values for all numeric fields
|
||||
# Uses firmware-validated max ranges (not raw u8/u16 max):
|
||||
# vital_win: 32-256, top_k: 1-32, power_duty: 10-100
|
||||
configs.append(NvsConfig(
|
||||
name="boundary-max",
|
||||
description="Boundary test: maximum valid values per firmware validation ranges",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("target_port", "data", "u16", "65535"),
|
||||
NvsEntry("node_id", "data", "u8", "255"),
|
||||
NvsEntry("edge_tier", "data", "u8", "2"),
|
||||
NvsEntry("pres_thresh", "data", "u16", "65535"),
|
||||
NvsEntry("fall_thresh", "data", "u16", "65535"),
|
||||
NvsEntry("vital_win", "data", "u16", "256"), # max validated
|
||||
NvsEntry("vital_int", "data", "u16", "10000"),
|
||||
NvsEntry("subk_count", "data", "u8", "32"),
|
||||
NvsEntry("power_duty", "data", "u8", "100"),
|
||||
],
|
||||
))
|
||||
|
||||
# 12. boundary-min - minimum VALID values for all numeric fields
|
||||
configs.append(NvsConfig(
|
||||
name="boundary-min",
|
||||
description="Boundary test: minimum valid values per firmware validation ranges",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("target_port", "data", "u16", "1024"),
|
||||
NvsEntry("node_id", "data", "u8", "0"),
|
||||
NvsEntry("edge_tier", "data", "u8", "0"),
|
||||
NvsEntry("pres_thresh", "data", "u16", "1"),
|
||||
NvsEntry("fall_thresh", "data", "u16", "100"), # min valid (0.1 rad/s²)
|
||||
NvsEntry("vital_win", "data", "u16", "32"), # min validated
|
||||
NvsEntry("vital_int", "data", "u16", "100"),
|
||||
NvsEntry("subk_count", "data", "u8", "1"),
|
||||
NvsEntry("power_duty", "data", "u8", "10"),
|
||||
],
|
||||
))
|
||||
|
||||
# 13. power-save - low power duty cycle configuration
|
||||
configs.append(NvsConfig(
|
||||
name="power-save",
|
||||
description="Power-save mode: 10% duty cycle for battery-powered nodes",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", "TestNetwork"),
|
||||
NvsEntry("password", "data", "string", "testpass123"),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
NvsEntry("edge_tier", "data", "u8", "1"),
|
||||
NvsEntry("power_duty", "data", "u8", "10"),
|
||||
],
|
||||
))
|
||||
|
||||
# 14. empty-strings - empty SSID/password to test fallback to Kconfig
|
||||
configs.append(NvsConfig(
|
||||
name="empty-strings",
|
||||
description="Empty SSID and password to verify Kconfig fallback",
|
||||
entries=[
|
||||
NvsEntry("ssid", "data", "string", ""),
|
||||
NvsEntry("password", "data", "string", ""),
|
||||
NvsEntry("target_ip", "data", "string", "10.0.2.2"),
|
||||
],
|
||||
))
|
||||
|
||||
return configs
|
||||
|
||||
|
||||
def generate_nvs_binary(csv_content: str, size: int) -> bytes:
|
||||
"""Generate an NVS partition binary from CSV content.
|
||||
|
||||
Tries multiple methods to find nvs_partition_gen:
|
||||
1. esp_idf_nvs_partition_gen pip package
|
||||
2. Legacy nvs_partition_gen pip package
|
||||
3. ESP-IDF bundled script (via IDF_PATH)
|
||||
4. Module invocation
|
||||
"""
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f_csv:
|
||||
f_csv.write(csv_content)
|
||||
csv_path = f_csv.name
|
||||
|
||||
bin_path = csv_path.replace(".csv", ".bin")
|
||||
|
||||
try:
|
||||
# Try pip-installed version first
|
||||
try:
|
||||
from esp_idf_nvs_partition_gen import nvs_partition_gen
|
||||
nvs_partition_gen.generate(csv_path, bin_path, size)
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Try legacy import
|
||||
try:
|
||||
import nvs_partition_gen
|
||||
nvs_partition_gen.generate(csv_path, bin_path, size)
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Try ESP-IDF bundled script
|
||||
idf_path = os.environ.get("IDF_PATH", "")
|
||||
gen_script = os.path.join(
|
||||
idf_path, "components", "nvs_flash",
|
||||
"nvs_partition_generator", "nvs_partition_gen.py"
|
||||
)
|
||||
if os.path.isfile(gen_script):
|
||||
subprocess.check_call([
|
||||
sys.executable, gen_script, "generate",
|
||||
csv_path, bin_path, hex(size)
|
||||
])
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
# Last resort: try as a module
|
||||
try:
|
||||
subprocess.check_call([
|
||||
sys.executable, "-m", "nvs_partition_gen", "generate",
|
||||
csv_path, bin_path, hex(size)
|
||||
])
|
||||
with open(bin_path, "rb") as f:
|
||||
return f.read()
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
print("ERROR: NVS partition generator tool not found.", file=sys.stderr)
|
||||
print("Install: pip install esp-idf-nvs-partition-gen", file=sys.stderr)
|
||||
print("Or set IDF_PATH to your ESP-IDF installation", file=sys.stderr)
|
||||
raise RuntimeError(
|
||||
"NVS partition generator not available. "
|
||||
"Install: pip install esp-idf-nvs-partition-gen"
|
||||
)
|
||||
|
||||
finally:
|
||||
for p in set((csv_path, bin_path)): # deduplicate in case paths are identical
|
||||
if os.path.isfile(p):
|
||||
os.unlink(p)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate NVS partition binaries for QEMU firmware test matrix (ADR-061)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", required=True,
|
||||
help="Directory to write NVS binary files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only", type=str, default=None,
|
||||
help="Comma-separated list of config names to generate (default: all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv-only", action="store_true",
|
||||
help="Only generate CSV files, skip binary generation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list", action="store_true", dest="list_configs",
|
||||
help="List all available configurations and exit",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
all_configs = define_configs()
|
||||
|
||||
if args.list_configs:
|
||||
print(f"{'Name':<20} {'Description'}")
|
||||
print("-" * 70)
|
||||
for cfg in all_configs:
|
||||
print(f"{cfg.name:<20} {cfg.description}")
|
||||
sys.exit(0)
|
||||
|
||||
# Filter configs if --only specified
|
||||
if args.only:
|
||||
selected = set(args.only.split(","))
|
||||
configs = [c for c in all_configs if c.name in selected]
|
||||
missing = selected - {c.name for c in configs}
|
||||
if missing:
|
||||
print(f"WARNING: Unknown config names: {', '.join(sorted(missing))}",
|
||||
file=sys.stderr)
|
||||
else:
|
||||
configs = all_configs
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"Generating {len(configs)} NVS configurations in {output_dir}/")
|
||||
print()
|
||||
|
||||
success = 0
|
||||
errors = 0
|
||||
|
||||
for cfg in configs:
|
||||
csv_content = cfg.to_csv()
|
||||
|
||||
# Always write the CSV for reference
|
||||
csv_path = output_dir / f"nvs_{cfg.name}.csv"
|
||||
csv_path.write_text(csv_content)
|
||||
|
||||
if cfg.name == "default" and not cfg.entries:
|
||||
# "default" means no NVS — just produce an empty marker
|
||||
print(f" [{cfg.name}] No NVS entries (uses Kconfig defaults)")
|
||||
# Write a zero-filled NVS partition (erased state = 0xFF)
|
||||
bin_path = output_dir / f"nvs_{cfg.name}.bin"
|
||||
bin_path.write_bytes(b"\xff" * NVS_PARTITION_SIZE)
|
||||
success += 1
|
||||
continue
|
||||
|
||||
if args.csv_only:
|
||||
print(f" [{cfg.name}] CSV only: {csv_path}")
|
||||
success += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
nvs_bin = generate_nvs_binary(csv_content, NVS_PARTITION_SIZE)
|
||||
bin_path = output_dir / f"nvs_{cfg.name}.bin"
|
||||
bin_path.write_bytes(nvs_bin)
|
||||
print(f" [{cfg.name}] {len(nvs_bin)} bytes -> {bin_path}")
|
||||
success += 1
|
||||
except Exception as e:
|
||||
print(f" [{cfg.name}] ERROR: {e}", file=sys.stderr)
|
||||
errors += 1
|
||||
|
||||
print()
|
||||
print(f"Done: {success} succeeded, {errors} failed")
|
||||
|
||||
if errors > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+258
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
QEMU Fault Injector — ADR-061 Layer 9
|
||||
|
||||
Connects to a QEMU monitor socket and injects a specified fault type.
|
||||
Used by qemu-chaos-test.sh to stress-test firmware resilience.
|
||||
|
||||
Supported faults:
|
||||
wifi_kill - Pause/resume VM (simulates WiFi reconnect)
|
||||
ring_flood - Send 1000 rapid commands to stress ring buffer
|
||||
heap_exhaust - Write to heap metadata region to simulate OOM
|
||||
timer_starvation - Pause VM for 500ms to starve FreeRTOS timers
|
||||
corrupt_frame - Write bad magic bytes to CSI frame buffer area
|
||||
nvs_corrupt - Write garbage to NVS flash region (offset 0x9000)
|
||||
|
||||
Usage:
|
||||
python3 inject_fault.py --socket /path/to/qemu.sock --fault wifi_kill
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
# Timeout for each monitor command (seconds)
|
||||
CMD_TIMEOUT = 5.0
|
||||
|
||||
# QEMU monitor response buffer size
|
||||
RECV_BUFSIZE = 4096
|
||||
|
||||
|
||||
def connect_monitor(sock_path: str, timeout: float = CMD_TIMEOUT) -> socket.socket:
|
||||
"""Connect to the QEMU monitor Unix domain socket."""
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.settimeout(timeout)
|
||||
try:
|
||||
s.connect(sock_path)
|
||||
except (socket.error, FileNotFoundError) as e:
|
||||
print(f"ERROR: Cannot connect to QEMU monitor at {sock_path}: {e}",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Read the initial QEMU monitor banner/prompt
|
||||
try:
|
||||
banner = s.recv(RECV_BUFSIZE).decode("utf-8", errors="replace")
|
||||
if banner:
|
||||
pass # Consume silently
|
||||
else:
|
||||
print(f"WARNING: Connected to {sock_path} but received no banner data. "
|
||||
f"QEMU monitor may not be ready.", file=sys.stderr)
|
||||
except socket.timeout:
|
||||
print(f"WARNING: Connected to {sock_path} but timed out waiting for banner "
|
||||
f"after {timeout}s. QEMU monitor may be unresponsive.", file=sys.stderr)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def send_cmd(s: socket.socket, cmd: str, timeout: float = CMD_TIMEOUT) -> str:
|
||||
"""Send a command to the QEMU monitor and return the response."""
|
||||
s.settimeout(timeout)
|
||||
try:
|
||||
s.sendall((cmd + "\n").encode("utf-8"))
|
||||
except (BrokenPipeError, ConnectionResetError) as e:
|
||||
print(f"ERROR: Lost connection to QEMU monitor: {e}", file=sys.stderr)
|
||||
return ""
|
||||
|
||||
# Read response (may be multi-line)
|
||||
response = ""
|
||||
try:
|
||||
while True:
|
||||
chunk = s.recv(RECV_BUFSIZE).decode("utf-8", errors="replace")
|
||||
if not chunk:
|
||||
break
|
||||
response += chunk
|
||||
# QEMU monitor prompt ends with "(qemu) "
|
||||
if "(qemu)" in chunk:
|
||||
break
|
||||
except socket.timeout:
|
||||
pass # Response may not have a clean prompt
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def fault_wifi_kill(s: socket.socket) -> None:
|
||||
"""Pause VM for 2s then resume — simulates WiFi disconnect/reconnect."""
|
||||
print("[wifi_kill] Pausing VM...")
|
||||
send_cmd(s, "stop")
|
||||
time.sleep(2.0)
|
||||
print("[wifi_kill] Resuming VM...")
|
||||
send_cmd(s, "cont")
|
||||
print("[wifi_kill] Injected: 2s pause/resume cycle")
|
||||
|
||||
|
||||
def fault_ring_flood(s: socket.socket) -> None:
|
||||
"""Send 1000 rapid NMI injections to stress the ring buffer.
|
||||
|
||||
On real hardware, scenario 7 is a high-rate CSI burst. Under QEMU
|
||||
we simulate this by rapidly triggering NMIs which the mock CSI
|
||||
handler processes as frame events.
|
||||
"""
|
||||
print("[ring_flood] Sending 1000 rapid commands...")
|
||||
sent = 0
|
||||
for i in range(1000):
|
||||
try:
|
||||
# Use 'nmi' to trigger interrupt handler (mock CSI frame path)
|
||||
s.sendall(b"nmi\n")
|
||||
sent += 1
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
print(f"[ring_flood] Connection lost after {sent} commands")
|
||||
break
|
||||
|
||||
# Drain any accumulated responses
|
||||
s.settimeout(1.0)
|
||||
try:
|
||||
while True:
|
||||
chunk = s.recv(RECV_BUFSIZE)
|
||||
if not chunk:
|
||||
break
|
||||
except socket.timeout:
|
||||
pass
|
||||
|
||||
print(f"[ring_flood] Injected: {sent}/1000 rapid NMI triggers")
|
||||
|
||||
|
||||
def fault_heap_exhaust(s: socket.socket, flash_path: str = None) -> None:
|
||||
"""Simulate memory pressure by pausing VM to trigger watchdog/heap checks.
|
||||
|
||||
Actual heap memory writes require a GDB stub (-gdb tcp::1234).
|
||||
This function probes the heap region and pauses the VM to stress
|
||||
heap management as a realistic simulation.
|
||||
"""
|
||||
heap_base = 0x3FC88000
|
||||
print("[heap_exhaust] Probing heap region...")
|
||||
resp = send_cmd(s, f"xp /4xw 0x{heap_base:08x}")
|
||||
print(f"[heap_exhaust] Heap header: {resp.strip()}")
|
||||
# Pause VM to stress memory management
|
||||
print("[heap_exhaust] Pausing VM for 3s to stress heap management...")
|
||||
send_cmd(s, "stop")
|
||||
time.sleep(3.0)
|
||||
send_cmd(s, "cont")
|
||||
print("[heap_exhaust] WARNING: Actual heap corruption requires GDB stub (-gdb tcp::1234)")
|
||||
print("[heap_exhaust] Injected: 3s VM pause (simulates memory pressure)")
|
||||
|
||||
|
||||
def fault_timer_starvation(s: socket.socket) -> None:
|
||||
"""Pause VM for 500ms — starves FreeRTOS tick and timer callbacks."""
|
||||
print("[timer_starvation] Pausing VM for 500ms...")
|
||||
send_cmd(s, "stop")
|
||||
time.sleep(0.5)
|
||||
send_cmd(s, "cont")
|
||||
print("[timer_starvation] Injected: 500ms execution pause")
|
||||
|
||||
|
||||
def fault_corrupt_frame(s: socket.socket, flash_path: str = None) -> None:
|
||||
"""Simulate CSI frame corruption by pausing VM during frame processing.
|
||||
|
||||
Actual memory writes to the frame buffer require a GDB stub
|
||||
(-gdb tcp::1234). This function probes the frame buffer region
|
||||
and pauses the VM mid-frame to simulate corruption effects.
|
||||
"""
|
||||
frame_buf_addr = 0x3FCA0000
|
||||
print(f"[corrupt_frame] Probing frame buffer at 0x{frame_buf_addr:08X}...")
|
||||
resp = send_cmd(s, f"xp /4xb 0x{frame_buf_addr:08x}")
|
||||
print(f"[corrupt_frame] Frame buffer: {resp.strip()}")
|
||||
# Pause VM briefly to disrupt frame processing timing
|
||||
print("[corrupt_frame] Pausing VM for 1s to disrupt frame processing...")
|
||||
send_cmd(s, "stop")
|
||||
time.sleep(1.0)
|
||||
send_cmd(s, "cont")
|
||||
print("[corrupt_frame] WARNING: Actual frame corruption requires GDB stub (-gdb tcp::1234)")
|
||||
print(f"[corrupt_frame] Injected: 1s VM pause during frame processing")
|
||||
|
||||
|
||||
def fault_nvs_corrupt(s: socket.socket, flash_path: str = None) -> None:
|
||||
"""Write garbage to the NVS flash region on disk.
|
||||
|
||||
When a flash image path is provided, writes random bytes directly
|
||||
to the NVS partition offset (0x9000) in the flash image file.
|
||||
Without a flash path, falls back to a read-only probe via monitor.
|
||||
"""
|
||||
if flash_path and os.path.isfile(flash_path):
|
||||
nvs_offset = 0x9000
|
||||
garbage = bytes(random.randint(0, 255) for _ in range(16))
|
||||
with open(flash_path, "r+b") as f:
|
||||
f.seek(nvs_offset)
|
||||
f.write(garbage)
|
||||
print(f"[nvs_corrupt] Wrote 16 garbage bytes at flash offset 0x{nvs_offset:X}")
|
||||
print(f"[nvs_corrupt] Flash image: {flash_path}")
|
||||
else:
|
||||
# Fallback: attempt via monitor (read-only probe)
|
||||
resp = send_cmd(s, f"xp /8xb 0x3C009000")
|
||||
print(f"[nvs_corrupt] NVS region (read-only probe): {resp.strip()}")
|
||||
print(f"[nvs_corrupt] WARNING: No --flash path provided; NVS corruption was NOT injected")
|
||||
print(f"[nvs_corrupt] Pass --flash /path/to/flash.bin for actual corruption")
|
||||
|
||||
|
||||
# Map fault names to injection functions
|
||||
FAULT_MAP = {
|
||||
"wifi_kill": fault_wifi_kill,
|
||||
"ring_flood": fault_ring_flood,
|
||||
"heap_exhaust": fault_heap_exhaust,
|
||||
"timer_starvation": fault_timer_starvation,
|
||||
"corrupt_frame": fault_corrupt_frame,
|
||||
"nvs_corrupt": fault_nvs_corrupt,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="QEMU Fault Injector — ADR-061 Layer 9",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--socket", required=True,
|
||||
help="Path to QEMU monitor Unix domain socket",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fault", required=True, choices=list(FAULT_MAP.keys()),
|
||||
help="Fault type to inject",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout", type=float, default=CMD_TIMEOUT,
|
||||
help=f"Per-command timeout in seconds (default: {CMD_TIMEOUT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--flash", default=None,
|
||||
help="Path to flash image (for nvs_corrupt direct file writes)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"[inject_fault] Connecting to {args.socket}...")
|
||||
s = connect_monitor(args.socket, timeout=args.timeout)
|
||||
|
||||
print(f"[inject_fault] Injecting fault: {args.fault}")
|
||||
try:
|
||||
fault_fn = FAULT_MAP[args.fault]
|
||||
# Pass flash_path to faults that accept it
|
||||
import inspect
|
||||
sig = inspect.signature(fault_fn)
|
||||
if "flash_path" in sig.parameters:
|
||||
fault_fn(s, flash_path=args.flash)
|
||||
else:
|
||||
fault_fn(s)
|
||||
except Exception as e:
|
||||
print(f"ERROR: Fault injection failed: {e}", file=sys.stderr)
|
||||
s.close()
|
||||
sys.exit(1)
|
||||
|
||||
s.close()
|
||||
print(f"[inject_fault] Complete: {args.fault}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,337 @@
|
||||
#!/bin/bash
|
||||
# install-qemu.sh — Install QEMU with ESP32-S3 support (Espressif fork)
|
||||
# Usage: bash scripts/install-qemu.sh [OPTIONS]
|
||||
set -euo pipefail
|
||||
|
||||
# ── Colors ────────────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'; CYAN='\033[0;36m'; BOLD='\033[1m'; NC='\033[0m'
|
||||
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
err() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
step() { echo -e "\n${CYAN}${BOLD}▶ $*${NC}"; }
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────────────
|
||||
INSTALL_DIR="$HOME/.espressif/qemu"
|
||||
BRANCH="esp-develop"
|
||||
JOBS=""
|
||||
SKIP_DEPS=false
|
||||
UNINSTALL=false
|
||||
CHECK_ONLY=false
|
||||
QEMU_REPO="https://github.com/espressif/qemu.git"
|
||||
|
||||
# ── Usage ─────────────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<EOF
|
||||
${BOLD}install-qemu.sh${NC} — Install QEMU with ESP32-S3 support (Espressif fork)
|
||||
|
||||
${BOLD}USAGE${NC}
|
||||
bash scripts/install-qemu.sh [OPTIONS]
|
||||
|
||||
${BOLD}OPTIONS${NC}
|
||||
--install-dir DIR Installation directory (default: ~/.espressif/qemu)
|
||||
--branch TAG QEMU branch or tag to build (default: esp-develop)
|
||||
--jobs N Parallel build jobs (default: nproc)
|
||||
--skip-deps Skip system dependency installation
|
||||
--uninstall Remove QEMU installation
|
||||
--check Verify existing installation and exit
|
||||
-h, --help Show this help
|
||||
|
||||
${BOLD}EXIT CODES${NC}
|
||||
0 Success
|
||||
1 Dependency installation failed
|
||||
2 Build failed
|
||||
3 Unsupported OS
|
||||
|
||||
${BOLD}EXAMPLES${NC}
|
||||
bash scripts/install-qemu.sh
|
||||
bash scripts/install-qemu.sh --install-dir /opt/qemu-esp --jobs 8
|
||||
bash scripts/install-qemu.sh --check
|
||||
bash scripts/install-qemu.sh --uninstall
|
||||
EOF
|
||||
}
|
||||
|
||||
# ── Parse args ────────────────────────────────────────────────────────────────
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--install-dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--branch) BRANCH="$2"; shift 2 ;;
|
||||
--jobs) JOBS="$2"; shift 2 ;;
|
||||
--skip-deps) SKIP_DEPS=true; shift ;;
|
||||
--uninstall) UNINSTALL=true; shift ;;
|
||||
--check) CHECK_ONLY=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) err "Unknown option: $1"; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── OS detection ──────────────────────────────────────────────────────────────
|
||||
detect_os() {
|
||||
OS="unknown"
|
||||
DISTRO="unknown"
|
||||
IS_WSL=false
|
||||
|
||||
case "$(uname -s)" in
|
||||
Linux)
|
||||
OS="linux"
|
||||
if grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
IS_WSL=true
|
||||
fi
|
||||
if [ -f /etc/os-release ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/os-release
|
||||
case "$ID" in
|
||||
ubuntu|debian|pop|linuxmint|elementary) DISTRO="debian" ;;
|
||||
fedora|rhel|centos|rocky|alma) DISTRO="fedora" ;;
|
||||
arch|manjaro|endeavouros) DISTRO="arch" ;;
|
||||
opensuse*|sles) DISTRO="suse" ;;
|
||||
*) DISTRO="$ID" ;;
|
||||
esac
|
||||
fi
|
||||
;;
|
||||
Darwin) OS="macos"; DISTRO="macos" ;;
|
||||
MINGW*|MSYS*)
|
||||
err "Native Windows/MINGW detected."
|
||||
err "QEMU ESP32-S3 must be built on Linux or macOS."
|
||||
err "Options:"
|
||||
err " 1. Use WSL: wsl bash scripts/install-qemu.sh"
|
||||
err " 2. Use Docker: docker run -it ubuntu:22.04 bash"
|
||||
err " 3. Download pre-built: https://github.com/espressif/qemu/releases"
|
||||
exit 3
|
||||
;;
|
||||
*) err "Unsupported OS: $(uname -s)"; exit 3 ;;
|
||||
esac
|
||||
|
||||
info "Detected: OS=${OS} Distro=${DISTRO} WSL=${IS_WSL}"
|
||||
}
|
||||
|
||||
# ── Check existing installation ───────────────────────────────────────────────
|
||||
check_installation() {
|
||||
local qemu_bin="$INSTALL_DIR/build/qemu-system-xtensa"
|
||||
if [ -x "$qemu_bin" ]; then
|
||||
local version
|
||||
version=$("$qemu_bin" --version 2>/dev/null | head -1) || true
|
||||
if [ -n "$version" ]; then
|
||||
ok "QEMU installed: $version"
|
||||
ok "Binary: $qemu_bin"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
# Check PATH
|
||||
if command -v qemu-system-xtensa &>/dev/null; then
|
||||
local version
|
||||
version=$(qemu-system-xtensa --version 2>/dev/null | head -1) || true
|
||||
ok "QEMU found in PATH: $version"
|
||||
return 0
|
||||
fi
|
||||
warn "QEMU with ESP32-S3 support not found"
|
||||
return 1
|
||||
}
|
||||
|
||||
if $CHECK_ONLY; then
|
||||
detect_os
|
||||
if check_installation; then exit 0; else exit 1; fi
|
||||
fi
|
||||
|
||||
# ── Uninstall ─────────────────────────────────────────────────────────────────
|
||||
if $UNINSTALL; then
|
||||
step "Uninstalling QEMU from $INSTALL_DIR"
|
||||
if [ -d "$INSTALL_DIR" ]; then
|
||||
rm -rf "$INSTALL_DIR"
|
||||
ok "Removed $INSTALL_DIR"
|
||||
else
|
||||
warn "Directory not found: $INSTALL_DIR"
|
||||
fi
|
||||
# Remove symlink
|
||||
local_bin="$HOME/.local/bin/qemu-system-xtensa"
|
||||
if [ -L "$local_bin" ]; then
|
||||
rm -f "$local_bin"
|
||||
ok "Removed symlink $local_bin"
|
||||
fi
|
||||
ok "Uninstall complete"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Main install flow ─────────────────────────────────────────────────────────
|
||||
detect_os
|
||||
|
||||
# Default jobs = nproc
|
||||
if [ -z "$JOBS" ]; then
|
||||
if command -v nproc &>/dev/null; then
|
||||
JOBS=$(nproc)
|
||||
elif command -v sysctl &>/dev/null; then
|
||||
JOBS=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
else
|
||||
JOBS=4
|
||||
fi
|
||||
fi
|
||||
info "Build parallelism: $JOBS jobs"
|
||||
|
||||
# ── Step 1: Install dependencies ──────────────────────────────────────────────
|
||||
install_deps() {
|
||||
step "Installing build dependencies"
|
||||
|
||||
case "$DISTRO" in
|
||||
debian)
|
||||
info "Using apt (Debian/Ubuntu)"
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq \
|
||||
git build-essential python3 python3-pip python3-venv \
|
||||
ninja-build pkg-config libglib2.0-dev libpixman-1-dev \
|
||||
libslirp-dev libgcrypt-dev
|
||||
;;
|
||||
fedora)
|
||||
info "Using dnf (Fedora/RHEL)"
|
||||
sudo dnf install -y \
|
||||
git gcc gcc-c++ make python3 python3-pip \
|
||||
ninja-build pkgconfig glib2-devel pixman-devel \
|
||||
libslirp-devel libgcrypt-devel
|
||||
;;
|
||||
arch)
|
||||
info "Using pacman (Arch)"
|
||||
sudo pacman -S --needed --noconfirm \
|
||||
git base-devel python python-pip \
|
||||
ninja pkgconf glib2 pixman libslirp libgcrypt
|
||||
;;
|
||||
suse)
|
||||
info "Using zypper (openSUSE)"
|
||||
sudo zypper install -y \
|
||||
git gcc gcc-c++ make python3 python3-pip \
|
||||
ninja pkg-config glib2-devel libpixman-1-0-devel \
|
||||
libslirp-devel libgcrypt-devel
|
||||
;;
|
||||
macos)
|
||||
info "Using Homebrew"
|
||||
if ! command -v brew &>/dev/null; then
|
||||
err "Homebrew not found. Install from https://brew.sh"
|
||||
exit 1
|
||||
fi
|
||||
brew install glib pixman ninja pkg-config libslirp libgcrypt || true
|
||||
;;
|
||||
*)
|
||||
warn "Unknown distro '$DISTRO' — install these manually:"
|
||||
warn " git, gcc/g++, python3, ninja, pkg-config, glib2-dev, pixman-dev, libslirp-dev"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
ok "Dependencies installed"
|
||||
}
|
||||
|
||||
if ! $SKIP_DEPS; then
|
||||
install_deps || { err "Dependency installation failed"; exit 1; }
|
||||
else
|
||||
info "Skipping dependency installation (--skip-deps)"
|
||||
fi
|
||||
|
||||
# ── Step 2: Clone Espressif QEMU fork ─────────────────────────────────────────
|
||||
step "Cloning Espressif QEMU fork"
|
||||
|
||||
SRC_DIR="$INSTALL_DIR"
|
||||
if [ -d "$SRC_DIR/.git" ]; then
|
||||
info "Repository already exists at $SRC_DIR"
|
||||
info "Fetching latest changes on branch $BRANCH"
|
||||
git -C "$SRC_DIR" fetch origin "$BRANCH" --depth=1
|
||||
git -C "$SRC_DIR" checkout "$BRANCH" 2>/dev/null || git -C "$SRC_DIR" checkout "origin/$BRANCH"
|
||||
ok "Updated to latest $BRANCH"
|
||||
else
|
||||
info "Cloning $QEMU_REPO (branch: $BRANCH)"
|
||||
mkdir -p "$(dirname "$SRC_DIR")"
|
||||
git clone --depth=1 --branch "$BRANCH" "$QEMU_REPO" "$SRC_DIR"
|
||||
ok "Cloned to $SRC_DIR"
|
||||
fi
|
||||
|
||||
# ── Step 3: Configure and build ───────────────────────────────────────────────
|
||||
step "Configuring QEMU (target: xtensa-softmmu)"
|
||||
|
||||
BUILD_DIR="$SRC_DIR/build"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
cd "$SRC_DIR"
|
||||
|
||||
./configure \
|
||||
--target-list=xtensa-softmmu \
|
||||
--enable-slirp \
|
||||
--enable-gcrypt \
|
||||
--prefix="$INSTALL_DIR/dist" \
|
||||
2>&1 | tail -5
|
||||
|
||||
step "Building QEMU ($JOBS parallel jobs)"
|
||||
make -j"$JOBS" -C "$BUILD_DIR" 2>&1 | tail -20
|
||||
|
||||
if [ ! -x "$BUILD_DIR/qemu-system-xtensa" ]; then
|
||||
err "Build failed — qemu-system-xtensa binary not found"
|
||||
err "Troubleshooting:"
|
||||
err " 1. Check build output above for errors"
|
||||
err " 2. Ensure all dependencies are installed: re-run without --skip-deps"
|
||||
err " 3. Try with fewer jobs: --jobs 1"
|
||||
err " 4. On macOS, ensure Xcode CLT: xcode-select --install"
|
||||
exit 2
|
||||
fi
|
||||
ok "Build succeeded: $BUILD_DIR/qemu-system-xtensa"
|
||||
|
||||
# ── Step 4: Create symlink / add to PATH ──────────────────────────────────────
|
||||
step "Setting up PATH access"
|
||||
|
||||
LOCAL_BIN="$HOME/.local/bin"
|
||||
mkdir -p "$LOCAL_BIN"
|
||||
ln -sf "$BUILD_DIR/qemu-system-xtensa" "$LOCAL_BIN/qemu-system-xtensa"
|
||||
ok "Symlinked to $LOCAL_BIN/qemu-system-xtensa"
|
||||
|
||||
# Check if ~/.local/bin is in PATH
|
||||
if ! echo "$PATH" | tr ':' '\n' | grep -qx "$LOCAL_BIN"; then
|
||||
warn "$LOCAL_BIN is not in your PATH"
|
||||
warn "Add this to your shell profile (~/.bashrc or ~/.zshrc):"
|
||||
echo -e " ${BOLD}export PATH=\"\$HOME/.local/bin:\$PATH\"${NC}"
|
||||
fi
|
||||
|
||||
# ── Step 5: Verify ────────────────────────────────────────────────────────────
|
||||
step "Verifying installation"
|
||||
|
||||
QEMU_VERSION=$("$BUILD_DIR/qemu-system-xtensa" --version | head -1)
|
||||
ok "$QEMU_VERSION"
|
||||
|
||||
# Check ESP32-S3 machine support
|
||||
if "$BUILD_DIR/qemu-system-xtensa" -machine help 2>/dev/null | grep -q esp32s3; then
|
||||
ok "ESP32-S3 machine type available"
|
||||
else
|
||||
warn "ESP32-S3 machine type not listed (may still work with newer builds)"
|
||||
fi
|
||||
|
||||
# ── Step 6: Install Python packages ──────────────────────────────────────────
|
||||
step "Installing Python packages (esptool, pyyaml, nvs-partition-gen)"
|
||||
|
||||
PIP_CMD="pip3"
|
||||
if ! command -v pip3 &>/dev/null; then
|
||||
PIP_CMD="python3 -m pip"
|
||||
fi
|
||||
|
||||
$PIP_CMD install --user --quiet \
|
||||
esptool \
|
||||
pyyaml \
|
||||
esp-idf-nvs-partition-gen \
|
||||
2>&1 || warn "Some Python packages failed to install (non-fatal)"
|
||||
|
||||
ok "Python packages installed"
|
||||
|
||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${GREEN}${BOLD}Installation complete!${NC}"
|
||||
echo ""
|
||||
echo -e "${BOLD}Next steps:${NC}"
|
||||
echo ""
|
||||
echo " 1. Run a smoke test:"
|
||||
echo -e " ${CYAN}qemu-system-xtensa -nographic -machine esp32s3 \\${NC}"
|
||||
echo -e " ${CYAN} -drive file=firmware.bin,if=mtd,format=raw \\${NC}"
|
||||
echo -e " ${CYAN} -serial mon:stdio${NC}"
|
||||
echo ""
|
||||
echo " 2. Run the project QEMU tests:"
|
||||
echo -e " ${CYAN}cd $(dirname "$0")/.."
|
||||
echo -e " pytest firmware/esp32-csi-node/tests/qemu/ -v${NC}"
|
||||
echo ""
|
||||
echo " 3. Binary location:"
|
||||
echo -e " ${CYAN}$BUILD_DIR/qemu-system-xtensa${NC}"
|
||||
echo ""
|
||||
echo -e " 4. Uninstall:"
|
||||
echo -e " ${CYAN}bash scripts/install-qemu.sh --uninstall${NC}"
|
||||
echo ""
|
||||
Executable
+397
@@ -0,0 +1,397 @@
|
||||
#!/bin/bash
|
||||
# QEMU Chaos / Fault Injection Test Runner — ADR-061 Layer 9
|
||||
#
|
||||
# Launches firmware under QEMU and injects a series of faults to verify
|
||||
# the firmware's resilience. Each fault is injected via the QEMU monitor
|
||||
# socket (or GDB stub), followed by a recovery window and health check.
|
||||
#
|
||||
# Fault types:
|
||||
# 1. wifi_kill — Pause/resume VM to simulate WiFi reconnect
|
||||
# 2. ring_flood — Inject 1000 rapid mock frames (ring buffer stress)
|
||||
# 3. heap_exhaust — Write to heap metadata to simulate low memory
|
||||
# 4. timer_starvation — Pause VM for 500ms to starve FreeRTOS timers
|
||||
# 5. corrupt_frame — Inject a CSI frame with bad magic bytes
|
||||
# 6. nvs_corrupt — Write garbage to NVS flash region
|
||||
#
|
||||
# Environment variables:
|
||||
# QEMU_PATH - Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
# QEMU_TIMEOUT - Boot timeout in seconds (default: 15)
|
||||
# FLASH_IMAGE - Path to merged flash image (default: build/qemu_flash.bin)
|
||||
# FAULT_WAIT - Seconds to wait after fault injection (default: 5)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PASS — all checks passed
|
||||
# 1 WARN — non-critical checks failed
|
||||
# 2 FAIL — critical checks failed
|
||||
# 3 FATAL — build error, crash, or infrastructure failure
|
||||
|
||||
# ── Help ──────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<'HELP'
|
||||
Usage: qemu-chaos-test.sh [OPTIONS]
|
||||
|
||||
Launch firmware under QEMU and inject a series of faults to verify the
|
||||
firmware's resilience. Each fault is injected via the QEMU monitor socket,
|
||||
followed by a recovery window and health check.
|
||||
|
||||
Fault types:
|
||||
wifi_kill Pause/resume VM to simulate WiFi reconnect
|
||||
ring_flood Inject 1000 rapid mock frames (ring buffer stress)
|
||||
heap_exhaust Write to heap metadata to simulate low memory
|
||||
timer_starvation Pause VM for 500ms to starve FreeRTOS timers
|
||||
corrupt_frame Inject a CSI frame with bad magic bytes
|
||||
nvs_corrupt Write garbage to NVS flash region
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message and exit
|
||||
|
||||
Environment variables:
|
||||
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
QEMU_TIMEOUT Boot timeout in seconds (default: 15)
|
||||
FLASH_IMAGE Path to merged flash image (default: build/qemu_flash.bin)
|
||||
FAULT_WAIT Seconds to wait after injection (default: 5)
|
||||
|
||||
Examples:
|
||||
./qemu-chaos-test.sh
|
||||
QEMU_TIMEOUT=30 FAULT_WAIT=10 ./qemu-chaos-test.sh
|
||||
FLASH_IMAGE=/path/to/image.bin ./qemu-chaos-test.sh
|
||||
|
||||
Exit codes:
|
||||
0 PASS — all checks passed
|
||||
1 WARN — non-critical checks failed
|
||||
2 FAIL — critical checks failed
|
||||
3 FATAL — build error, crash, or infrastructure failure
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
case "${1:-}" in -h|--help) usage ;; esac
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
BUILD_DIR="$FIRMWARE_DIR/build"
|
||||
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
|
||||
FLASH_IMAGE="${FLASH_IMAGE:-$BUILD_DIR/qemu_flash.bin}"
|
||||
BOOT_TIMEOUT="${QEMU_TIMEOUT:-15}"
|
||||
FAULT_WAIT="${FAULT_WAIT:-5}"
|
||||
MONITOR_SOCK="$BUILD_DIR/qemu-chaos.sock"
|
||||
LOG_DIR="$BUILD_DIR/chaos-tests"
|
||||
UART_LOG="$LOG_DIR/qemu_uart.log"
|
||||
QEMU_PID=""
|
||||
|
||||
# Fault definitions
|
||||
FAULTS=("wifi_kill" "ring_flood" "heap_exhaust" "timer_starvation" "corrupt_frame" "nvs_corrupt")
|
||||
declare -a FAULT_RESULTS=()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Cleanup
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "[cleanup] Shutting down QEMU and removing socket..."
|
||||
if [ -n "$QEMU_PID" ] && kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
kill "$QEMU_PID" 2>/dev/null || true
|
||||
wait "$QEMU_PID" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$MONITOR_SOCK"
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
monitor_cmd() {
|
||||
local cmd="$1"
|
||||
local timeout="${2:-5}"
|
||||
echo "$cmd" | socat - "UNIX-CONNECT:$MONITOR_SOCK,connect-timeout=$timeout" 2>/dev/null
|
||||
}
|
||||
|
||||
log_line_count() {
|
||||
wc -l < "$UART_LOG" 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
wait_for_boot() {
|
||||
local elapsed=0
|
||||
while [ "$elapsed" -lt "$BOOT_TIMEOUT" ]; do
|
||||
if [ -f "$UART_LOG" ] && grep -qE "app_main|main_task|ESP32-S3|mock_csi" "$UART_LOG" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Fault injection functions
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
inject_wifi_kill() {
|
||||
# Simulate WiFi disconnect/reconnect by pausing and resuming the VM.
|
||||
# The firmware should handle the time gap gracefully.
|
||||
echo " [inject] Pausing VM for 2s (simulating WiFi disconnect)..."
|
||||
monitor_cmd "stop"
|
||||
sleep 2
|
||||
echo " [inject] Resuming VM (simulating WiFi reconnect)..."
|
||||
monitor_cmd "cont"
|
||||
}
|
||||
|
||||
inject_ring_flood() {
|
||||
# Send 1000 rapid mock frames by triggering scenario 7 repeatedly.
|
||||
# This stresses the ring buffer and tests backpressure handling.
|
||||
echo " [inject] Flooding ring buffer with 1000 rapid frame triggers..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault ring_flood
|
||||
}
|
||||
|
||||
inject_heap_exhaust() {
|
||||
# Simulate memory pressure by pausing the VM to stress heap management.
|
||||
# Actual heap memory writes require GDB stub.
|
||||
echo " [inject] Simulating heap pressure via VM pause..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault heap_exhaust
|
||||
}
|
||||
|
||||
inject_timer_starvation() {
|
||||
# Pause execution for 500ms to starve FreeRTOS timer callbacks.
|
||||
# Tests watchdog recovery and timer resilience.
|
||||
echo " [inject] Starving timers (500ms pause)..."
|
||||
monitor_cmd "stop"
|
||||
sleep 0.5
|
||||
monitor_cmd "cont"
|
||||
}
|
||||
|
||||
inject_corrupt_frame() {
|
||||
# Inject a CSI frame with bad magic bytes via monitor memory write.
|
||||
# The frame parser should reject it without crashing.
|
||||
echo " [inject] Injecting corrupt CSI frame (bad magic)..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault corrupt_frame
|
||||
}
|
||||
|
||||
inject_nvs_corrupt() {
|
||||
# Write garbage to the NVS flash region (offset 0x9000) via direct file write.
|
||||
# The firmware should detect NVS corruption and fall back to defaults.
|
||||
echo " [inject] Corrupting NVS flash region..."
|
||||
python3 "$SCRIPT_DIR/inject_fault.py" \
|
||||
--socket "$MONITOR_SOCK" \
|
||||
--fault nvs_corrupt \
|
||||
--flash "$FLASH_IMAGE"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Pre-flight checks
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "=== QEMU Chaos Test Runner — ADR-061 Layer 9 ==="
|
||||
echo "QEMU binary: $QEMU_BIN"
|
||||
echo "Flash image: $FLASH_IMAGE"
|
||||
echo "Boot timeout: ${BOOT_TIMEOUT}s"
|
||||
echo "Fault wait: ${FAULT_WAIT}s"
|
||||
echo "Faults: ${FAULTS[*]}"
|
||||
echo ""
|
||||
|
||||
if ! command -v "$QEMU_BIN" &>/dev/null; then
|
||||
echo "ERROR: QEMU binary not found: $QEMU_BIN"
|
||||
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
|
||||
echo " Install: brew install qemu # macOS"
|
||||
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v socat &>/dev/null; then
|
||||
echo "ERROR: socat not found (needed for QEMU monitor communication)."
|
||||
echo " Install: sudo apt install socat # Debian/Ubuntu"
|
||||
echo " Install: brew install socat # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
echo "ERROR: python3 not found (needed for fault injection scripts)."
|
||||
echo " Install: sudo apt install python3 # Debian/Ubuntu"
|
||||
echo " Install: brew install python # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ ! -f "$FLASH_IMAGE" ]; then
|
||||
echo "ERROR: Flash image not found: $FLASH_IMAGE"
|
||||
echo "Run qemu-esp32s3-test.sh first to build the flash image."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Launch QEMU
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Launching QEMU ──"
|
||||
echo ""
|
||||
|
||||
rm -f "$MONITOR_SOCK"
|
||||
> "$UART_LOG"
|
||||
|
||||
QEMU_ARGS=(
|
||||
-machine esp32s3
|
||||
-nographic
|
||||
-drive "file=$FLASH_IMAGE,if=mtd,format=raw"
|
||||
-serial "file:$UART_LOG"
|
||||
-no-reboot
|
||||
-monitor "unix:$MONITOR_SOCK,server,nowait"
|
||||
)
|
||||
|
||||
"$QEMU_BIN" "${QEMU_ARGS[@]}" &
|
||||
QEMU_PID=$!
|
||||
echo "[qemu] PID=$QEMU_PID"
|
||||
|
||||
# Wait for monitor socket
|
||||
waited=0
|
||||
while [ ! -S "$MONITOR_SOCK" ] && [ "$waited" -lt 10 ]; do
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
|
||||
if [ ! -S "$MONITOR_SOCK" ]; then
|
||||
echo "ERROR: QEMU monitor socket did not appear after 10s"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Wait for boot
|
||||
echo "[boot] Waiting for firmware boot (up to ${BOOT_TIMEOUT}s)..."
|
||||
if wait_for_boot; then
|
||||
echo "[boot] Firmware booted successfully."
|
||||
else
|
||||
echo "[boot] No boot indicator found (continuing anyway)."
|
||||
fi
|
||||
|
||||
# Let firmware stabilize for a few seconds
|
||||
echo "[boot] Stabilizing (3s)..."
|
||||
sleep 3
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Fault injection loop
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Fault Injection ──"
|
||||
echo ""
|
||||
|
||||
MAX_EXIT=0
|
||||
|
||||
for fault in "${FAULTS[@]}"; do
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Fault: $fault"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Record log position before injection
|
||||
pre_lines=$(log_line_count)
|
||||
|
||||
# Check QEMU is still alive
|
||||
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
echo " ERROR: QEMU process died before fault injection"
|
||||
FAULT_RESULTS+=("${fault}:3")
|
||||
MAX_EXIT=3
|
||||
break
|
||||
fi
|
||||
|
||||
# Inject the fault
|
||||
case "$fault" in
|
||||
wifi_kill) inject_wifi_kill ;;
|
||||
ring_flood) inject_ring_flood ;;
|
||||
heap_exhaust) inject_heap_exhaust ;;
|
||||
timer_starvation) inject_timer_starvation ;;
|
||||
corrupt_frame) inject_corrupt_frame ;;
|
||||
nvs_corrupt) inject_nvs_corrupt ;;
|
||||
*)
|
||||
echo " ERROR: Unknown fault type: $fault"
|
||||
FAULT_RESULTS+=("${fault}:2")
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Wait for firmware to respond/recover
|
||||
echo " [recovery] Waiting ${FAULT_WAIT}s for recovery..."
|
||||
sleep "$FAULT_WAIT"
|
||||
|
||||
# Extract post-fault log segment
|
||||
post_lines=$(log_line_count)
|
||||
new_lines=$((post_lines - pre_lines))
|
||||
fault_log="$LOG_DIR/fault_${fault}.log"
|
||||
|
||||
if [ "$new_lines" -gt 0 ]; then
|
||||
tail -n "$new_lines" "$UART_LOG" > "$fault_log"
|
||||
else
|
||||
# Grab last 50 lines as context
|
||||
tail -n 50 "$UART_LOG" > "$fault_log"
|
||||
fi
|
||||
|
||||
echo " [check] Captured $new_lines new log lines"
|
||||
|
||||
# Health check
|
||||
fault_exit=0
|
||||
python3 "$SCRIPT_DIR/check_health.py" \
|
||||
--log "$fault_log" \
|
||||
--after-fault "$fault" || fault_exit=$?
|
||||
|
||||
case "$fault_exit" in
|
||||
0) echo " [result] HEALTHY — firmware recovered gracefully" ;;
|
||||
1) echo " [result] DEGRADED — firmware running but with issues" ;;
|
||||
*) echo " [result] UNHEALTHY — firmware in bad state" ;;
|
||||
esac
|
||||
|
||||
FAULT_RESULTS+=("${fault}:${fault_exit}")
|
||||
if [ "$fault_exit" -gt "$MAX_EXIT" ]; then
|
||||
MAX_EXIT=$fault_exit
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Summary
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Chaos Test Results ──"
|
||||
echo ""
|
||||
|
||||
PASS=0
|
||||
DEGRADED=0
|
||||
FAIL=0
|
||||
|
||||
for result in "${FAULT_RESULTS[@]}"; do
|
||||
name="${result%%:*}"
|
||||
code="${result##*:}"
|
||||
case "$code" in
|
||||
0) echo " [PASS] $name"; PASS=$((PASS + 1)) ;;
|
||||
1) echo " [DEGRADED] $name"; DEGRADED=$((DEGRADED + 1)) ;;
|
||||
*) echo " [FAIL] $name"; FAIL=$((FAIL + 1)) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " $PASS passed, $DEGRADED degraded, $FAIL failed out of ${#FAULTS[@]} faults"
|
||||
echo ""
|
||||
|
||||
# Check if QEMU survived all faults
|
||||
if kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
echo " QEMU process survived all fault injections."
|
||||
else
|
||||
echo " WARNING: QEMU process died during fault injection."
|
||||
if [ "$MAX_EXIT" -lt 3 ]; then
|
||||
MAX_EXIT=3
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Chaos Test Complete (exit code: $MAX_EXIT) ==="
|
||||
exit "$MAX_EXIT"
|
||||
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# qemu-cli.sh — Unified QEMU ESP32-S3 testing CLI (ADR-061)
|
||||
# Version: 1.0.0
|
||||
#
|
||||
# Single entry point for all QEMU testing operations.
|
||||
# Run `qemu-cli.sh help` or `qemu-cli.sh --help` for usage.
|
||||
# ============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="1.0.0"
|
||||
|
||||
# --- Colors ----------------------------------------------------------------
|
||||
if [[ -t 1 ]]; then
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'; CYAN='\033[0;36m'; BOLD='\033[1m'; RST='\033[0m'
|
||||
else
|
||||
RED=''; GREEN=''; YELLOW=''; BLUE=''; CYAN=''; BOLD=''; RST=''
|
||||
fi
|
||||
|
||||
# --- Resolve paths ---------------------------------------------------------
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
FUZZ_DIR="$FIRMWARE_DIR/test"
|
||||
|
||||
# --- Helpers ---------------------------------------------------------------
|
||||
info() { echo -e "${BLUE}[INFO]${RST} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${RST} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${RST} $*"; }
|
||||
err() { echo -e "${RED}[ERROR]${RST} $*" >&2; }
|
||||
die() { err "$@"; exit 1; }
|
||||
|
||||
need_qemu() {
|
||||
detect_qemu >/dev/null 2>&1 || \
|
||||
die "QEMU not found. Install with: ${CYAN}qemu-cli.sh install${RST}"
|
||||
}
|
||||
|
||||
detect_qemu() {
|
||||
# 1. Explicit env var
|
||||
if [[ -n "${QEMU_PATH:-}" ]] && [[ -x "$QEMU_PATH" ]]; then
|
||||
echo "$QEMU_PATH"; return 0
|
||||
fi
|
||||
# 2. On PATH
|
||||
local qemu
|
||||
qemu="$(command -v qemu-system-xtensa 2>/dev/null || true)"
|
||||
if [[ -n "$qemu" ]]; then echo "$qemu"; return 0; fi
|
||||
# 3. Espressif default build location
|
||||
local espressif_qemu="$HOME/.espressif/qemu/build/qemu-system-xtensa"
|
||||
if [[ -x "$espressif_qemu" ]]; then echo "$espressif_qemu"; return 0; fi
|
||||
return 1
|
||||
}
|
||||
|
||||
detect_python() {
|
||||
command -v python3 2>/dev/null || command -v python 2>/dev/null || echo "python3"
|
||||
}
|
||||
|
||||
# --- Command: help ---------------------------------------------------------
|
||||
cmd_help() {
|
||||
cat <<EOF
|
||||
${BOLD}qemu-cli.sh${RST} v${VERSION} — Unified QEMU ESP32-S3 testing CLI
|
||||
|
||||
${BOLD}USAGE${RST}
|
||||
qemu-cli.sh <command> [options]
|
||||
|
||||
${BOLD}COMMANDS${RST}
|
||||
${CYAN}install${RST} Install QEMU with ESP32-S3 support
|
||||
${CYAN}test${RST} Run single-node firmware test
|
||||
${CYAN}mesh${RST} [N] Run multi-node mesh test (default: 3 nodes)
|
||||
${CYAN}swarm${RST} [args] Run swarm configurator (qemu_swarm.py)
|
||||
${CYAN}snapshot${RST} [args] Run snapshot-based tests
|
||||
${CYAN}chaos${RST} [args] Run chaos / fault injection tests
|
||||
${CYAN}fuzz${RST} [--duration N] Run all 3 fuzz targets (clang libFuzzer)
|
||||
${CYAN}nvs${RST} [args] Generate NVS test matrix
|
||||
${CYAN}health${RST} <logfile> Check firmware health from QEMU log
|
||||
${CYAN}status${RST} Show installation status and versions
|
||||
${CYAN}help${RST} Show this help message
|
||||
|
||||
${BOLD}EXAMPLES${RST}
|
||||
qemu-cli.sh install # Install QEMU
|
||||
qemu-cli.sh test # Run basic firmware test
|
||||
qemu-cli.sh test --timeout 120 # Test with longer timeout
|
||||
qemu-cli.sh swarm --preset smoke # Quick swarm test
|
||||
qemu-cli.sh swarm --preset standard # Standard 3-node test
|
||||
qemu-cli.sh swarm --list-presets # List available presets
|
||||
qemu-cli.sh mesh 3 # 3-node mesh test
|
||||
qemu-cli.sh chaos # Run chaos tests
|
||||
qemu-cli.sh fuzz --duration 60 # Fuzz for 60 seconds
|
||||
qemu-cli.sh nvs --list # List NVS configs
|
||||
qemu-cli.sh health build/qemu_output.log
|
||||
qemu-cli.sh status # Show what's installed
|
||||
|
||||
${BOLD}TAB COMPLETION${RST}
|
||||
Source the completions in your shell:
|
||||
eval "\$(qemu-cli.sh --completions)"
|
||||
|
||||
${BOLD}ENVIRONMENT${RST}
|
||||
QEMU_PATH Path to qemu-system-xtensa binary (auto-detected)
|
||||
FUZZ_DURATION Override fuzz duration in seconds (default: 30)
|
||||
FUZZ_JOBS Parallel fuzzing jobs (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# --- Command: install ------------------------------------------------------
|
||||
cmd_install() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh install"
|
||||
echo "Install QEMU with Espressif ESP32-S3 support."
|
||||
return 0
|
||||
fi
|
||||
local installer="$SCRIPT_DIR/install-qemu.sh"
|
||||
if [[ -f "$installer" ]]; then
|
||||
info "Running install-qemu.sh ..."
|
||||
bash "$installer" "$@"
|
||||
else
|
||||
info "No install-qemu.sh found. Showing manual install steps."
|
||||
cat <<EOF
|
||||
|
||||
${BOLD}Manual QEMU ESP32-S3 installation:${RST}
|
||||
1. git clone https://github.com/espressif/qemu.git ~/.espressif/qemu-src
|
||||
2. cd ~/.espressif/qemu-src
|
||||
3. ./configure --target-list=xtensa-softmmu --prefix=\$HOME/.espressif/qemu/build \\
|
||||
--enable-gcrypt --disable-bsd-user --disable-docs
|
||||
4. make -j\$(nproc) && make install
|
||||
5. Add to PATH: export PATH="\$HOME/.espressif/qemu/build/bin:\$PATH"
|
||||
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Command: test ----------------------------------------------------------
|
||||
cmd_test() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh test [--timeout N] [extra args...]"
|
||||
echo "Run single-node QEMU ESP32-S3 firmware test."
|
||||
return 0
|
||||
fi
|
||||
need_qemu
|
||||
info "Running single-node firmware test ..."
|
||||
bash "$SCRIPT_DIR/qemu-esp32s3-test.sh" "$@"
|
||||
}
|
||||
|
||||
# --- Command: mesh ----------------------------------------------------------
|
||||
cmd_mesh() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh mesh [N] [extra args...]"
|
||||
echo "Run multi-node mesh test. N = number of nodes (default: 3)."
|
||||
return 0
|
||||
fi
|
||||
need_qemu
|
||||
local nodes="${1:-3}"
|
||||
shift 2>/dev/null || true
|
||||
info "Running ${nodes}-node mesh test ..."
|
||||
bash "$SCRIPT_DIR/qemu-mesh-test.sh" "$nodes" "$@"
|
||||
}
|
||||
|
||||
# --- Command: swarm ---------------------------------------------------------
|
||||
cmd_swarm() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh swarm [--preset NAME] [--list-presets] [args...]"
|
||||
echo "Run QEMU swarm configurator (qemu_swarm.py)."
|
||||
echo ""
|
||||
echo "Presets: smoke, standard, full, stress"
|
||||
echo "List: qemu-cli.sh swarm --list-presets"
|
||||
return 0
|
||||
fi
|
||||
need_qemu
|
||||
local py; py="$(detect_python)"
|
||||
info "Running swarm configurator ..."
|
||||
"$py" "$SCRIPT_DIR/qemu_swarm.py" "$@"
|
||||
}
|
||||
|
||||
# --- Command: snapshot ------------------------------------------------------
|
||||
cmd_snapshot() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh snapshot [args...]"
|
||||
echo "Run snapshot-based QEMU tests."
|
||||
return 0
|
||||
fi
|
||||
need_qemu
|
||||
info "Running snapshot tests ..."
|
||||
bash "$SCRIPT_DIR/qemu-snapshot-test.sh" "$@"
|
||||
}
|
||||
|
||||
# --- Command: chaos ---------------------------------------------------------
|
||||
cmd_chaos() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh chaos [args...]"
|
||||
echo "Run chaos / fault injection tests."
|
||||
return 0
|
||||
fi
|
||||
need_qemu
|
||||
info "Running chaos tests ..."
|
||||
bash "$SCRIPT_DIR/qemu-chaos-test.sh" "$@"
|
||||
}
|
||||
|
||||
# --- Command: fuzz ----------------------------------------------------------
|
||||
cmd_fuzz() {
|
||||
local duration="${FUZZ_DURATION:-30}"
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh fuzz [--duration N]"
|
||||
echo "Build and run all 3 fuzz targets (clang libFuzzer)."
|
||||
echo "Requires: clang with libFuzzer support."
|
||||
return 0
|
||||
fi
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--duration) duration="$2"; shift 2 ;;
|
||||
*) warn "Unknown fuzz option: $1"; shift ;;
|
||||
esac
|
||||
done
|
||||
if ! command -v clang >/dev/null 2>&1; then
|
||||
die "clang not found. Fuzz targets require clang with libFuzzer."
|
||||
fi
|
||||
info "Building and running fuzz targets (${duration}s each) ..."
|
||||
make -C "$FUZZ_DIR" run_all FUZZ_DURATION="$duration"
|
||||
ok "Fuzz testing complete."
|
||||
}
|
||||
|
||||
# --- Command: nvs -----------------------------------------------------------
|
||||
cmd_nvs() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh nvs [--list] [args...]"
|
||||
echo "Generate NVS test configuration matrix."
|
||||
return 0
|
||||
fi
|
||||
local py; py="$(detect_python)"
|
||||
info "Running NVS matrix generator ..."
|
||||
"$py" "$SCRIPT_DIR/generate_nvs_matrix.py" "$@"
|
||||
}
|
||||
|
||||
# --- Command: health --------------------------------------------------------
|
||||
cmd_health() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
echo "Usage: qemu-cli.sh health <logfile>"
|
||||
echo "Analyze firmware health from a QEMU output log."
|
||||
return 0
|
||||
fi
|
||||
local logfile="${1:-}"
|
||||
if [[ -z "$logfile" ]]; then
|
||||
die "Usage: qemu-cli.sh health <logfile>"
|
||||
fi
|
||||
if [[ ! -f "$logfile" ]]; then
|
||||
die "Log file not found: $logfile"
|
||||
fi
|
||||
local py; py="$(detect_python)"
|
||||
info "Analyzing health from: $logfile"
|
||||
"$py" "$SCRIPT_DIR/check_health.py" --log "$logfile" --after-fault manual
|
||||
}
|
||||
|
||||
# --- Command: status --------------------------------------------------------
|
||||
cmd_status() {
|
||||
# Status should never fail — disable errexit locally
|
||||
set +e
|
||||
echo -e "${BOLD}=== QEMU ESP32-S3 Testing Status ===${RST}"
|
||||
echo ""
|
||||
|
||||
# QEMU
|
||||
local qemu_bin
|
||||
qemu_bin="$(detect_qemu 2>/dev/null)"
|
||||
if [[ -n "$qemu_bin" ]]; then
|
||||
local qemu_ver
|
||||
qemu_ver="$("$qemu_bin" --version 2>/dev/null | head -1 || echo "unknown")"
|
||||
ok "QEMU: ${GREEN}installed${RST} ($qemu_ver)"
|
||||
echo " Path: $qemu_bin"
|
||||
else
|
||||
warn "QEMU: ${YELLOW}not found${RST} (run: qemu-cli.sh install)"
|
||||
fi
|
||||
|
||||
# ESP-IDF
|
||||
if [[ -n "${IDF_PATH:-}" ]] && [[ -d "$IDF_PATH" ]]; then
|
||||
ok "ESP-IDF: ${GREEN}available${RST} ($IDF_PATH)"
|
||||
else
|
||||
warn "ESP-IDF: ${YELLOW}IDF_PATH not set${RST}"
|
||||
fi
|
||||
|
||||
# Python
|
||||
local py; py="$(detect_python)"
|
||||
if command -v "$py" >/dev/null 2>&1; then
|
||||
ok "Python: ${GREEN}$("$py" --version 2>&1)${RST}"
|
||||
else
|
||||
warn "Python: ${YELLOW}not found${RST}"
|
||||
fi
|
||||
|
||||
# Clang (for fuzz)
|
||||
if command -v clang >/dev/null 2>&1; then
|
||||
ok "Clang: ${GREEN}$(clang --version 2>/dev/null | head -1)${RST}"
|
||||
else
|
||||
warn "Clang: ${YELLOW}not found${RST} (needed for fuzz targets only)"
|
||||
fi
|
||||
|
||||
# Firmware binary
|
||||
local fw_bin="$FIRMWARE_DIR/build/esp32-csi-node.bin"
|
||||
if [[ -f "$fw_bin" ]]; then
|
||||
local fw_size
|
||||
fw_size="$(stat -c%s "$fw_bin" 2>/dev/null || stat -f%z "$fw_bin" 2>/dev/null || echo "?")"
|
||||
ok "Firmware: ${GREEN}built${RST} ($fw_bin, ${fw_size} bytes)"
|
||||
else
|
||||
warn "Firmware: ${YELLOW}not built${RST} (expected at $fw_bin)"
|
||||
fi
|
||||
|
||||
# Swarm presets
|
||||
local preset_dir="$SCRIPT_DIR/swarm_presets"
|
||||
if [[ -d "$preset_dir" ]]; then
|
||||
local presets
|
||||
presets="$(ls "$preset_dir"/ 2>/dev/null | \
|
||||
sed 's/\.\(yaml\|json\)$//' | sort -u | tr '\n' ', ' | sed 's/,$//')"
|
||||
if [[ -n "$presets" ]]; then
|
||||
ok "Presets: ${GREEN}${presets}${RST}"
|
||||
else
|
||||
warn "Presets: ${YELLOW}none found${RST} in $preset_dir"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
set -e
|
||||
}
|
||||
|
||||
# --- Completions output -----------------------------------------------------
|
||||
print_completions() {
|
||||
cat <<'COMP'
|
||||
_qemu_cli_completions() {
|
||||
local cmds="install test mesh swarm snapshot chaos fuzz nvs health status help"
|
||||
local cur="${COMP_WORDS[COMP_CWORD]}"
|
||||
if [[ $COMP_CWORD -eq 1 ]]; then
|
||||
COMPREPLY=( $(compgen -W "$cmds" -- "$cur") )
|
||||
fi
|
||||
}
|
||||
complete -F _qemu_cli_completions qemu-cli.sh
|
||||
COMP
|
||||
}
|
||||
|
||||
# --- Main dispatch ----------------------------------------------------------
|
||||
main() {
|
||||
local cmd="${1:-help}"
|
||||
shift 2>/dev/null || true
|
||||
|
||||
case "$cmd" in
|
||||
install) cmd_install "$@" ;;
|
||||
test) cmd_test "$@" ;;
|
||||
mesh) cmd_mesh "$@" ;;
|
||||
swarm) cmd_swarm "$@" ;;
|
||||
snapshot) cmd_snapshot "$@" ;;
|
||||
chaos) cmd_chaos "$@" ;;
|
||||
fuzz) cmd_fuzz "$@" ;;
|
||||
nvs) cmd_nvs "$@" ;;
|
||||
health) cmd_health "$@" ;;
|
||||
status) cmd_status "$@" ;;
|
||||
help|-h|--help) cmd_help ;;
|
||||
--version) echo "qemu-cli.sh v${VERSION}" ;;
|
||||
--completions) print_completions ;;
|
||||
*)
|
||||
err "Unknown command: ${BOLD}${cmd}${RST}"
|
||||
echo ""
|
||||
cmd_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+212
@@ -0,0 +1,212 @@
|
||||
#!/bin/bash
|
||||
# QEMU ESP32-S3 Firmware Test Runner (ADR-061)
|
||||
#
|
||||
# Builds the firmware with mock CSI enabled, merges binaries into a single
|
||||
# flash image, optionally injects a pre-provisioned NVS partition, runs the
|
||||
# image under QEMU with a timeout, and validates the UART output.
|
||||
#
|
||||
# Environment variables:
|
||||
# QEMU_PATH - Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
# QEMU_TIMEOUT - Timeout in seconds (default: 60)
|
||||
# SKIP_BUILD - Set to "1" to skip the idf.py build step
|
||||
# NVS_BIN - Path to a pre-built NVS binary to inject (optional)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PASS — all checks passed
|
||||
# 1 WARN — non-critical checks failed
|
||||
# 2 FAIL — critical checks failed
|
||||
# 3 FATAL — build error, crash, or infrastructure failure
|
||||
|
||||
# ── Help ──────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<'HELP'
|
||||
Usage: qemu-esp32s3-test.sh [OPTIONS]
|
||||
|
||||
Build ESP32-S3 firmware with mock CSI, merge binaries into a single flash
|
||||
image, run under QEMU with a timeout, and validate the UART output.
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message and exit
|
||||
|
||||
Environment variables:
|
||||
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
QEMU_TIMEOUT Timeout in seconds (default: 60)
|
||||
SKIP_BUILD Set to "1" to skip idf.py build (default: unset)
|
||||
NVS_BIN Path to pre-built NVS binary (optional)
|
||||
QEMU_NET Set to "0" to disable networking (default: 1)
|
||||
|
||||
Examples:
|
||||
./qemu-esp32s3-test.sh
|
||||
SKIP_BUILD=1 ./qemu-esp32s3-test.sh
|
||||
QEMU_PATH=/opt/qemu/bin/qemu-system-xtensa QEMU_TIMEOUT=120 ./qemu-esp32s3-test.sh
|
||||
|
||||
Exit codes:
|
||||
0 PASS — all checks passed
|
||||
1 WARN — non-critical checks failed
|
||||
2 FAIL — critical checks failed
|
||||
3 FATAL — build error, crash, or infrastructure failure
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
case "${1:-}" in -h|--help) usage ;; esac
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
BUILD_DIR="$FIRMWARE_DIR/build"
|
||||
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
|
||||
FLASH_IMAGE="$BUILD_DIR/qemu_flash.bin"
|
||||
LOG_FILE="$BUILD_DIR/qemu_output.log"
|
||||
TIMEOUT_SEC="${QEMU_TIMEOUT:-60}"
|
||||
|
||||
echo "=== QEMU ESP32-S3 Firmware Test (ADR-061) ==="
|
||||
echo "Firmware dir: $FIRMWARE_DIR"
|
||||
echo "QEMU binary: $QEMU_BIN"
|
||||
echo "Timeout: ${TIMEOUT_SEC}s"
|
||||
echo ""
|
||||
|
||||
# ── Prerequisite checks ───────────────────────────────────────────────
|
||||
if ! command -v "$QEMU_BIN" &>/dev/null; then
|
||||
echo "ERROR: QEMU binary not found: $QEMU_BIN"
|
||||
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
|
||||
echo " Install: brew install qemu # macOS"
|
||||
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
echo "ERROR: python3 not found."
|
||||
echo " Install: sudo apt install python3 # Debian/Ubuntu"
|
||||
echo " Install: brew install python # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! python3 -m esptool version &>/dev/null 2>&1; then
|
||||
echo "ERROR: esptool not found (needed to merge flash binaries)."
|
||||
echo " Install: pip install esptool"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# ── SKIP_BUILD precheck ──────────────────────────────────────────────
|
||||
if [ "${SKIP_BUILD:-}" = "1" ] && [ ! -f "$BUILD_DIR/esp32-csi-node.bin" ]; then
|
||||
echo "ERROR: SKIP_BUILD=1 but flash image not found: $BUILD_DIR/esp32-csi-node.bin"
|
||||
echo "Build the firmware first: ./qemu-esp32s3-test.sh (without SKIP_BUILD)"
|
||||
echo "Or unset SKIP_BUILD to build automatically."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# 1. Build with mock CSI enabled (skip if already built)
|
||||
if [ "${SKIP_BUILD:-}" != "1" ]; then
|
||||
echo "[1/4] Building firmware (mock CSI mode)..."
|
||||
idf.py -C "$FIRMWARE_DIR" \
|
||||
-D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" \
|
||||
build
|
||||
echo ""
|
||||
else
|
||||
echo "[1/4] Skipping build (SKIP_BUILD=1)"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Verify build artifacts exist
|
||||
for artifact in \
|
||||
"$BUILD_DIR/bootloader/bootloader.bin" \
|
||||
"$BUILD_DIR/partition_table/partition-table.bin" \
|
||||
"$BUILD_DIR/esp32-csi-node.bin"; do
|
||||
if [ ! -f "$artifact" ]; then
|
||||
echo "ERROR: Build artifact not found: $artifact"
|
||||
echo "Run without SKIP_BUILD=1 or build the firmware first."
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
|
||||
# 2. Merge binaries into single flash image
|
||||
echo "[2/4] Creating merged flash image..."
|
||||
|
||||
# Check for ota_data_initial.bin; some builds don't produce it
|
||||
OTA_DATA_ARGS=""
|
||||
if [ -f "$BUILD_DIR/ota_data_initial.bin" ]; then
|
||||
OTA_DATA_ARGS="0xf000 $BUILD_DIR/ota_data_initial.bin"
|
||||
fi
|
||||
|
||||
python3 -m esptool --chip esp32s3 merge_bin -o "$FLASH_IMAGE" \
|
||||
--flash_mode dio --flash_freq 80m --flash_size 8MB \
|
||||
0x0 "$BUILD_DIR/bootloader/bootloader.bin" \
|
||||
0x8000 "$BUILD_DIR/partition_table/partition-table.bin" \
|
||||
$OTA_DATA_ARGS \
|
||||
0x20000 "$BUILD_DIR/esp32-csi-node.bin"
|
||||
|
||||
echo "Flash image: $FLASH_IMAGE ($(stat -c%s "$FLASH_IMAGE" 2>/dev/null || stat -f%z "$FLASH_IMAGE") bytes)"
|
||||
|
||||
# 2b. Optionally inject pre-provisioned NVS partition
|
||||
NVS_FILE="${NVS_BIN:-$BUILD_DIR/nvs_test.bin}"
|
||||
if [ -f "$NVS_FILE" ]; then
|
||||
echo "[2b] Injecting NVS partition from: $NVS_FILE"
|
||||
# NVS partition offset = 0x9000 = 36864
|
||||
dd if="$NVS_FILE" of="$FLASH_IMAGE" \
|
||||
bs=1 seek=$((0x9000)) conv=notrunc 2>/dev/null
|
||||
echo "NVS injected ($(stat -c%s "$NVS_FILE" 2>/dev/null || stat -f%z "$NVS_FILE") bytes at 0x9000)"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 3. Run in QEMU with timeout, capture UART output
|
||||
echo "[3/4] Running QEMU (timeout: ${TIMEOUT_SEC}s)..."
|
||||
echo "------- QEMU UART output -------"
|
||||
|
||||
# Use timeout command; fall back to gtimeout on macOS
|
||||
TIMEOUT_CMD="timeout"
|
||||
if ! command -v timeout &>/dev/null; then
|
||||
if command -v gtimeout &>/dev/null; then
|
||||
TIMEOUT_CMD="gtimeout"
|
||||
else
|
||||
echo "WARNING: 'timeout' command not found. QEMU may run indefinitely."
|
||||
TIMEOUT_CMD=""
|
||||
fi
|
||||
fi
|
||||
|
||||
QEMU_EXIT=0
|
||||
|
||||
# Common QEMU arguments
|
||||
QEMU_ARGS=(
|
||||
-machine esp32s3
|
||||
-nographic
|
||||
-drive "file=$FLASH_IMAGE,if=mtd,format=raw"
|
||||
-serial mon:stdio
|
||||
-no-reboot
|
||||
)
|
||||
|
||||
# Enable SLIRP user-mode networking for UDP if available
|
||||
if [ "${QEMU_NET:-1}" != "0" ]; then
|
||||
QEMU_ARGS+=(-nic "user,model=open_eth,net=10.0.2.0/24,host=10.0.2.2")
|
||||
fi
|
||||
|
||||
if [ -n "$TIMEOUT_CMD" ]; then
|
||||
$TIMEOUT_CMD "$TIMEOUT_SEC" "$QEMU_BIN" "${QEMU_ARGS[@]}" \
|
||||
2>&1 | tee "$LOG_FILE" || QEMU_EXIT=$?
|
||||
else
|
||||
"$QEMU_BIN" "${QEMU_ARGS[@]}" \
|
||||
2>&1 | tee "$LOG_FILE" || QEMU_EXIT=$?
|
||||
fi
|
||||
|
||||
echo "------- End QEMU output -------"
|
||||
echo ""
|
||||
|
||||
# timeout returns 124 when the process is killed by timeout — that's expected
|
||||
if [ "$QEMU_EXIT" -eq 124 ]; then
|
||||
echo "QEMU exited via timeout (expected for firmware that loops forever)."
|
||||
elif [ "$QEMU_EXIT" -ne 0 ]; then
|
||||
echo "WARNING: QEMU exited with code $QEMU_EXIT"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 4. Validate expected output
|
||||
echo "[4/4] Validating output..."
|
||||
python3 "$SCRIPT_DIR/validate_qemu_output.py" "$LOG_FILE"
|
||||
VALIDATE_EXIT=$?
|
||||
|
||||
echo ""
|
||||
echo "=== Test Complete (exit code: $VALIDATE_EXIT) ==="
|
||||
exit $VALIDATE_EXIT
|
||||
@@ -0,0 +1,414 @@
|
||||
#!/bin/bash
|
||||
# QEMU ESP32-S3 Multi-Node Mesh Simulation (ADR-061 Layer 3)
|
||||
#
|
||||
# Spawns N ESP32-S3 QEMU instances connected via a Linux bridge, each with
|
||||
# unique NVS provisioning (node ID, TDM slot), and a Rust aggregator that
|
||||
# collects frames from all nodes. After a configurable timeout the script
|
||||
# tears everything down and runs validate_mesh_test.py.
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./qemu-mesh-test.sh [N_NODES]
|
||||
#
|
||||
# Environment variables:
|
||||
# QEMU_PATH - Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
# QEMU_TIMEOUT - Timeout in seconds (default: 45)
|
||||
# MESH_TIMEOUT - Deprecated alias for QEMU_TIMEOUT
|
||||
# SKIP_BUILD - Set to "1" to skip the idf.py build step
|
||||
# BRIDGE_NAME - Bridge interface name (default: qemu-br0)
|
||||
# BRIDGE_SUBNET - Bridge IP/mask (default: 10.0.0.1/24)
|
||||
# AGGREGATOR_PORT - UDP port the aggregator listens on (default: 5005)
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Linux with bridge-utils and iproute2
|
||||
# - QEMU with ESP32-S3 machine support (qemu-system-xtensa)
|
||||
# - provision.py capable of --dry-run NVS generation
|
||||
# - Rust workspace with wifi-densepose-hardware crate (aggregator binary)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PASS — all checks passed
|
||||
# 1 WARN — non-critical checks failed
|
||||
# 2 FAIL — critical checks failed
|
||||
# 3 FATAL — build error, crash, or infrastructure failure
|
||||
|
||||
# ── Help ──────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<'HELP'
|
||||
Usage: sudo ./qemu-mesh-test.sh [OPTIONS] [N_NODES]
|
||||
|
||||
Spawn N ESP32-S3 QEMU instances connected via a Linux bridge, each with
|
||||
unique NVS provisioning (node ID, TDM slot), and a Rust aggregator that
|
||||
collects frames from all nodes.
|
||||
|
||||
NOTE: Requires root/sudo for TAP/bridge creation.
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message and exit
|
||||
|
||||
Positional:
|
||||
N_NODES Number of mesh nodes (default: 3, minimum: 2)
|
||||
|
||||
Environment variables:
|
||||
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
QEMU_TIMEOUT Timeout in seconds (default: 45)
|
||||
MESH_TIMEOUT Alias for QEMU_TIMEOUT (deprecated)(default: 45)
|
||||
SKIP_BUILD Set to "1" to skip idf.py build (default: unset)
|
||||
BRIDGE_NAME Bridge interface name (default: qemu-br0)
|
||||
BRIDGE_SUBNET Bridge IP/mask (default: 10.0.0.1/24)
|
||||
AGGREGATOR_PORT UDP port for aggregator (default: 5005)
|
||||
|
||||
Examples:
|
||||
sudo ./qemu-mesh-test.sh
|
||||
sudo QEMU_TIMEOUT=90 ./qemu-mesh-test.sh 5
|
||||
sudo SKIP_BUILD=1 ./qemu-mesh-test.sh 4
|
||||
|
||||
Exit codes:
|
||||
0 PASS — all checks passed
|
||||
1 WARN — non-critical checks failed
|
||||
2 FAIL — critical checks failed
|
||||
3 FATAL — build error, crash, or infrastructure failure
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
case "${1:-}" in -h|--help) usage ;; esac
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
BUILD_DIR="$FIRMWARE_DIR/build"
|
||||
RUST_DIR="$PROJECT_ROOT/rust-port/wifi-densepose-rs"
|
||||
PROVISION_SCRIPT="$FIRMWARE_DIR/provision.py"
|
||||
VALIDATE_SCRIPT="$SCRIPT_DIR/validate_mesh_test.py"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
N_NODES="${1:-3}"
|
||||
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
|
||||
TIMEOUT="${QEMU_TIMEOUT:-${MESH_TIMEOUT:-45}}"
|
||||
BRIDGE="${BRIDGE_NAME:-qemu-br0}"
|
||||
BRIDGE_IP="${BRIDGE_SUBNET:-10.0.0.1/24}"
|
||||
AGG_PORT="${AGGREGATOR_PORT:-5005}"
|
||||
RESULTS_FILE="$BUILD_DIR/mesh_test_results.json"
|
||||
|
||||
echo "=== QEMU Multi-Node Mesh Test (ADR-061 Layer 3) ==="
|
||||
echo "Nodes: $N_NODES"
|
||||
echo "Bridge: $BRIDGE ($BRIDGE_IP)"
|
||||
echo "Aggregator: 0.0.0.0:$AGG_PORT"
|
||||
echo "QEMU binary: $QEMU_BIN"
|
||||
echo "Timeout: ${TIMEOUT}s"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preflight checks
|
||||
# ---------------------------------------------------------------------------
|
||||
if [ "$N_NODES" -lt 2 ]; then
|
||||
echo "ERROR: Need at least 2 nodes for mesh simulation (got $N_NODES)"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v "$QEMU_BIN" &>/dev/null; then
|
||||
echo "ERROR: QEMU binary not found: $QEMU_BIN"
|
||||
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
|
||||
echo " Install: brew install qemu # macOS"
|
||||
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
echo "ERROR: python3 not found."
|
||||
echo " Install: sudo apt install python3 # Debian/Ubuntu"
|
||||
echo " Install: brew install python # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v ip &>/dev/null; then
|
||||
echo "ERROR: 'ip' command not found."
|
||||
echo " Install: sudo apt install iproute2 # Debian/Ubuntu"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v brctl &>/dev/null && ! ip link help bridge &>/dev/null 2>&1; then
|
||||
echo "WARNING: bridge-utils not found; will use 'ip link' for bridge creation."
|
||||
fi
|
||||
|
||||
if command -v socat &>/dev/null; then
|
||||
true # optional, available
|
||||
else
|
||||
echo "NOTE: socat not found (optional, used for advanced monitor communication)."
|
||||
echo " Install: sudo apt install socat # Debian/Ubuntu"
|
||||
echo " Install: brew install socat # macOS"
|
||||
fi
|
||||
|
||||
if ! command -v cargo &>/dev/null; then
|
||||
echo "ERROR: cargo not found (needed to build the Rust aggregator)."
|
||||
echo " Install: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "ERROR: This script must be run as root (for TAP/bridge creation)."
|
||||
echo "Usage: sudo $0 [N_NODES]"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup trap — runs on EXIT regardless of success/failure
|
||||
# ---------------------------------------------------------------------------
|
||||
QEMU_PIDS=()
|
||||
AGG_PID=""
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "--- Cleaning up ---"
|
||||
|
||||
# Kill QEMU instances
|
||||
for pid in "${QEMU_PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Kill aggregator
|
||||
if [ -n "$AGG_PID" ] && kill -0 "$AGG_PID" 2>/dev/null; then
|
||||
kill "$AGG_PID" 2>/dev/null || true
|
||||
wait "$AGG_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Tear down TAP interfaces and bridge
|
||||
for i in $(seq 0 $((N_NODES - 1))); do
|
||||
local tap="tap${i}"
|
||||
if ip link show "$tap" &>/dev/null; then
|
||||
ip link set "$tap" down 2>/dev/null || true
|
||||
ip link delete "$tap" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
if ip link show "$BRIDGE" &>/dev/null; then
|
||||
ip link set "$BRIDGE" down 2>/dev/null || true
|
||||
ip link delete "$BRIDGE" type bridge 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "Cleanup complete."
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Build flash image (if not already built)
|
||||
# ---------------------------------------------------------------------------
|
||||
if [ "${SKIP_BUILD:-}" != "1" ]; then
|
||||
echo "[1/6] Building firmware (mock CSI + QEMU overlay)..."
|
||||
idf.py -C "$FIRMWARE_DIR" \
|
||||
-D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" \
|
||||
build
|
||||
echo ""
|
||||
else
|
||||
echo "[1/6] Skipping build (SKIP_BUILD=1)"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Verify build artifacts
|
||||
FLASH_IMAGE_BASE="$BUILD_DIR/qemu_flash_base.bin"
|
||||
for artifact in \
|
||||
"$BUILD_DIR/bootloader/bootloader.bin" \
|
||||
"$BUILD_DIR/partition_table/partition-table.bin" \
|
||||
"$BUILD_DIR/esp32-csi-node.bin"; do
|
||||
if [ ! -f "$artifact" ]; then
|
||||
echo "ERROR: Build artifact not found: $artifact"
|
||||
echo "Run without SKIP_BUILD=1 or build the firmware first."
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
|
||||
# Merge into base flash image
|
||||
echo "[2/6] Creating base flash image..."
|
||||
OTA_DATA_ARGS=""
|
||||
if [ -f "$BUILD_DIR/ota_data_initial.bin" ]; then
|
||||
OTA_DATA_ARGS="0xf000 $BUILD_DIR/ota_data_initial.bin"
|
||||
fi
|
||||
|
||||
python3 -m esptool --chip esp32s3 merge_bin -o "$FLASH_IMAGE_BASE" \
|
||||
--flash_mode dio --flash_freq 80m --flash_size 8MB \
|
||||
0x0 "$BUILD_DIR/bootloader/bootloader.bin" \
|
||||
0x8000 "$BUILD_DIR/partition_table/partition-table.bin" \
|
||||
$OTA_DATA_ARGS \
|
||||
0x20000 "$BUILD_DIR/esp32-csi-node.bin"
|
||||
|
||||
echo "Base flash image: $FLASH_IMAGE_BASE ($(stat -c%s "$FLASH_IMAGE_BASE" 2>/dev/null || stat -f%z "$FLASH_IMAGE_BASE") bytes)"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Generate per-node NVS and flash images
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[3/6] Generating per-node NVS images..."
|
||||
|
||||
# Extract the aggregator IP from the bridge subnet (first host)
|
||||
AGG_IP="${BRIDGE_IP%%/*}"
|
||||
|
||||
for i in $(seq 0 $((N_NODES - 1))); do
|
||||
NVS_BIN="$BUILD_DIR/nvs_node${i}.bin"
|
||||
NODE_FLASH="$BUILD_DIR/qemu_flash_node${i}.bin"
|
||||
|
||||
# Generate NVS with provision.py --dry-run
|
||||
# --port is required by argparse but unused in dry-run; pass a dummy
|
||||
python3 "$PROVISION_SCRIPT" \
|
||||
--port /dev/null \
|
||||
--dry-run \
|
||||
--node-id "$i" \
|
||||
--tdm-slot "$i" \
|
||||
--tdm-total "$N_NODES" \
|
||||
--target-ip "$AGG_IP" \
|
||||
--target-port "$AGG_PORT"
|
||||
|
||||
# provision.py --dry-run writes to nvs_provision.bin in CWD
|
||||
if [ -f "nvs_provision.bin" ]; then
|
||||
mv "nvs_provision.bin" "$NVS_BIN"
|
||||
else
|
||||
echo "ERROR: provision.py did not produce nvs_provision.bin for node $i"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Copy base image and inject NVS at 0x9000
|
||||
cp "$FLASH_IMAGE_BASE" "$NODE_FLASH"
|
||||
dd if="$NVS_BIN" of="$NODE_FLASH" \
|
||||
bs=1 seek=$((0x9000)) conv=notrunc 2>/dev/null
|
||||
|
||||
echo " Node $i: flash=$NODE_FLASH nvs=$NVS_BIN (TDM slot $i/$N_NODES)"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Create bridge and TAP interfaces
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[4/6] Setting up network bridge and TAP interfaces..."
|
||||
|
||||
# Create bridge
|
||||
ip link add name "$BRIDGE" type bridge 2>/dev/null || true
|
||||
ip addr add "$BRIDGE_IP" dev "$BRIDGE" 2>/dev/null || true
|
||||
ip link set "$BRIDGE" up
|
||||
|
||||
# Create TAP interfaces and attach to bridge
|
||||
for i in $(seq 0 $((N_NODES - 1))); do
|
||||
TAP="tap${i}"
|
||||
ip tuntap add dev "$TAP" mode tap 2>/dev/null || true
|
||||
ip link set "$TAP" master "$BRIDGE"
|
||||
ip link set "$TAP" up
|
||||
echo " $TAP -> $BRIDGE"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Start aggregator and QEMU instances
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[5/6] Starting aggregator and $N_NODES QEMU nodes..."
|
||||
|
||||
# Start Rust aggregator in background
|
||||
echo " Starting aggregator: listen=0.0.0.0:$AGG_PORT expect-nodes=$N_NODES"
|
||||
cargo run --manifest-path "$RUST_DIR/Cargo.toml" \
|
||||
-p wifi-densepose-hardware --bin aggregator -- \
|
||||
--listen "0.0.0.0:$AGG_PORT" \
|
||||
--expect-nodes "$N_NODES" \
|
||||
--output "$RESULTS_FILE" \
|
||||
> "$BUILD_DIR/aggregator.log" 2>&1 &
|
||||
AGG_PID=$!
|
||||
echo " Aggregator PID: $AGG_PID"
|
||||
|
||||
# Give aggregator a moment to bind
|
||||
sleep 1
|
||||
|
||||
if ! kill -0 "$AGG_PID" 2>/dev/null; then
|
||||
echo "ERROR: Aggregator failed to start. Check $BUILD_DIR/aggregator.log"
|
||||
cat "$BUILD_DIR/aggregator.log" 2>/dev/null || true
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Launch QEMU instances
|
||||
for i in $(seq 0 $((N_NODES - 1))); do
|
||||
TAP="tap${i}"
|
||||
NODE_FLASH="$BUILD_DIR/qemu_flash_node${i}.bin"
|
||||
NODE_LOG="$BUILD_DIR/qemu_node${i}.log"
|
||||
NODE_MAC=$(printf "52:54:00:00:00:%02x" "$i")
|
||||
|
||||
echo " Starting QEMU node $i (tap=$TAP, mac=$NODE_MAC)..."
|
||||
|
||||
"$QEMU_BIN" \
|
||||
-machine esp32s3 \
|
||||
-nographic \
|
||||
-drive "file=$NODE_FLASH,if=mtd,format=raw" \
|
||||
-serial "file:$NODE_LOG" \
|
||||
-no-reboot \
|
||||
-nic "tap,ifname=$TAP,script=no,downscript=no,mac=$NODE_MAC" \
|
||||
> /dev/null 2>&1 &
|
||||
|
||||
QEMU_PIDS+=($!)
|
||||
echo " PID: ${QEMU_PIDS[-1]}, log: $NODE_LOG"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "All nodes launched. Waiting ${TIMEOUT}s for mesh simulation..."
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wait for timeout
|
||||
# ---------------------------------------------------------------------------
|
||||
sleep "$TIMEOUT"
|
||||
|
||||
echo "Timeout reached. Stopping all processes..."
|
||||
|
||||
# Kill QEMU instances (aggregator killed in cleanup)
|
||||
for pid in "${QEMU_PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Give aggregator a moment to flush results
|
||||
sleep 2
|
||||
|
||||
# Kill aggregator
|
||||
if [ -n "$AGG_PID" ] && kill -0 "$AGG_PID" 2>/dev/null; then
|
||||
kill "$AGG_PID" 2>/dev/null || true
|
||||
wait "$AGG_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. Validate results
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[6/6] Validating mesh test results..."
|
||||
|
||||
VALIDATE_ARGS=("--nodes" "$N_NODES")
|
||||
|
||||
# Pass results file if it was produced
|
||||
if [ -f "$RESULTS_FILE" ]; then
|
||||
VALIDATE_ARGS+=("--results" "$RESULTS_FILE")
|
||||
else
|
||||
echo "WARNING: Aggregator results file not found: $RESULTS_FILE"
|
||||
echo "Validation will rely on node logs only."
|
||||
fi
|
||||
|
||||
# Pass node log files
|
||||
for i in $(seq 0 $((N_NODES - 1))); do
|
||||
NODE_LOG="$BUILD_DIR/qemu_node${i}.log"
|
||||
if [ -f "$NODE_LOG" ]; then
|
||||
VALIDATE_ARGS+=("--log" "$NODE_LOG")
|
||||
fi
|
||||
done
|
||||
|
||||
python3 "$VALIDATE_SCRIPT" "${VALIDATE_ARGS[@]}"
|
||||
VALIDATE_EXIT=$?
|
||||
|
||||
echo ""
|
||||
echo "=== Mesh Test Complete (exit code: $VALIDATE_EXIT) ==="
|
||||
exit $VALIDATE_EXIT
|
||||
Executable
+373
@@ -0,0 +1,373 @@
|
||||
#!/bin/bash
|
||||
# QEMU Snapshot-Based Test Runner — ADR-061 Layer 8
|
||||
#
|
||||
# Uses QEMU VM snapshots to accelerate repeated test runs.
|
||||
# Instead of rebooting and re-initializing for each test scenario,
|
||||
# we snapshot the VM state after boot and after the first CSI frame,
|
||||
# then restore from the snapshot for each individual test.
|
||||
#
|
||||
# This dramatically reduces per-test wall time from ~15s (full boot)
|
||||
# to ~2s (snapshot restore + execution).
|
||||
#
|
||||
# Environment variables:
|
||||
# QEMU_PATH - Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
# QEMU_TIMEOUT - Per-test timeout in seconds (default: 10)
|
||||
# FLASH_IMAGE - Path to merged flash image (default: build/qemu_flash.bin)
|
||||
# SKIP_SNAPSHOT - Set to "1" to run without snapshots (baseline timing)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PASS — all checks passed
|
||||
# 1 WARN — non-critical checks failed
|
||||
# 2 FAIL — critical checks failed
|
||||
# 3 FATAL — build error, crash, or infrastructure failure
|
||||
|
||||
# ── Help ──────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<'HELP'
|
||||
Usage: qemu-snapshot-test.sh [OPTIONS]
|
||||
|
||||
Use QEMU VM snapshots to accelerate repeated test runs. Snapshots the VM
|
||||
state after boot and after the first CSI frame, then restores from the
|
||||
snapshot for each individual test (~2s vs ~15s per test).
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message and exit
|
||||
|
||||
Environment variables:
|
||||
QEMU_PATH Path to qemu-system-xtensa (default: qemu-system-xtensa)
|
||||
QEMU_TIMEOUT Per-test timeout in seconds (default: 10)
|
||||
FLASH_IMAGE Path to merged flash image (default: build/qemu_flash.bin)
|
||||
SKIP_SNAPSHOT Set to "1" to run without snapshots (baseline timing)
|
||||
|
||||
Examples:
|
||||
./qemu-snapshot-test.sh
|
||||
QEMU_TIMEOUT=20 ./qemu-snapshot-test.sh
|
||||
FLASH_IMAGE=/path/to/image.bin ./qemu-snapshot-test.sh
|
||||
|
||||
Exit codes:
|
||||
0 PASS — all checks passed
|
||||
1 WARN — non-critical checks failed
|
||||
2 FAIL — critical checks failed
|
||||
3 FATAL — build error, crash, or infrastructure failure
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
case "${1:-}" in -h|--help) usage ;; esac
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
FIRMWARE_DIR="$PROJECT_ROOT/firmware/esp32-csi-node"
|
||||
BUILD_DIR="$FIRMWARE_DIR/build"
|
||||
QEMU_BIN="${QEMU_PATH:-qemu-system-xtensa}"
|
||||
FLASH_IMAGE="${FLASH_IMAGE:-$BUILD_DIR/qemu_flash.bin}"
|
||||
TIMEOUT_SEC="${QEMU_TIMEOUT:-10}"
|
||||
MONITOR_SOCK="$BUILD_DIR/qemu-monitor.sock"
|
||||
LOG_DIR="$BUILD_DIR/snapshot-tests"
|
||||
QEMU_PID=""
|
||||
|
||||
# Timing accumulators
|
||||
SNAPSHOT_TOTAL_MS=0
|
||||
BASELINE_TOTAL_MS=0
|
||||
|
||||
# Track test results: array of "test_name:exit_code"
|
||||
declare -a TEST_RESULTS=()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Cleanup
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "[cleanup] Shutting down QEMU and removing socket..."
|
||||
if [ -n "$QEMU_PID" ] && kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
kill "$QEMU_PID" 2>/dev/null || true
|
||||
wait "$QEMU_PID" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$MONITOR_SOCK"
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
now_ms() {
|
||||
# Millisecond timestamp (portable: Linux date +%s%N, macOS perl fallback)
|
||||
local ns
|
||||
ns=$(date +%s%N 2>/dev/null)
|
||||
if [[ "$ns" =~ ^[0-9]+$ ]]; then
|
||||
echo $(( ns / 1000000 ))
|
||||
else
|
||||
perl -MTime::HiRes=time -e 'printf "%d\n", time()*1000' 2>/dev/null || \
|
||||
echo $(( $(date +%s) * 1000 ))
|
||||
fi
|
||||
}
|
||||
|
||||
monitor_cmd() {
|
||||
# Send a command to QEMU monitor via socat and capture response
|
||||
local cmd="$1"
|
||||
local timeout="${2:-5}"
|
||||
if ! command -v socat &>/dev/null; then
|
||||
echo "ERROR: socat not found (required for QEMU monitor)" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "$cmd" | socat - "UNIX-CONNECT:$MONITOR_SOCK,connect-timeout=$timeout" 2>/dev/null
|
||||
}
|
||||
|
||||
wait_for_pattern() {
|
||||
# Wait until a pattern appears in the log file, or timeout
|
||||
local log_file="$1"
|
||||
local pattern="$2"
|
||||
local timeout="$3"
|
||||
local elapsed=0
|
||||
while [ "$elapsed" -lt "$timeout" ]; do
|
||||
if [ -f "$log_file" ] && grep -q "$pattern" "$log_file" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
start_qemu() {
|
||||
# Launch QEMU in background with monitor socket
|
||||
echo "[qemu] Launching QEMU with monitor socket..."
|
||||
|
||||
rm -f "$MONITOR_SOCK"
|
||||
|
||||
local qemu_args=(
|
||||
-machine esp32s3
|
||||
-nographic
|
||||
-drive "file=$FLASH_IMAGE,if=mtd,format=raw"
|
||||
-serial "file:$LOG_DIR/qemu_uart.log"
|
||||
-no-reboot
|
||||
-monitor "unix:$MONITOR_SOCK,server,nowait"
|
||||
)
|
||||
|
||||
"$QEMU_BIN" "${qemu_args[@]}" &
|
||||
QEMU_PID=$!
|
||||
echo "[qemu] PID=$QEMU_PID"
|
||||
|
||||
# Wait for monitor socket to appear
|
||||
local waited=0
|
||||
while [ ! -S "$MONITOR_SOCK" ] && [ "$waited" -lt 10 ]; do
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
|
||||
if [ ! -S "$MONITOR_SOCK" ]; then
|
||||
echo "ERROR: QEMU monitor socket did not appear after 10s"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Verify QEMU is still running
|
||||
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||
echo "ERROR: QEMU process exited prematurely"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "[qemu] Monitor socket ready: $MONITOR_SOCK"
|
||||
}
|
||||
|
||||
save_snapshot() {
|
||||
local name="$1"
|
||||
echo "[snapshot] Saving snapshot: $name"
|
||||
monitor_cmd "savevm $name" 5
|
||||
echo "[snapshot] Saved: $name"
|
||||
}
|
||||
|
||||
restore_snapshot() {
|
||||
local name="$1"
|
||||
echo "[snapshot] Restoring snapshot: $name"
|
||||
monitor_cmd "loadvm $name" 5
|
||||
echo "[snapshot] Restored: $name"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Pre-flight checks
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "=== QEMU Snapshot Test Runner — ADR-061 Layer 8 ==="
|
||||
echo "QEMU binary: $QEMU_BIN"
|
||||
echo "Flash image: $FLASH_IMAGE"
|
||||
echo "Timeout/test: ${TIMEOUT_SEC}s"
|
||||
echo ""
|
||||
|
||||
if ! command -v "$QEMU_BIN" &>/dev/null; then
|
||||
echo "ERROR: QEMU binary not found: $QEMU_BIN"
|
||||
echo " Install: sudo apt install qemu-system-misc # Debian/Ubuntu"
|
||||
echo " Install: brew install qemu # macOS"
|
||||
echo " Or set QEMU_PATH to the qemu-system-xtensa binary."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v qemu-img &>/dev/null; then
|
||||
echo "ERROR: qemu-img not found (needed for snapshot disk management)."
|
||||
echo " Install: sudo apt install qemu-utils # Debian/Ubuntu"
|
||||
echo " Install: brew install qemu # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if ! command -v socat &>/dev/null; then
|
||||
echo "ERROR: socat not found (needed for QEMU monitor communication)."
|
||||
echo " Install: sudo apt install socat # Debian/Ubuntu"
|
||||
echo " Install: brew install socat # macOS"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ ! -f "$FLASH_IMAGE" ]; then
|
||||
echo "ERROR: Flash image not found: $FLASH_IMAGE"
|
||||
echo "Run qemu-esp32s3-test.sh first to build the flash image."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Phase 1: Boot and create snapshots
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Phase 1: Boot and snapshot creation ──"
|
||||
echo ""
|
||||
|
||||
# Clear any previous UART log
|
||||
> "$LOG_DIR/qemu_uart.log"
|
||||
|
||||
start_qemu
|
||||
|
||||
# Wait for boot (look for boot indicators, max 5s)
|
||||
echo "[boot] Waiting for firmware boot (up to 5s)..."
|
||||
if wait_for_pattern "$LOG_DIR/qemu_uart.log" "app_main\|main_task\|ESP32-S3" 5; then
|
||||
echo "[boot] Firmware booted successfully."
|
||||
else
|
||||
echo "[boot] No boot indicator found after 5s (continuing anyway)."
|
||||
fi
|
||||
|
||||
# Save post-boot snapshot
|
||||
save_snapshot "post_boot"
|
||||
echo ""
|
||||
|
||||
# Wait for first mock CSI frame (additional 5s)
|
||||
echo "[frame] Waiting for first CSI frame (up to 5s)..."
|
||||
if wait_for_pattern "$LOG_DIR/qemu_uart.log" "frame\|CSI\|mock_csi\|iq_data\|subcarrier" 5; then
|
||||
echo "[frame] First CSI frame detected."
|
||||
else
|
||||
echo "[frame] No frame indicator found after 5s (continuing anyway)."
|
||||
fi
|
||||
|
||||
# Save post-first-frame snapshot
|
||||
save_snapshot "post_first_frame"
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Phase 2: Run tests from snapshot
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Phase 2: Running tests from snapshot ──"
|
||||
echo ""
|
||||
|
||||
TESTS=("test_presence" "test_fall" "test_multi_person")
|
||||
MAX_EXIT=0
|
||||
|
||||
for test_name in "${TESTS[@]}"; do
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Test: $test_name"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
test_log="$LOG_DIR/${test_name}.log"
|
||||
t_start=$(now_ms)
|
||||
|
||||
# Restore to post_first_frame state
|
||||
restore_snapshot "post_first_frame"
|
||||
|
||||
# Record current log length so we can extract only new lines
|
||||
pre_lines=$(wc -l < "$LOG_DIR/qemu_uart.log" 2>/dev/null || echo 0)
|
||||
|
||||
# Let execution continue for TIMEOUT_SEC seconds
|
||||
echo "[test] Running for ${TIMEOUT_SEC}s..."
|
||||
sleep "$TIMEOUT_SEC"
|
||||
|
||||
# Capture only the new log lines produced during this test
|
||||
tail -n +$((pre_lines + 1)) "$LOG_DIR/qemu_uart.log" > "$test_log"
|
||||
|
||||
t_end=$(now_ms)
|
||||
elapsed_ms=$((t_end - t_start))
|
||||
SNAPSHOT_TOTAL_MS=$((SNAPSHOT_TOTAL_MS + elapsed_ms))
|
||||
|
||||
echo "[test] Captured $(wc -l < "$test_log") lines in ${elapsed_ms}ms"
|
||||
|
||||
# Validate
|
||||
echo "[test] Validating..."
|
||||
test_exit=0
|
||||
python3 "$SCRIPT_DIR/validate_qemu_output.py" "$test_log" || test_exit=$?
|
||||
|
||||
TEST_RESULTS+=("${test_name}:${test_exit}")
|
||||
if [ "$test_exit" -gt "$MAX_EXIT" ]; then
|
||||
MAX_EXIT=$test_exit
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Phase 3: Baseline timing (without snapshots) for comparison
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Phase 3: Timing comparison ──"
|
||||
echo ""
|
||||
|
||||
# Estimate baseline: full boot (5s) + frame wait (5s) + test run per test
|
||||
BASELINE_PER_TEST=$((5 + 5 + TIMEOUT_SEC))
|
||||
BASELINE_TOTAL_MS=$((BASELINE_PER_TEST * ${#TESTS[@]} * 1000))
|
||||
SNAPSHOT_PER_TEST=$((SNAPSHOT_TOTAL_MS / ${#TESTS[@]}))
|
||||
|
||||
echo "Timing Summary:"
|
||||
echo " Tests run: ${#TESTS[@]}"
|
||||
echo " With snapshots:"
|
||||
echo " Total wall time: ${SNAPSHOT_TOTAL_MS}ms"
|
||||
echo " Per-test average: ${SNAPSHOT_PER_TEST}ms"
|
||||
echo " Without snapshots (estimated):"
|
||||
echo " Total wall time: ${BASELINE_TOTAL_MS}ms"
|
||||
echo " Per-test average: $((BASELINE_PER_TEST * 1000))ms"
|
||||
echo ""
|
||||
|
||||
if [ "$SNAPSHOT_TOTAL_MS" -gt 0 ] && [ "$BASELINE_TOTAL_MS" -gt 0 ]; then
|
||||
SPEEDUP=$((BASELINE_TOTAL_MS * 100 / SNAPSHOT_TOTAL_MS))
|
||||
echo " Speedup: ${SPEEDUP}% (${SPEEDUP}x/100)"
|
||||
else
|
||||
echo " Speedup: N/A (insufficient data)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Summary
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "── Test Results Summary ──"
|
||||
echo ""
|
||||
PASS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
for result in "${TEST_RESULTS[@]}"; do
|
||||
name="${result%%:*}"
|
||||
code="${result##*:}"
|
||||
if [ "$code" -le 1 ]; then
|
||||
echo " [PASS] $name (exit=$code)"
|
||||
PASS_COUNT=$((PASS_COUNT + 1))
|
||||
else
|
||||
echo " [FAIL] $name (exit=$code)"
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " $PASS_COUNT passed, $FAIL_COUNT failed out of ${#TESTS[@]} tests"
|
||||
echo ""
|
||||
echo "=== Snapshot Test Complete (exit code: $MAX_EXIT) ==="
|
||||
exit "$MAX_EXIT"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,671 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
QEMU Swarm Health Oracle (ADR-062)
|
||||
|
||||
Validates collective health of a multi-node ESP32-S3 QEMU swarm.
|
||||
Checks cross-node assertions like TDM ordering, inter-node communication,
|
||||
and swarm-level frame rates.
|
||||
|
||||
Usage:
|
||||
python3 swarm_health.py --config swarm_config.yaml --log-dir build/swarm_logs/
|
||||
python3 swarm_health.py --log-dir build/swarm_logs/ --assertions all_nodes_boot no_crashes
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None # type: ignore[assignment]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ANSI helpers (disabled when not a TTY)
|
||||
# ---------------------------------------------------------------------------
|
||||
USE_COLOR = sys.stdout.isatty()
|
||||
|
||||
|
||||
def _color(text: str, code: str) -> str:
|
||||
return f"\033[{code}m{text}\033[0m" if USE_COLOR else text
|
||||
|
||||
|
||||
def green(t: str) -> str:
|
||||
return _color(t, "32")
|
||||
|
||||
|
||||
def yellow(t: str) -> str:
|
||||
return _color(t, "33")
|
||||
|
||||
|
||||
def red(t: str) -> str:
|
||||
return _color(t, "1;31")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class AssertionResult:
|
||||
"""Result of a single swarm-level assertion."""
|
||||
name: str
|
||||
passed: bool
|
||||
message: str
|
||||
severity: int # 0 = pass, 1 = warn, 2 = fail
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeLog:
|
||||
"""Parsed log for a single QEMU node."""
|
||||
node_id: int
|
||||
lines: List[str]
|
||||
text: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Log loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_logs(log_dir: Path, node_count: int) -> List[NodeLog]:
|
||||
"""Load qemu_node{i}.log (or node_{i}.log fallback) from *log_dir*."""
|
||||
logs: List[NodeLog] = []
|
||||
for i in range(node_count):
|
||||
path = log_dir / f"qemu_node{i}.log"
|
||||
if not path.exists():
|
||||
path = log_dir / f"node_{i}.log"
|
||||
if path.exists():
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
else:
|
||||
text = ""
|
||||
logs.append(NodeLog(node_id=i, lines=text.splitlines(), text=text))
|
||||
return logs
|
||||
|
||||
|
||||
def _node_count_from_dir(log_dir: Path) -> int:
|
||||
"""Auto-detect node count by scanning for qemu_node*.log (or node_*.log) files."""
|
||||
count = 0
|
||||
while (log_dir / f"qemu_node{count}.log").exists() or (log_dir / f"node_{count}.log").exists():
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Individual assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BOOT_PATTERNS = [
|
||||
r"app_main\(\)", r"main_task:", r"main:", r"ESP32-S3 CSI Node",
|
||||
]
|
||||
|
||||
_CRASH_PATTERNS = [
|
||||
r"Guru Meditation", r"assert failed", r"abort\(\)", r"panic",
|
||||
r"LoadProhibited", r"StoreProhibited", r"InstrFetchProhibited",
|
||||
r"IllegalInstruction", r"Unhandled debug exception", r"Fatal exception",
|
||||
]
|
||||
|
||||
_HEAP_PATTERNS = [
|
||||
r"HEAP_ERROR", r"out of memory", r"heap_caps_alloc.*failed",
|
||||
r"malloc.*fail", r"heap corruption", r"CORRUPT HEAP",
|
||||
r"multi_heap", r"heap_lock",
|
||||
]
|
||||
|
||||
_FRAME_PATTERNS = [
|
||||
r"frame", r"CSI", r"mock_csi", r"iq_data", r"subcarrier",
|
||||
r"csi_collector", r"enqueue",
|
||||
]
|
||||
|
||||
_FALL_PATTERNS = [r"fall[=: ]+1", r"fall detected", r"fall_event"]
|
||||
|
||||
|
||||
def assert_all_nodes_boot(logs: List[NodeLog], timeout_s: float = 10.0) -> AssertionResult:
|
||||
"""Check each node's log for boot patterns."""
|
||||
missing: List[int] = []
|
||||
for nl in logs:
|
||||
found = any(
|
||||
re.search(p, nl.text) for p in _BOOT_PATTERNS
|
||||
)
|
||||
if not found:
|
||||
missing.append(nl.node_id)
|
||||
|
||||
if not missing:
|
||||
return AssertionResult(
|
||||
name="all_nodes_boot", passed=True,
|
||||
message=f"All {len(logs)} nodes booted (timeout={timeout_s}s)",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="all_nodes_boot", passed=False,
|
||||
message=f"Nodes missing boot indicator: {missing}",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
|
||||
def assert_no_crashes(logs: List[NodeLog]) -> AssertionResult:
|
||||
"""Check no node has crash patterns."""
|
||||
crashed: List[str] = []
|
||||
for nl in logs:
|
||||
for line in nl.lines:
|
||||
for pat in _CRASH_PATTERNS:
|
||||
if re.search(pat, line):
|
||||
crashed.append(f"node_{nl.node_id}: {line.strip()[:100]}")
|
||||
break
|
||||
if crashed and crashed[-1].startswith(f"node_{nl.node_id}:"):
|
||||
break # one crash per node is enough
|
||||
|
||||
if not crashed:
|
||||
return AssertionResult(
|
||||
name="no_crashes", passed=True,
|
||||
message="No crash indicators in any node",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="no_crashes", passed=False,
|
||||
message=f"Crashes found: {crashed[0]}" + (
|
||||
f" (+{len(crashed)-1} more)" if len(crashed) > 1 else ""
|
||||
),
|
||||
severity=2,
|
||||
)
|
||||
|
||||
|
||||
def assert_tdm_no_collision(logs: List[NodeLog]) -> AssertionResult:
|
||||
"""Parse TDM slot assignments from logs, verify uniqueness."""
|
||||
slot_map: Dict[int, List[int]] = {} # slot -> [node_ids]
|
||||
tdm_pat = re.compile(r"tdm[_ ]?slot[=: ]+(\d+)", re.IGNORECASE)
|
||||
|
||||
for nl in logs:
|
||||
for line in nl.lines:
|
||||
m = tdm_pat.search(line)
|
||||
if m:
|
||||
slot = int(m.group(1))
|
||||
slot_map.setdefault(slot, [])
|
||||
if nl.node_id not in slot_map[slot]:
|
||||
slot_map[slot].append(nl.node_id)
|
||||
break # first occurrence per node
|
||||
|
||||
collisions = {s: nids for s, nids in slot_map.items() if len(nids) > 1}
|
||||
|
||||
if not slot_map:
|
||||
return AssertionResult(
|
||||
name="tdm_no_collision", passed=True,
|
||||
message="No TDM slot assignments found (may be N/A)",
|
||||
severity=0,
|
||||
)
|
||||
if not collisions:
|
||||
return AssertionResult(
|
||||
name="tdm_no_collision", passed=True,
|
||||
message=f"TDM slots unique across {len(slot_map)} assignments",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="tdm_no_collision", passed=False,
|
||||
message=f"TDM collisions: {collisions}",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
|
||||
def assert_all_nodes_produce_frames(
|
||||
logs: List[NodeLog],
|
||||
sensor_ids: Optional[List[int]] = None,
|
||||
) -> AssertionResult:
|
||||
"""Each sensor node has CSI frame output.
|
||||
|
||||
Args:
|
||||
logs: Parsed node logs.
|
||||
sensor_ids: If provided, only check these node IDs (skip coordinators).
|
||||
If None, check all nodes (legacy behavior).
|
||||
"""
|
||||
silent: List[int] = []
|
||||
for nl in logs:
|
||||
if sensor_ids is not None and nl.node_id not in sensor_ids:
|
||||
continue
|
||||
found = any(
|
||||
re.search(p, line, re.IGNORECASE)
|
||||
for line in nl.lines for p in _FRAME_PATTERNS
|
||||
)
|
||||
if not found:
|
||||
silent.append(nl.node_id)
|
||||
|
||||
checked = len(sensor_ids) if sensor_ids is not None else len(logs)
|
||||
if not silent:
|
||||
return AssertionResult(
|
||||
name="all_nodes_produce_frames", passed=True,
|
||||
message=f"All {checked} checked nodes show frame activity",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="all_nodes_produce_frames", passed=False,
|
||||
message=f"Nodes with no frame activity: {silent}",
|
||||
severity=1,
|
||||
)
|
||||
|
||||
|
||||
def assert_coordinator_receives_from_all(
|
||||
logs: List[NodeLog],
|
||||
coordinator_id: int = 0,
|
||||
sensor_ids: Optional[List[int]] = None,
|
||||
) -> AssertionResult:
|
||||
"""Coordinator log shows frames from each sensor's node_id."""
|
||||
coord_log = None
|
||||
for nl in logs:
|
||||
if nl.node_id == coordinator_id:
|
||||
coord_log = nl
|
||||
break
|
||||
|
||||
if coord_log is None:
|
||||
return AssertionResult(
|
||||
name="coordinator_receives_from_all", passed=False,
|
||||
message=f"Coordinator node_{coordinator_id} log not found",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
if sensor_ids is None:
|
||||
sensor_ids = [nl.node_id for nl in logs if nl.node_id != coordinator_id]
|
||||
|
||||
missing: List[int] = []
|
||||
recv_pat = re.compile(r"(from|node_id|src)[=: ]+(\d+)", re.IGNORECASE)
|
||||
received_ids: set = set()
|
||||
for line in coord_log.lines:
|
||||
m = recv_pat.search(line)
|
||||
if m:
|
||||
received_ids.add(int(m.group(2)))
|
||||
|
||||
for sid in sensor_ids:
|
||||
if sid not in received_ids:
|
||||
missing.append(sid)
|
||||
|
||||
if not missing:
|
||||
return AssertionResult(
|
||||
name="coordinator_receives_from_all", passed=True,
|
||||
message=f"Coordinator received from all sensors: {sensor_ids}",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="coordinator_receives_from_all", passed=False,
|
||||
message=f"Coordinator missing frames from nodes: {missing}",
|
||||
severity=1,
|
||||
)
|
||||
|
||||
|
||||
def assert_fall_detected(logs: List[NodeLog], node_id: int) -> AssertionResult:
|
||||
"""Specific node reports fall detection."""
|
||||
for nl in logs:
|
||||
if nl.node_id == node_id:
|
||||
found = any(
|
||||
re.search(p, line, re.IGNORECASE)
|
||||
for line in nl.lines for p in _FALL_PATTERNS
|
||||
)
|
||||
if found:
|
||||
return AssertionResult(
|
||||
name=f"fall_detected_node_{node_id}", passed=True,
|
||||
message=f"Node {node_id} reported fall event",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name=f"fall_detected_node_{node_id}", passed=False,
|
||||
message=f"Node {node_id} did not report fall event",
|
||||
severity=1,
|
||||
)
|
||||
|
||||
return AssertionResult(
|
||||
name=f"fall_detected_node_{node_id}", passed=False,
|
||||
message=f"Node {node_id} log not found",
|
||||
severity=2,
|
||||
)
|
||||
|
||||
|
||||
def assert_frame_rate_above(logs: List[NodeLog], min_fps: float = 10.0) -> AssertionResult:
|
||||
"""Each node meets minimum frame rate."""
|
||||
fps_pat = re.compile(r"(?:fps|frame.?rate)[=: ]+([0-9.]+)", re.IGNORECASE)
|
||||
count_pat = re.compile(r"(?:frame[_ ]?count|frames)[=: ]+(\d+)", re.IGNORECASE)
|
||||
below: List[str] = []
|
||||
|
||||
for nl in logs:
|
||||
best_fps: Optional[float] = None
|
||||
# Try explicit FPS
|
||||
for line in nl.lines:
|
||||
m = fps_pat.search(line)
|
||||
if m:
|
||||
try:
|
||||
best_fps = max(best_fps or 0.0, float(m.group(1)))
|
||||
except ValueError:
|
||||
pass
|
||||
# Fallback: estimate from frame count (assume 1-second intervals)
|
||||
if best_fps is None:
|
||||
counts = []
|
||||
for line in nl.lines:
|
||||
m = count_pat.search(line)
|
||||
if m:
|
||||
try:
|
||||
counts.append(int(m.group(1)))
|
||||
except ValueError:
|
||||
pass
|
||||
if len(counts) >= 2:
|
||||
best_fps = float(counts[-1] - counts[0]) / max(len(counts) - 1, 1)
|
||||
|
||||
if best_fps is not None and best_fps < min_fps:
|
||||
below.append(f"node_{nl.node_id}={best_fps:.1f}")
|
||||
|
||||
if not below:
|
||||
return AssertionResult(
|
||||
name="frame_rate_above", passed=True,
|
||||
message=f"All nodes meet minimum {min_fps} fps",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="frame_rate_above", passed=False,
|
||||
message=f"Nodes below {min_fps} fps: {', '.join(below)}",
|
||||
severity=1,
|
||||
)
|
||||
|
||||
|
||||
def assert_max_boot_time(logs: List[NodeLog], max_seconds: float = 10.0) -> AssertionResult:
|
||||
"""All nodes boot within N seconds (based on timestamp in log)."""
|
||||
boot_time_pat = re.compile(r"\((\d+)\)\s", re.IGNORECASE)
|
||||
slow: List[str] = []
|
||||
|
||||
for nl in logs:
|
||||
boot_found = False
|
||||
for line in nl.lines:
|
||||
if any(re.search(p, line) for p in _BOOT_PATTERNS):
|
||||
boot_found = True
|
||||
m = boot_time_pat.search(line)
|
||||
if m:
|
||||
ms = int(m.group(1))
|
||||
if ms > max_seconds * 1000:
|
||||
slow.append(f"node_{nl.node_id}={ms}ms")
|
||||
break
|
||||
if not boot_found:
|
||||
slow.append(f"node_{nl.node_id}=no_boot")
|
||||
|
||||
if not slow:
|
||||
return AssertionResult(
|
||||
name="max_boot_time", passed=True,
|
||||
message=f"All nodes booted within {max_seconds}s",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="max_boot_time", passed=False,
|
||||
message=f"Slow/missing boot: {', '.join(slow)}",
|
||||
severity=1,
|
||||
)
|
||||
|
||||
|
||||
def assert_no_heap_errors(logs: List[NodeLog]) -> AssertionResult:
|
||||
"""No OOM/heap errors in any log."""
|
||||
errors: List[str] = []
|
||||
for nl in logs:
|
||||
for line in nl.lines:
|
||||
for pat in _HEAP_PATTERNS:
|
||||
if re.search(pat, line, re.IGNORECASE):
|
||||
errors.append(f"node_{nl.node_id}: {line.strip()[:100]}")
|
||||
break
|
||||
if errors and errors[-1].startswith(f"node_{nl.node_id}:"):
|
||||
break
|
||||
|
||||
if not errors:
|
||||
return AssertionResult(
|
||||
name="no_heap_errors", passed=True,
|
||||
message="No heap errors in any node",
|
||||
severity=0,
|
||||
)
|
||||
return AssertionResult(
|
||||
name="no_heap_errors", passed=False,
|
||||
message=f"Heap errors: {errors[0]}" + (
|
||||
f" (+{len(errors)-1} more)" if len(errors) > 1 else ""
|
||||
),
|
||||
severity=2,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertion registry & dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ASSERTION_REGISTRY: Dict[str, Any] = {
|
||||
"all_nodes_boot": assert_all_nodes_boot,
|
||||
"no_crashes": assert_no_crashes,
|
||||
"tdm_no_collision": assert_tdm_no_collision,
|
||||
"all_nodes_produce_frames": assert_all_nodes_produce_frames,
|
||||
"coordinator_receives_from_all": assert_coordinator_receives_from_all,
|
||||
"frame_rate_above": assert_frame_rate_above,
|
||||
"max_boot_time": assert_max_boot_time,
|
||||
"no_heap_errors": assert_no_heap_errors,
|
||||
# fall_detected is parameterized, handled separately
|
||||
}
|
||||
|
||||
|
||||
def _parse_assertion_spec(spec: Any) -> tuple:
|
||||
"""Parse a YAML assertion entry into (name, kwargs).
|
||||
|
||||
Supported forms:
|
||||
- "all_nodes_boot" -> ("all_nodes_boot", {})
|
||||
- {"frame_rate_above": 15} -> ("frame_rate_above", {"min_fps": 15})
|
||||
- "fall_detected_by_node_2" -> ("fall_detected", {"node_id": 2})
|
||||
- {"max_boot_time_s": 10} -> ("max_boot_time", {"max_seconds": 10})
|
||||
"""
|
||||
if isinstance(spec, str):
|
||||
# Check for fall_detected_by_node_N pattern
|
||||
m = re.match(r"fall_detected_by_node_(\d+)", spec)
|
||||
if m:
|
||||
return ("fall_detected", {"node_id": int(m.group(1))})
|
||||
return (spec, {})
|
||||
|
||||
if isinstance(spec, dict):
|
||||
for key, val in spec.items():
|
||||
m = re.match(r"fall_detected_by_node_(\d+)", str(key))
|
||||
if m:
|
||||
return ("fall_detected", {"node_id": int(m.group(1))})
|
||||
if key == "frame_rate_above":
|
||||
return ("frame_rate_above", {"min_fps": float(val)})
|
||||
if key == "max_boot_time_s":
|
||||
return ("max_boot_time", {"max_seconds": float(val)})
|
||||
if key == "coordinator_receives_from_all":
|
||||
return ("coordinator_receives_from_all", {})
|
||||
return (str(key), {})
|
||||
|
||||
return (str(spec), {})
|
||||
|
||||
|
||||
def run_assertions(
|
||||
logs: List[NodeLog],
|
||||
assertion_specs: List[Any],
|
||||
config: Optional[Dict] = None,
|
||||
) -> List[AssertionResult]:
|
||||
"""Run all requested assertions against loaded logs."""
|
||||
results: List[AssertionResult] = []
|
||||
|
||||
# Derive coordinator/sensor IDs from config if available
|
||||
coordinator_id = 0
|
||||
sensor_ids: Optional[List[int]] = None
|
||||
if config and "nodes" in config:
|
||||
for node_def in config["nodes"]:
|
||||
if node_def.get("role") == "coordinator":
|
||||
coordinator_id = node_def.get("node_id", 0)
|
||||
sensor_ids = [
|
||||
n["node_id"] for n in config["nodes"]
|
||||
if n.get("role") == "sensor"
|
||||
]
|
||||
|
||||
for spec in assertion_specs:
|
||||
name, kwargs = _parse_assertion_spec(spec)
|
||||
|
||||
if name == "fall_detected":
|
||||
results.append(assert_fall_detected(logs, **kwargs))
|
||||
elif name == "coordinator_receives_from_all":
|
||||
results.append(assert_coordinator_receives_from_all(
|
||||
logs, coordinator_id=coordinator_id, sensor_ids=sensor_ids,
|
||||
))
|
||||
elif name == "all_nodes_produce_frames":
|
||||
results.append(assert_all_nodes_produce_frames(
|
||||
logs, sensor_ids=sensor_ids, **kwargs,
|
||||
))
|
||||
elif name in ASSERTION_REGISTRY:
|
||||
fn = ASSERTION_REGISTRY[name]
|
||||
results.append(fn(logs, **kwargs))
|
||||
else:
|
||||
results.append(AssertionResult(
|
||||
name=name, passed=False,
|
||||
message=f"Unknown assertion: {name}",
|
||||
severity=1,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report printing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def print_report(results: List[AssertionResult], swarm_name: str = "") -> int:
|
||||
"""Print the assertion report and return max severity."""
|
||||
header = "QEMU Swarm Health Report (ADR-062)"
|
||||
if swarm_name:
|
||||
header += f" - {swarm_name}"
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f" {header}")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
max_sev = 0
|
||||
for r in results:
|
||||
if r.severity == 0:
|
||||
icon = green("PASS")
|
||||
elif r.severity == 1:
|
||||
icon = yellow("WARN")
|
||||
else:
|
||||
icon = red("FAIL")
|
||||
|
||||
print(f" [{icon}] {r.name}: {r.message}")
|
||||
max_sev = max(max_sev, r.severity)
|
||||
|
||||
print()
|
||||
passed = sum(1 for r in results if r.passed)
|
||||
total = len(results)
|
||||
summary = f" {passed}/{total} assertions passed"
|
||||
|
||||
if max_sev == 0:
|
||||
print(green(summary))
|
||||
elif max_sev == 1:
|
||||
print(yellow(summary + " (with warnings)"))
|
||||
else:
|
||||
print(red(summary + " (with failures)"))
|
||||
|
||||
print()
|
||||
return max_sev
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="QEMU Swarm Health Oracle (ADR-062)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"Example:\n"
|
||||
" python3 swarm_health.py --config scripts/swarm_presets/standard.yaml \\\n"
|
||||
" --log-dir build/swarm_logs/\n"
|
||||
"\n"
|
||||
" python3 swarm_health.py --log-dir build/swarm_logs/ \\\n"
|
||||
" --assertions all_nodes_boot no_crashes\n"
|
||||
"\n"
|
||||
"Example output:\n"
|
||||
" ============================================================\n"
|
||||
" QEMU Swarm Health Report (ADR-062) - standard\n"
|
||||
" ============================================================\n"
|
||||
"\n"
|
||||
" [PASS] all_nodes_boot: All 3 nodes booted (timeout=10.0s)\n"
|
||||
" [PASS] no_crashes: No crash indicators in any node\n"
|
||||
" [PASS] tdm_no_collision: TDM slots unique across 3 assignments\n"
|
||||
" [PASS] all_nodes_produce_frames: All 3 nodes show frame activity\n"
|
||||
" [PASS] coordinator_receives_from_all: Coordinator received from all\n"
|
||||
" [WARN] fall_detected_node_2: Node 2 did not report fall event\n"
|
||||
" [PASS] frame_rate_above: All nodes meet minimum 15.0 fps\n"
|
||||
"\n"
|
||||
" 6/7 assertions passed (with warnings)\n"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config", type=str, default=None,
|
||||
help="Path to swarm YAML config (defines nodes and assertions)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-dir", type=str, required=True,
|
||||
help="Directory containing node_0.log, node_1.log, etc.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--assertions", nargs="*", default=None,
|
||||
help="Override assertions (space-separated). Ignores YAML assertion list.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--node-count", type=int, default=None,
|
||||
help="Number of nodes (auto-detected from log files if omitted)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
log_dir = Path(args.log_dir)
|
||||
if not log_dir.is_dir():
|
||||
print(f"ERROR: Log directory not found: {log_dir}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Load YAML config if provided
|
||||
config: Optional[Dict] = None
|
||||
swarm_name = ""
|
||||
yaml_assertions: List[Any] = []
|
||||
|
||||
if args.config:
|
||||
if yaml is None:
|
||||
print("ERROR: PyYAML is required for --config. Install with: pip install pyyaml",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
config_path = Path(args.config)
|
||||
if not config_path.exists():
|
||||
print(f"ERROR: Config file not found: {config_path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
with open(config_path, "r") as f:
|
||||
config = yaml.safe_load(f)
|
||||
swarm_name = config.get("swarm", {}).get("name", "")
|
||||
yaml_assertions = config.get("assertions", [])
|
||||
|
||||
# Determine node count
|
||||
if args.node_count is not None:
|
||||
node_count = args.node_count
|
||||
elif config and "nodes" in config:
|
||||
node_count = len(config["nodes"])
|
||||
else:
|
||||
node_count = _node_count_from_dir(log_dir)
|
||||
|
||||
if node_count == 0:
|
||||
print("ERROR: No node logs found and node count not specified.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Load logs
|
||||
logs = load_logs(log_dir, node_count)
|
||||
|
||||
# Determine which assertions to run
|
||||
if args.assertions is not None:
|
||||
assertion_specs = args.assertions
|
||||
elif yaml_assertions:
|
||||
assertion_specs = yaml_assertions
|
||||
else:
|
||||
# Default set
|
||||
assertion_specs = ["all_nodes_boot", "no_crashes", "no_heap_errors"]
|
||||
|
||||
# Run assertions
|
||||
results = run_assertions(logs, assertion_specs, config)
|
||||
|
||||
# Print report and exit
|
||||
max_sev = print_report(results, swarm_name)
|
||||
sys.exit(max_sev)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
# CI-optimized preset: 3 nodes, star topology, 30s, minimal assertions
|
||||
swarm:
|
||||
name: ci-matrix
|
||||
duration_s: 30
|
||||
topology: star
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- max_boot_time_s: 10
|
||||
@@ -0,0 +1,49 @@
|
||||
# Mixed scenarios: 5 nodes with different CSI scenarios, star topology, 90s
|
||||
swarm:
|
||||
name: heterogeneous
|
||||
duration_s: 90
|
||||
topology: star
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
- role: sensor
|
||||
node_id: 3
|
||||
scenario: 3
|
||||
channel: 6
|
||||
tdm_slot: 3
|
||||
|
||||
- role: sensor
|
||||
node_id: 4
|
||||
scenario: 5
|
||||
channel: 11
|
||||
tdm_slot: 4
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- coordinator_receives_from_all
|
||||
- fall_detected_by_node_3
|
||||
- no_heap_errors
|
||||
- frame_rate_above: 12
|
||||
- max_boot_time_s: 12
|
||||
@@ -0,0 +1,54 @@
|
||||
# Scale test: 6 fully-connected nodes in mesh topology, 90s
|
||||
swarm:
|
||||
name: large-mesh
|
||||
duration_s: 90
|
||||
topology: mesh
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
- role: sensor
|
||||
node_id: 3
|
||||
scenario: 3
|
||||
channel: 6
|
||||
tdm_slot: 3
|
||||
|
||||
- role: sensor
|
||||
node_id: 4
|
||||
scenario: 4
|
||||
channel: 6
|
||||
tdm_slot: 4
|
||||
|
||||
- role: sensor
|
||||
node_id: 5
|
||||
scenario: 5
|
||||
channel: 6
|
||||
tdm_slot: 5
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- coordinator_receives_from_all
|
||||
- no_heap_errors
|
||||
- frame_rate_above: 10
|
||||
- max_boot_time_s: 15
|
||||
@@ -0,0 +1,39 @@
|
||||
# Multi-hop relay chain: 4 nodes in line topology, 60s
|
||||
swarm:
|
||||
name: line-relay
|
||||
duration_s: 60
|
||||
topology: line
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: gateway
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true
|
||||
|
||||
- role: coordinator
|
||||
node_id: 1
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
- role: sensor
|
||||
node_id: 3
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 3
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- max_boot_time_s: 12
|
||||
@@ -0,0 +1,41 @@
|
||||
# Ring topology with fault injection: 4 nodes, 75s
|
||||
swarm:
|
||||
name: ring-fault
|
||||
duration_s: 75
|
||||
topology: ring
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
- role: sensor
|
||||
node_id: 3
|
||||
scenario: 3
|
||||
channel: 6
|
||||
tdm_slot: 3
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- coordinator_receives_from_all
|
||||
- no_heap_errors
|
||||
- max_boot_time_s: 12
|
||||
@@ -0,0 +1,24 @@
|
||||
# Quick CI smoke test: 2 nodes, star topology, 15s duration
|
||||
swarm:
|
||||
name: smoke
|
||||
duration_s: 15
|
||||
topology: star
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 1
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- max_boot_time_s: 10
|
||||
@@ -0,0 +1,36 @@
|
||||
# Standard 3-node test: 2 sensors + 1 coordinator, star topology, 60s
|
||||
swarm:
|
||||
name: standard
|
||||
duration_s: 60
|
||||
topology: star
|
||||
aggregator_port: 5005
|
||||
|
||||
nodes:
|
||||
- role: coordinator
|
||||
node_id: 0
|
||||
scenario: 0
|
||||
channel: 6
|
||||
edge_tier: 2
|
||||
is_gateway: true
|
||||
|
||||
- role: sensor
|
||||
node_id: 1
|
||||
scenario: 2
|
||||
channel: 6
|
||||
tdm_slot: 1
|
||||
|
||||
- role: sensor
|
||||
node_id: 2
|
||||
scenario: 3
|
||||
channel: 6
|
||||
tdm_slot: 2
|
||||
|
||||
assertions:
|
||||
- all_nodes_boot
|
||||
- no_crashes
|
||||
- tdm_no_collision
|
||||
- all_nodes_produce_frames
|
||||
- coordinator_receives_from_all
|
||||
- fall_detected_by_node_2
|
||||
- frame_rate_above: 15
|
||||
- max_boot_time_s: 10
|
||||
@@ -0,0 +1,504 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
QEMU Multi-Node Mesh Validation (ADR-061 Layer 3)
|
||||
|
||||
Validates the output of a multi-node mesh simulation run by qemu-mesh-test.sh.
|
||||
Parses the aggregator results JSON and per-node UART logs, then runs 6 checks:
|
||||
|
||||
1. All nodes booted - every node log contains a boot indicator
|
||||
2. TDM ordering - slot assignments are sequential 0..N-1
|
||||
3. No slot collision - no two nodes share a TDM slot
|
||||
4. Frame count balance - per-node frame counts within +/-10%
|
||||
5. ADR-018 compliance - magic 0xC5110001 present in frames
|
||||
6. Vitals per node - each node produced vitals output
|
||||
|
||||
Usage:
|
||||
python3 validate_mesh_test.py --nodes N [results.json] [--log node0.log] ...
|
||||
|
||||
Exit codes:
|
||||
0 All checks passed (or only SKIP-level)
|
||||
1 Warnings (non-critical checks failed)
|
||||
2 Errors (critical checks failed)
|
||||
3 Fatal (crash or missing nodes)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from enum import IntEnum
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Severity / reporting (matches validate_qemu_output.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Severity(IntEnum):
|
||||
PASS = 0
|
||||
SKIP = 1
|
||||
WARN = 2
|
||||
ERROR = 3
|
||||
FATAL = 4
|
||||
|
||||
|
||||
USE_COLOR = sys.stdout.isatty()
|
||||
|
||||
|
||||
def color(text: str, code: str) -> str:
|
||||
if not USE_COLOR:
|
||||
return text
|
||||
return f"\033[{code}m{text}\033[0m"
|
||||
|
||||
|
||||
def green(text: str) -> str:
|
||||
return color(text, "32")
|
||||
|
||||
|
||||
def yellow(text: str) -> str:
|
||||
return color(text, "33")
|
||||
|
||||
|
||||
def red(text: str) -> str:
|
||||
return color(text, "31")
|
||||
|
||||
|
||||
def bold_red(text: str) -> str:
|
||||
return color(text, "1;31")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
name: str
|
||||
severity: Severity
|
||||
message: str
|
||||
count: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationReport:
|
||||
checks: List[CheckResult] = field(default_factory=list)
|
||||
|
||||
def add(self, name: str, severity: Severity, message: str, count: int = 0):
|
||||
self.checks.append(CheckResult(name, severity, message, count))
|
||||
|
||||
@property
|
||||
def max_severity(self) -> Severity:
|
||||
if not self.checks:
|
||||
return Severity.PASS
|
||||
return max(c.severity for c in self.checks)
|
||||
|
||||
def print_report(self):
|
||||
print("\n" + "=" * 60)
|
||||
print(" Multi-Node Mesh Validation Report (ADR-061 Layer 3)")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
for check in self.checks:
|
||||
if check.severity == Severity.PASS:
|
||||
icon = green("PASS")
|
||||
elif check.severity == Severity.SKIP:
|
||||
icon = yellow("SKIP")
|
||||
elif check.severity == Severity.WARN:
|
||||
icon = yellow("WARN")
|
||||
elif check.severity == Severity.ERROR:
|
||||
icon = red("FAIL")
|
||||
else:
|
||||
icon = bold_red("FATAL")
|
||||
|
||||
count_str = f" (count={check.count})" if check.count > 0 else ""
|
||||
print(f" [{icon}] {check.name}: {check.message}{count_str}")
|
||||
|
||||
print()
|
||||
|
||||
passed = sum(1 for c in self.checks if c.severity <= Severity.SKIP)
|
||||
total = len(self.checks)
|
||||
summary = f" {passed}/{total} checks passed"
|
||||
|
||||
max_sev = self.max_severity
|
||||
if max_sev <= Severity.SKIP:
|
||||
print(green(summary))
|
||||
elif max_sev == Severity.WARN:
|
||||
print(yellow(summary + " (with warnings)"))
|
||||
elif max_sev == Severity.ERROR:
|
||||
print(red(summary + " (with errors)"))
|
||||
else:
|
||||
print(bold_red(summary + " (FATAL issues detected)"))
|
||||
|
||||
print()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Log parsing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_node_booted(log_text: str) -> bool:
|
||||
"""Return True if the log shows a boot indicator."""
|
||||
boot_patterns = [r"app_main\(\)", r"main_task:", r"main:", r"ESP32-S3 CSI Node"]
|
||||
return any(re.search(p, log_text) for p in boot_patterns)
|
||||
|
||||
|
||||
def check_node_crashed(log_text: str) -> Optional[str]:
|
||||
"""Return first crash line or None."""
|
||||
crash_patterns = [
|
||||
r"Guru Meditation", r"assert failed", r"abort\(\)",
|
||||
r"panic", r"LoadProhibited", r"StoreProhibited",
|
||||
r"InstrFetchProhibited", r"IllegalInstruction",
|
||||
]
|
||||
for line in log_text.splitlines():
|
||||
for pat in crash_patterns:
|
||||
if re.search(pat, line):
|
||||
return line.strip()[:120]
|
||||
return None
|
||||
|
||||
|
||||
def extract_node_id_from_log(log_text: str) -> Optional[int]:
|
||||
"""Try to extract the node_id from UART log lines."""
|
||||
patterns = [
|
||||
r"node_id[=: ]+(\d+)",
|
||||
r"Node ID[=: ]+(\d+)",
|
||||
r"TDM slot[=: ]+(\d+)",
|
||||
]
|
||||
for line in log_text.splitlines():
|
||||
for pat in patterns:
|
||||
m = re.search(pat, line, re.IGNORECASE)
|
||||
if m:
|
||||
try:
|
||||
return int(m.group(1))
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def check_vitals_in_log(log_text: str) -> bool:
|
||||
"""Return True if the log contains vitals output."""
|
||||
vitals_patterns = [r"vitals", r"breathing", r"breathing_bpm",
|
||||
r"heart_rate", r"heartrate"]
|
||||
return any(
|
||||
re.search(p, line, re.IGNORECASE)
|
||||
for line in log_text.splitlines()
|
||||
for p in vitals_patterns
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def validate_mesh(
|
||||
n_nodes: int,
|
||||
results_path: Optional[Path],
|
||||
log_paths: List[Path],
|
||||
) -> ValidationReport:
|
||||
"""Run all 6 mesh validation checks."""
|
||||
report = ValidationReport()
|
||||
|
||||
# Load aggregator results if available
|
||||
results: Optional[dict] = None
|
||||
if results_path:
|
||||
if not results_path.exists():
|
||||
print(f"WARNING: Aggregator results file not found: {results_path}",
|
||||
file=sys.stderr)
|
||||
report.add("Results JSON", Severity.WARN,
|
||||
f"Results file not found: {results_path}")
|
||||
else:
|
||||
try:
|
||||
results = json.loads(results_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
report.add("Results JSON", Severity.ERROR,
|
||||
f"Failed to parse results: {exc}")
|
||||
|
||||
# Load per-node logs
|
||||
node_logs: Dict[int, str] = {}
|
||||
for idx, lp in enumerate(log_paths):
|
||||
if lp.exists():
|
||||
node_logs[idx] = lp.read_text(encoding="utf-8", errors="replace")
|
||||
else:
|
||||
node_logs[idx] = ""
|
||||
|
||||
# ---- Check 1: All nodes booted ----
|
||||
booted = []
|
||||
not_booted = []
|
||||
crashed = []
|
||||
for idx in range(n_nodes):
|
||||
log_text = node_logs.get(idx, "")
|
||||
if not log_text.strip():
|
||||
not_booted.append(idx)
|
||||
continue
|
||||
crash_line = check_node_crashed(log_text)
|
||||
if crash_line:
|
||||
crashed.append((idx, crash_line))
|
||||
if check_node_booted(log_text):
|
||||
booted.append(idx)
|
||||
else:
|
||||
not_booted.append(idx)
|
||||
|
||||
if crashed:
|
||||
crash_desc = "; ".join(f"node {i}: {msg}" for i, msg in crashed)
|
||||
report.add("All nodes booted", Severity.FATAL,
|
||||
f"Crash detected: {crash_desc}", count=len(crashed))
|
||||
elif len(booted) == n_nodes:
|
||||
report.add("All nodes booted", Severity.PASS,
|
||||
f"All {n_nodes} nodes booted successfully", count=n_nodes)
|
||||
elif len(booted) == 0:
|
||||
report.add("All nodes booted", Severity.FATAL,
|
||||
f"No nodes booted (expected {n_nodes})")
|
||||
else:
|
||||
missing = ", ".join(str(i) for i in not_booted)
|
||||
report.add("All nodes booted", Severity.ERROR,
|
||||
f"{len(booted)}/{n_nodes} booted; missing: [{missing}]",
|
||||
count=len(booted))
|
||||
|
||||
# ---- Check 2: TDM ordering ----
|
||||
# Extract TDM slots either from aggregator results or from logs
|
||||
tdm_slots: Dict[int, int] = {}
|
||||
|
||||
# Try aggregator results first
|
||||
if results and "nodes" in results:
|
||||
for node_entry in results["nodes"]:
|
||||
nid = node_entry.get("node_id")
|
||||
slot = node_entry.get("tdm_slot")
|
||||
if nid is not None and slot is not None:
|
||||
tdm_slots[int(nid)] = int(slot)
|
||||
|
||||
# Fall back to log extraction
|
||||
if not tdm_slots:
|
||||
for idx in range(n_nodes):
|
||||
log_text = node_logs.get(idx, "")
|
||||
nid = extract_node_id_from_log(log_text)
|
||||
if nid is not None:
|
||||
tdm_slots[idx] = nid
|
||||
|
||||
if len(tdm_slots) == n_nodes:
|
||||
expected = list(range(n_nodes))
|
||||
actual = [tdm_slots.get(i, -1) for i in range(n_nodes)]
|
||||
if actual == expected:
|
||||
report.add("TDM ordering", Severity.PASS,
|
||||
f"Slots sequential 0..{n_nodes - 1}")
|
||||
else:
|
||||
report.add("TDM ordering", Severity.ERROR,
|
||||
f"Expected slots {expected}, got {actual}")
|
||||
elif len(tdm_slots) > 0:
|
||||
report.add("TDM ordering", Severity.WARN,
|
||||
f"Only {len(tdm_slots)}/{n_nodes} TDM slots detected",
|
||||
count=len(tdm_slots))
|
||||
else:
|
||||
report.add("TDM ordering", Severity.SKIP,
|
||||
"No TDM slot info found in results or logs")
|
||||
|
||||
# ---- Check 3: No slot collision ----
|
||||
if tdm_slots:
|
||||
slot_to_nodes: Dict[int, List[int]] = {}
|
||||
for nid, slot in tdm_slots.items():
|
||||
slot_to_nodes.setdefault(slot, []).append(nid)
|
||||
|
||||
collisions = {s: nodes for s, nodes in slot_to_nodes.items() if len(nodes) > 1}
|
||||
if not collisions:
|
||||
report.add("No slot collision", Severity.PASS,
|
||||
f"All {len(tdm_slots)} slots unique")
|
||||
else:
|
||||
desc = "; ".join(f"slot {s}: nodes {ns}" for s, ns in collisions.items())
|
||||
report.add("No slot collision", Severity.ERROR,
|
||||
f"Slot collisions: {desc}", count=len(collisions))
|
||||
else:
|
||||
report.add("No slot collision", Severity.SKIP,
|
||||
"No TDM slot data to check for collisions")
|
||||
|
||||
# ---- Check 4: Frame count balance (within +/-10%) ----
|
||||
frame_counts: Dict[int, int] = {}
|
||||
|
||||
# Try aggregator results
|
||||
if results and "nodes" in results:
|
||||
for node_entry in results["nodes"]:
|
||||
nid = node_entry.get("node_id")
|
||||
fc = node_entry.get("frame_count", node_entry.get("frames", 0))
|
||||
if nid is not None:
|
||||
frame_counts[int(nid)] = int(fc)
|
||||
|
||||
# Fall back to log extraction
|
||||
if not frame_counts:
|
||||
for idx in range(n_nodes):
|
||||
log_text = node_logs.get(idx, "")
|
||||
frame_pats = [
|
||||
r"frame[_ ]count[=: ]+(\d+)",
|
||||
r"frames?[=: ]+(\d+)",
|
||||
r"emitted[=: ]+(\d+)",
|
||||
]
|
||||
max_fc = 0
|
||||
for line in log_text.splitlines():
|
||||
for pat in frame_pats:
|
||||
m = re.search(pat, line, re.IGNORECASE)
|
||||
if m:
|
||||
try:
|
||||
max_fc = max(max_fc, int(m.group(1)))
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
if max_fc > 0:
|
||||
frame_counts[idx] = max_fc
|
||||
|
||||
if len(frame_counts) >= 2:
|
||||
counts = list(frame_counts.values())
|
||||
avg = sum(counts) / len(counts)
|
||||
if avg > 0:
|
||||
max_deviation = max(abs(c - avg) / avg for c in counts)
|
||||
details = ", ".join(f"node {nid}={fc}" for nid, fc in sorted(frame_counts.items()))
|
||||
if max_deviation <= 0.10:
|
||||
report.add("Frame count balance", Severity.PASS,
|
||||
f"Within +/-10% (avg={avg:.0f}): {details}",
|
||||
count=int(avg))
|
||||
elif max_deviation <= 0.25:
|
||||
report.add("Frame count balance", Severity.WARN,
|
||||
f"Deviation {max_deviation:.0%} exceeds 10%: {details}",
|
||||
count=int(avg))
|
||||
else:
|
||||
report.add("Frame count balance", Severity.ERROR,
|
||||
f"Severe imbalance {max_deviation:.0%}: {details}",
|
||||
count=int(avg))
|
||||
else:
|
||||
report.add("Frame count balance", Severity.ERROR,
|
||||
"All frame counts are zero")
|
||||
elif len(frame_counts) == 1:
|
||||
report.add("Frame count balance", Severity.WARN,
|
||||
f"Only 1 node reported frames: {frame_counts}")
|
||||
else:
|
||||
report.add("Frame count balance", Severity.WARN,
|
||||
"No frame count data found")
|
||||
|
||||
# ---- Check 5: ADR-018 compliance (magic 0xC5110001) ----
|
||||
ADR018_MAGIC = "c5110001"
|
||||
magic_found = False
|
||||
|
||||
# Check aggregator results
|
||||
if results:
|
||||
results_str = json.dumps(results).lower()
|
||||
if ADR018_MAGIC in results_str or "0xc5110001" in results_str:
|
||||
magic_found = True
|
||||
# Also check a dedicated field
|
||||
if results.get("adr018_magic") or results.get("magic"):
|
||||
magic_found = True
|
||||
# Check per-node entries
|
||||
if "nodes" in results:
|
||||
for node_entry in results["nodes"]:
|
||||
magic = node_entry.get("magic", "")
|
||||
if isinstance(magic, str) and ADR018_MAGIC in magic.lower():
|
||||
magic_found = True
|
||||
elif isinstance(magic, int) and magic == 0xC5110001:
|
||||
magic_found = True
|
||||
|
||||
# Check logs for serialization/ADR-018 markers
|
||||
if not magic_found:
|
||||
for idx in range(n_nodes):
|
||||
log_text = node_logs.get(idx, "")
|
||||
adr018_pats = [
|
||||
r"0xC5110001",
|
||||
r"c5110001",
|
||||
r"ADR-018",
|
||||
r"magic[=: ]+0x[Cc]5110001",
|
||||
]
|
||||
if any(re.search(p, log_text, re.IGNORECASE) for p in adr018_pats):
|
||||
magic_found = True
|
||||
break
|
||||
|
||||
if magic_found:
|
||||
report.add("ADR-018 compliance", Severity.PASS,
|
||||
"Magic 0xC5110001 found in frame data")
|
||||
else:
|
||||
report.add("ADR-018 compliance", Severity.WARN,
|
||||
"Magic 0xC5110001 not found (may require deeper frame inspection)")
|
||||
|
||||
# ---- Check 6: Vitals per node ----
|
||||
vitals_nodes = []
|
||||
no_vitals_nodes = []
|
||||
for idx in range(n_nodes):
|
||||
log_text = node_logs.get(idx, "")
|
||||
if check_vitals_in_log(log_text):
|
||||
vitals_nodes.append(idx)
|
||||
else:
|
||||
no_vitals_nodes.append(idx)
|
||||
|
||||
# Also check aggregator results for vitals data
|
||||
if results and "nodes" in results:
|
||||
for node_entry in results["nodes"]:
|
||||
nid = node_entry.get("node_id")
|
||||
has_vitals = (
|
||||
node_entry.get("vitals") is not None
|
||||
or node_entry.get("breathing_bpm") is not None
|
||||
or node_entry.get("heart_rate") is not None
|
||||
)
|
||||
if has_vitals and nid is not None and int(nid) not in vitals_nodes:
|
||||
vitals_nodes.append(int(nid))
|
||||
if int(nid) in no_vitals_nodes:
|
||||
no_vitals_nodes.remove(int(nid))
|
||||
|
||||
if len(vitals_nodes) == n_nodes:
|
||||
report.add("Vitals per node", Severity.PASS,
|
||||
f"All {n_nodes} nodes produced vitals output",
|
||||
count=n_nodes)
|
||||
elif len(vitals_nodes) > 0:
|
||||
missing = ", ".join(str(i) for i in no_vitals_nodes)
|
||||
report.add("Vitals per node", Severity.WARN,
|
||||
f"{len(vitals_nodes)}/{n_nodes} nodes have vitals; "
|
||||
f"missing: [{missing}]",
|
||||
count=len(vitals_nodes))
|
||||
else:
|
||||
report.add("Vitals per node", Severity.WARN,
|
||||
"No vitals output found from any node")
|
||||
|
||||
return report
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate multi-node mesh QEMU test output (ADR-061 Layer 3)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"Examples:\n"
|
||||
" python3 validate_mesh_test.py --nodes 3 --results mesh_results.json\n"
|
||||
" python3 validate_mesh_test.py --nodes 4 --log node0.log --log node1.log"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--results", default=None,
|
||||
help="Path to mesh_test_results.json from aggregator")
|
||||
parser.add_argument("--nodes", "-n", type=int, required=True,
|
||||
help="Expected number of mesh nodes")
|
||||
parser.add_argument("--log", action="append", default=[],
|
||||
help="Path to a per-node QEMU log (can be repeated)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.nodes < 2:
|
||||
print("ERROR: --nodes must be >= 2", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
|
||||
results_path = Path(args.results) if args.results else None
|
||||
log_paths = [Path(lp) for lp in args.log]
|
||||
|
||||
# If no log files given, try the conventional paths
|
||||
if not log_paths:
|
||||
for i in range(args.nodes):
|
||||
candidate = Path(f"build/qemu_node{i}.log")
|
||||
if candidate.exists():
|
||||
log_paths.append(candidate)
|
||||
|
||||
report = validate_mesh(args.nodes, results_path, log_paths)
|
||||
report.print_report()
|
||||
|
||||
# Map max severity to exit code
|
||||
max_sev = report.max_severity
|
||||
if max_sev <= Severity.SKIP:
|
||||
sys.exit(0)
|
||||
elif max_sev == Severity.WARN:
|
||||
sys.exit(1)
|
||||
elif max_sev == Severity.ERROR:
|
||||
sys.exit(2)
|
||||
else:
|
||||
sys.exit(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,408 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
QEMU ESP32-S3 UART Output Validator (ADR-061)
|
||||
|
||||
Parses the UART log captured from a QEMU firmware run and validates
|
||||
16 checks covering boot, NVS, mock CSI, edge processing, vitals,
|
||||
presence/fall detection, serialization, crash indicators, scenario
|
||||
completion, and frame rate sanity.
|
||||
|
||||
Usage:
|
||||
python3 validate_qemu_output.py <log_file>
|
||||
|
||||
Exit codes:
|
||||
0 All checks passed (or only INFO-level skips)
|
||||
1 Warnings (non-critical checks failed)
|
||||
2 Errors (critical checks failed)
|
||||
3 Fatal (crash or corruption detected)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from enum import IntEnum
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class Severity(IntEnum):
|
||||
PASS = 0
|
||||
SKIP = 1
|
||||
WARN = 2
|
||||
ERROR = 3
|
||||
FATAL = 4
|
||||
|
||||
|
||||
# ANSI color codes (disabled if not a TTY)
|
||||
USE_COLOR = sys.stdout.isatty()
|
||||
|
||||
|
||||
def color(text: str, code: str) -> str:
|
||||
if not USE_COLOR:
|
||||
return text
|
||||
return f"\033[{code}m{text}\033[0m"
|
||||
|
||||
|
||||
def green(text: str) -> str:
|
||||
return color(text, "32")
|
||||
|
||||
|
||||
def yellow(text: str) -> str:
|
||||
return color(text, "33")
|
||||
|
||||
|
||||
def red(text: str) -> str:
|
||||
return color(text, "31")
|
||||
|
||||
|
||||
def bold_red(text: str) -> str:
|
||||
return color(text, "1;31")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
name: str
|
||||
severity: Severity
|
||||
message: str
|
||||
count: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationReport:
|
||||
checks: List[CheckResult] = field(default_factory=list)
|
||||
|
||||
def add(self, name: str, severity: Severity, message: str, count: int = 0):
|
||||
self.checks.append(CheckResult(name, severity, message, count))
|
||||
|
||||
@property
|
||||
def max_severity(self) -> Severity:
|
||||
if not self.checks:
|
||||
return Severity.PASS
|
||||
return max(c.severity for c in self.checks)
|
||||
|
||||
def print_report(self):
|
||||
print("\n" + "=" * 60)
|
||||
print(" QEMU Firmware Validation Report (ADR-061)")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
for check in self.checks:
|
||||
if check.severity == Severity.PASS:
|
||||
icon = green("PASS")
|
||||
elif check.severity == Severity.SKIP:
|
||||
icon = yellow("SKIP")
|
||||
elif check.severity == Severity.WARN:
|
||||
icon = yellow("WARN")
|
||||
elif check.severity == Severity.ERROR:
|
||||
icon = red("FAIL")
|
||||
else:
|
||||
icon = bold_red("FATAL")
|
||||
|
||||
count_str = f" (count={check.count})" if check.count > 0 else ""
|
||||
print(f" [{icon}] {check.name}: {check.message}{count_str}")
|
||||
|
||||
print()
|
||||
|
||||
passed = sum(1 for c in self.checks if c.severity <= Severity.SKIP)
|
||||
total = len(self.checks)
|
||||
summary = f" {passed}/{total} checks passed"
|
||||
|
||||
max_sev = self.max_severity
|
||||
if max_sev <= Severity.SKIP:
|
||||
print(green(summary))
|
||||
elif max_sev == Severity.WARN:
|
||||
print(yellow(summary + " (with warnings)"))
|
||||
elif max_sev == Severity.ERROR:
|
||||
print(red(summary + " (with errors)"))
|
||||
else:
|
||||
print(bold_red(summary + " (FATAL issues detected)"))
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def validate_log(log_text: str) -> ValidationReport:
|
||||
"""Run all 16 validation checks against the UART log text."""
|
||||
report = ValidationReport()
|
||||
lines = log_text.splitlines()
|
||||
log_lower = log_text.lower()
|
||||
|
||||
# ---- Check 1: Boot ----
|
||||
# Look for app_main() entry or main_task: tag
|
||||
boot_patterns = [r"app_main\(\)", r"main_task:", r"main:", r"ESP32-S3 CSI Node"]
|
||||
boot_found = any(re.search(p, log_text) for p in boot_patterns)
|
||||
if boot_found:
|
||||
report.add("Boot", Severity.PASS, "Firmware booted successfully")
|
||||
else:
|
||||
report.add("Boot", Severity.FATAL, "No boot indicator found (app_main / main_task)")
|
||||
|
||||
# ---- Check 2: NVS load ----
|
||||
nvs_patterns = [r"nvs_config:", r"nvs_config_load", r"NVS", r"csi_cfg"]
|
||||
nvs_found = any(re.search(p, log_text) for p in nvs_patterns)
|
||||
if nvs_found:
|
||||
report.add("NVS load", Severity.PASS, "NVS configuration loaded")
|
||||
else:
|
||||
report.add("NVS load", Severity.WARN, "No NVS load indicator found")
|
||||
|
||||
# ---- Check 3: Mock CSI init ----
|
||||
mock_patterns = [r"mock_csi:", r"mock_csi_init", r"Mock CSI", r"MOCK_CSI"]
|
||||
mock_found = any(re.search(p, log_text) for p in mock_patterns)
|
||||
if mock_found:
|
||||
report.add("Mock CSI init", Severity.PASS, "Mock CSI generator initialized")
|
||||
else:
|
||||
# This is only expected when mock is enabled
|
||||
report.add("Mock CSI init", Severity.SKIP,
|
||||
"No mock CSI indicator (expected if mock not enabled)")
|
||||
|
||||
# ---- Check 4: Frame generation ----
|
||||
# Count frame-related log lines
|
||||
frame_patterns = [
|
||||
r"frame[_ ]count[=: ]+(\d+)",
|
||||
r"frames?[=: ]+(\d+)",
|
||||
r"emitted[=: ]+(\d+)",
|
||||
r"mock_csi:.*frame",
|
||||
r"csi_collector:.*frame",
|
||||
r"CSI frame",
|
||||
]
|
||||
frame_count = 0
|
||||
for line in lines:
|
||||
for pat in frame_patterns:
|
||||
m = re.search(pat, line, re.IGNORECASE)
|
||||
if m:
|
||||
if m.lastindex and m.lastindex >= 1:
|
||||
try:
|
||||
frame_count = max(frame_count, int(m.group(1)))
|
||||
except (ValueError, IndexError):
|
||||
frame_count = max(frame_count, 1)
|
||||
else:
|
||||
frame_count = max(frame_count, 1)
|
||||
|
||||
if frame_count > 0:
|
||||
report.add("Frame generation", Severity.PASS,
|
||||
f"Frames detected", count=frame_count)
|
||||
else:
|
||||
# Also count lines mentioning IQ data or subcarriers
|
||||
iq_lines = sum(1 for line in lines
|
||||
if re.search(r"(iq_data|subcarrier|I/Q|enqueue)", line, re.IGNORECASE))
|
||||
if iq_lines > 0:
|
||||
report.add("Frame generation", Severity.PASS,
|
||||
"I/Q data activity detected", count=iq_lines)
|
||||
else:
|
||||
report.add("Frame generation", Severity.WARN,
|
||||
"No frame generation activity detected")
|
||||
|
||||
# ---- Check 5: Edge pipeline ----
|
||||
edge_patterns = [r"edge_processing:", r"DSP task", r"edge_init", r"edge_tier"]
|
||||
edge_found = any(re.search(p, log_text) for p in edge_patterns)
|
||||
if edge_found:
|
||||
report.add("Edge pipeline", Severity.PASS, "Edge processing pipeline active")
|
||||
else:
|
||||
report.add("Edge pipeline", Severity.WARN,
|
||||
"No edge processing indicator found")
|
||||
|
||||
# ---- Check 6: Vitals output ----
|
||||
vitals_patterns = [r"vitals", r"breathing", r"presence", r"heartrate",
|
||||
r"breathing_bpm", r"heart_rate"]
|
||||
vitals_count = sum(1 for line in lines
|
||||
if any(re.search(p, line, re.IGNORECASE) for p in vitals_patterns))
|
||||
if vitals_count > 0:
|
||||
report.add("Vitals output", Severity.PASS,
|
||||
"Vitals/breathing/presence output detected", count=vitals_count)
|
||||
else:
|
||||
report.add("Vitals output", Severity.WARN,
|
||||
"No vitals output lines found")
|
||||
|
||||
# ---- Check 7: Presence detection ----
|
||||
presence_patterns = [
|
||||
r"presence[=: ]+1",
|
||||
r"presence_score[=: ]+([0-9.]+)",
|
||||
r"presence detected",
|
||||
]
|
||||
presence_found = False
|
||||
for line in lines:
|
||||
for pat in presence_patterns:
|
||||
m = re.search(pat, line, re.IGNORECASE)
|
||||
if m:
|
||||
if m.lastindex and m.lastindex >= 1:
|
||||
try:
|
||||
score = float(m.group(1))
|
||||
if score > 0:
|
||||
presence_found = True
|
||||
except (ValueError, IndexError):
|
||||
presence_found = True
|
||||
else:
|
||||
presence_found = True
|
||||
|
||||
if presence_found:
|
||||
report.add("Presence detection", Severity.PASS, "Presence detected in output")
|
||||
else:
|
||||
report.add("Presence detection", Severity.WARN,
|
||||
"No presence=1 or presence_score>0 found")
|
||||
|
||||
# ---- Check 8: Fall detection ----
|
||||
fall_patterns = [r"fall[=: ]+1", r"fall detected", r"fall_event"]
|
||||
fall_found = any(
|
||||
re.search(p, line, re.IGNORECASE)
|
||||
for line in lines for p in fall_patterns
|
||||
)
|
||||
if fall_found:
|
||||
report.add("Fall detection", Severity.PASS, "Fall event detected in output")
|
||||
else:
|
||||
report.add("Fall detection", Severity.SKIP,
|
||||
"No fall event (expected if fall scenario not run)")
|
||||
|
||||
# ---- Check 9: MAC filter ----
|
||||
mac_patterns = [r"MAC filter", r"mac_filter", r"dropped.*MAC",
|
||||
r"filter_mac", r"filtered"]
|
||||
mac_found = any(
|
||||
re.search(p, line, re.IGNORECASE)
|
||||
for line in lines for p in mac_patterns
|
||||
)
|
||||
if mac_found:
|
||||
report.add("MAC filter", Severity.PASS, "MAC filter activity detected")
|
||||
else:
|
||||
report.add("MAC filter", Severity.SKIP,
|
||||
"No MAC filter activity (expected if filter scenario not run)")
|
||||
|
||||
# ---- Check 10: ADR-018 serialize ----
|
||||
serialize_patterns = [r"[Ss]erializ", r"ADR-018", r"stream_sender",
|
||||
r"UDP.*send", r"udp.*sent"]
|
||||
serialize_count = sum(1 for line in lines
|
||||
if any(re.search(p, line) for p in serialize_patterns))
|
||||
if serialize_count > 0:
|
||||
report.add("ADR-018 serialize", Severity.PASS,
|
||||
"Serialization/streaming activity detected", count=serialize_count)
|
||||
else:
|
||||
report.add("ADR-018 serialize", Severity.WARN,
|
||||
"No serialization activity detected")
|
||||
|
||||
# ---- Check 11: No crash ----
|
||||
crash_patterns = [r"Guru Meditation", r"assert failed", r"abort\(\)",
|
||||
r"panic", r"LoadProhibited", r"StoreProhibited",
|
||||
r"InstrFetchProhibited", r"IllegalInstruction"]
|
||||
crash_found = []
|
||||
for line in lines:
|
||||
for pat in crash_patterns:
|
||||
if re.search(pat, line):
|
||||
crash_found.append(line.strip()[:120])
|
||||
|
||||
if not crash_found:
|
||||
report.add("No crash", Severity.PASS, "No crash indicators found")
|
||||
else:
|
||||
report.add("No crash", Severity.FATAL,
|
||||
f"Crash detected: {crash_found[0]}",
|
||||
count=len(crash_found))
|
||||
|
||||
# ---- Check 12: Heap OK ----
|
||||
heap_patterns = [r"HEAP_ERROR", r"out of memory", r"heap_caps_alloc.*failed",
|
||||
r"malloc.*fail", r"heap corruption"]
|
||||
heap_errors = [line.strip()[:120] for line in lines
|
||||
if any(re.search(p, line, re.IGNORECASE) for p in heap_patterns)]
|
||||
if not heap_errors:
|
||||
report.add("Heap OK", Severity.PASS, "No heap errors found")
|
||||
else:
|
||||
report.add("Heap OK", Severity.ERROR,
|
||||
f"Heap error: {heap_errors[0]}",
|
||||
count=len(heap_errors))
|
||||
|
||||
# ---- Check 13: Stack OK ----
|
||||
stack_patterns = [r"[Ss]tack overflow", r"stack_overflow",
|
||||
r"vApplicationStackOverflowHook"]
|
||||
stack_errors = [line.strip()[:120] for line in lines
|
||||
if any(re.search(p, line) for p in stack_patterns)]
|
||||
if not stack_errors:
|
||||
report.add("Stack OK", Severity.PASS, "No stack overflow detected")
|
||||
else:
|
||||
report.add("Stack OK", Severity.FATAL,
|
||||
f"Stack overflow: {stack_errors[0]}",
|
||||
count=len(stack_errors))
|
||||
|
||||
# ---- Check 14: Clean exit ----
|
||||
reboot_patterns = [r"Rebooting\.\.\.", r"rst:0x"]
|
||||
reboot_found = any(
|
||||
re.search(p, line)
|
||||
for line in lines for p in reboot_patterns
|
||||
)
|
||||
if not reboot_found:
|
||||
report.add("Clean exit", Severity.PASS,
|
||||
"No unexpected reboot detected")
|
||||
else:
|
||||
report.add("Clean exit", Severity.WARN,
|
||||
"Reboot detected (may indicate crash or watchdog)")
|
||||
|
||||
# ---- Check 15: Scenario completion (when running all scenarios) ----
|
||||
all_scenarios_pattern = r"All (\d+) scenarios complete"
|
||||
scenario_match = re.search(all_scenarios_pattern, log_text)
|
||||
if scenario_match:
|
||||
n_scenarios = int(scenario_match.group(1))
|
||||
report.add("Scenario completion", Severity.PASS,
|
||||
f"All {n_scenarios} scenarios completed", count=n_scenarios)
|
||||
else:
|
||||
# Check if individual scenario started indicators exist
|
||||
scenario_starts = re.findall(r"=== Scenario (\d+) started ===", log_text)
|
||||
if scenario_starts:
|
||||
report.add("Scenario completion", Severity.WARN,
|
||||
f"Started {len(scenario_starts)} scenarios but no completion marker",
|
||||
count=len(scenario_starts))
|
||||
else:
|
||||
report.add("Scenario completion", Severity.SKIP,
|
||||
"No scenario tracking (single scenario or mock not enabled)")
|
||||
|
||||
# ---- Check 16: Frame rate sanity ----
|
||||
# Extract scenario frame counts and check they're reasonable
|
||||
frame_reports = re.findall(r"scenario=\d+ frames=(\d+)", log_text)
|
||||
if frame_reports:
|
||||
max_frames = max(int(f) for f in frame_reports)
|
||||
if max_frames > 0:
|
||||
report.add("Frame rate", Severity.PASS,
|
||||
f"Peak frame counter: {max_frames}", count=max_frames)
|
||||
else:
|
||||
report.add("Frame rate", Severity.ERROR,
|
||||
"Frame counters are all zero")
|
||||
else:
|
||||
report.add("Frame rate", Severity.SKIP,
|
||||
"No periodic frame reports found")
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate QEMU ESP32-S3 UART output (ADR-061)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="Example: python3 validate_qemu_output.py build/qemu_output.log",
|
||||
)
|
||||
parser.add_argument(
|
||||
"log_file",
|
||||
help="Path to QEMU UART log file",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
log_path = Path(args.log_file)
|
||||
if not log_path.exists():
|
||||
print(f"ERROR: Log file not found: {log_path}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
|
||||
log_text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
if not log_text.strip():
|
||||
print("ERROR: Log file is empty. QEMU may have failed to start.",
|
||||
file=sys.stderr)
|
||||
sys.exit(3)
|
||||
|
||||
report = validate_log(log_text)
|
||||
report.print_report()
|
||||
|
||||
# Map max severity to exit code
|
||||
max_sev = report.max_severity
|
||||
if max_sev <= Severity.SKIP:
|
||||
sys.exit(0)
|
||||
elif max_sev == Severity.WARN:
|
||||
sys.exit(1)
|
||||
elif max_sev == Severity.ERROR:
|
||||
sys.exit(2)
|
||||
else:
|
||||
sys.exit(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user