updates

src/services/__init__.py

@@ -0,0 +1,13 @@
+"""
+Services package for WiFi-DensePose API
+"""
+
+from .orchestrator import ServiceOrchestrator
+from .health_check import HealthCheckService
+from .metrics import MetricsService
+
+__all__ = [
+    'ServiceOrchestrator',
+    'HealthCheckService',
+    'MetricsService'
+]

src/services/health_check.py

@@ -0,0 +1,465 @@
+"""
+Health check service for WiFi-DensePose API
+"""
+
+import asyncio
+import logging
+import time
+from typing import Dict, Any, List, Optional
+from datetime import datetime, timedelta
+from dataclasses import dataclass, field
+from enum import Enum
+
+from src.config.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+
+class HealthStatus(Enum):
+    """Health status enumeration."""
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class HealthCheck:
+    """Health check result."""
+    name: str
+    status: HealthStatus
+    message: str
+    timestamp: datetime = field(default_factory=datetime.utcnow)
+    duration_ms: float = 0.0
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ServiceHealth:
+    """Service health information."""
+    name: str
+    status: HealthStatus
+    last_check: Optional[datetime] = None
+    checks: List[HealthCheck] = field(default_factory=list)
+    uptime: float = 0.0
+    error_count: int = 0
+    last_error: Optional[str] = None
+
+
+class HealthCheckService:
+    """Service for monitoring application health."""
+    
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        self._services: Dict[str, ServiceHealth] = {}
+        self._start_time = time.time()
+        self._initialized = False
+        self._running = False
+    
+    async def initialize(self):
+        """Initialize health check service."""
+        if self._initialized:
+            return
+        
+        logger.info("Initializing health check service")
+        
+        # Initialize service health tracking
+        self._services = {
+            "api": ServiceHealth("api", HealthStatus.UNKNOWN),
+            "database": ServiceHealth("database", HealthStatus.UNKNOWN),
+            "redis": ServiceHealth("redis", HealthStatus.UNKNOWN),
+            "hardware": ServiceHealth("hardware", HealthStatus.UNKNOWN),
+            "pose": ServiceHealth("pose", HealthStatus.UNKNOWN),
+            "stream": ServiceHealth("stream", HealthStatus.UNKNOWN),
+        }
+        
+        self._initialized = True
+        logger.info("Health check service initialized")
+    
+    async def start(self):
+        """Start health check service."""
+        if not self._initialized:
+            await self.initialize()
+        
+        self._running = True
+        logger.info("Health check service started")
+    
+    async def shutdown(self):
+        """Shutdown health check service."""
+        self._running = False
+        logger.info("Health check service shut down")
+    
+    async def perform_health_checks(self) -> Dict[str, HealthCheck]:
+        """Perform all health checks."""
+        if not self._running:
+            return {}
+        
+        logger.debug("Performing health checks")
+        results = {}
+        
+        # Perform individual health checks
+        checks = [
+            self._check_api_health(),
+            self._check_database_health(),
+            self._check_redis_health(),
+            self._check_hardware_health(),
+            self._check_pose_health(),
+            self._check_stream_health(),
+        ]
+        
+        # Run checks concurrently
+        check_results = await asyncio.gather(*checks, return_exceptions=True)
+        
+        # Process results
+        for i, result in enumerate(check_results):
+            check_name = ["api", "database", "redis", "hardware", "pose", "stream"][i]
+            
+            if isinstance(result, Exception):
+                health_check = HealthCheck(
+                    name=check_name,
+                    status=HealthStatus.UNHEALTHY,
+                    message=f"Health check failed: {result}"
+                )
+            else:
+                health_check = result
+            
+            results[check_name] = health_check
+            self._update_service_health(check_name, health_check)
+        
+        logger.debug(f"Completed {len(results)} health checks")
+        return results
+    
+    async def _check_api_health(self) -> HealthCheck:
+        """Check API health."""
+        start_time = time.time()
+        
+        try:
+            # Basic API health check
+            uptime = time.time() - self._start_time
+            
+            status = HealthStatus.HEALTHY
+            message = "API is running normally"
+            details = {
+                "uptime_seconds": uptime,
+                "uptime_formatted": str(timedelta(seconds=int(uptime)))
+            }
+            
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"API health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="api",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    async def _check_database_health(self) -> HealthCheck:
+        """Check database health."""
+        start_time = time.time()
+        
+        try:
+            # Import here to avoid circular imports
+            from src.database.connection import get_database_manager
+            
+            db_manager = get_database_manager()
+            
+            if not db_manager.is_connected():
+                status = HealthStatus.UNHEALTHY
+                message = "Database is not connected"
+                details = {"connected": False}
+            else:
+                # Test database connection
+                await db_manager.test_connection()
+                
+                status = HealthStatus.HEALTHY
+                message = "Database is connected and responsive"
+                details = {
+                    "connected": True,
+                    "pool_size": db_manager.get_pool_size(),
+                    "active_connections": db_manager.get_active_connections()
+                }
+        
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"Database health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="database",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    async def _check_redis_health(self) -> HealthCheck:
+        """Check Redis health."""
+        start_time = time.time()
+        
+        try:
+            redis_config = self.settings.get_redis_url()
+            
+            if not redis_config:
+                status = HealthStatus.UNKNOWN
+                message = "Redis is not configured"
+                details = {"configured": False}
+            else:
+                # Test Redis connection
+                import redis.asyncio as redis
+                
+                redis_client = redis.from_url(redis_config)
+                await redis_client.ping()
+                await redis_client.close()
+                
+                status = HealthStatus.HEALTHY
+                message = "Redis is connected and responsive"
+                details = {"connected": True}
+        
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"Redis health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="redis",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    async def _check_hardware_health(self) -> HealthCheck:
+        """Check hardware service health."""
+        start_time = time.time()
+        
+        try:
+            # Import here to avoid circular imports
+            from src.api.dependencies import get_hardware_service
+            
+            hardware_service = get_hardware_service()
+            
+            if hasattr(hardware_service, 'get_status'):
+                status_info = await hardware_service.get_status()
+                
+                if status_info.get("status") == "healthy":
+                    status = HealthStatus.HEALTHY
+                    message = "Hardware service is operational"
+                else:
+                    status = HealthStatus.DEGRADED
+                    message = f"Hardware service status: {status_info.get('status', 'unknown')}"
+                
+                details = status_info
+            else:
+                status = HealthStatus.UNKNOWN
+                message = "Hardware service status unavailable"
+                details = {}
+        
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"Hardware health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="hardware",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    async def _check_pose_health(self) -> HealthCheck:
+        """Check pose service health."""
+        start_time = time.time()
+        
+        try:
+            # Import here to avoid circular imports
+            from src.api.dependencies import get_pose_service
+            
+            pose_service = get_pose_service()
+            
+            if hasattr(pose_service, 'get_status'):
+                status_info = await pose_service.get_status()
+                
+                if status_info.get("status") == "healthy":
+                    status = HealthStatus.HEALTHY
+                    message = "Pose service is operational"
+                else:
+                    status = HealthStatus.DEGRADED
+                    message = f"Pose service status: {status_info.get('status', 'unknown')}"
+                
+                details = status_info
+            else:
+                status = HealthStatus.UNKNOWN
+                message = "Pose service status unavailable"
+                details = {}
+        
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"Pose health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="pose",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    async def _check_stream_health(self) -> HealthCheck:
+        """Check stream service health."""
+        start_time = time.time()
+        
+        try:
+            # Import here to avoid circular imports
+            from src.api.dependencies import get_stream_service
+            
+            stream_service = get_stream_service()
+            
+            if hasattr(stream_service, 'get_status'):
+                status_info = await stream_service.get_status()
+                
+                if status_info.get("status") == "healthy":
+                    status = HealthStatus.HEALTHY
+                    message = "Stream service is operational"
+                else:
+                    status = HealthStatus.DEGRADED
+                    message = f"Stream service status: {status_info.get('status', 'unknown')}"
+                
+                details = status_info
+            else:
+                status = HealthStatus.UNKNOWN
+                message = "Stream service status unavailable"
+                details = {}
+        
+        except Exception as e:
+            status = HealthStatus.UNHEALTHY
+            message = f"Stream health check failed: {e}"
+            details = {"error": str(e)}
+        
+        duration_ms = (time.time() - start_time) * 1000
+        
+        return HealthCheck(
+            name="stream",
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            details=details
+        )
+    
+    def _update_service_health(self, service_name: str, health_check: HealthCheck):
+        """Update service health information."""
+        if service_name not in self._services:
+            self._services[service_name] = ServiceHealth(service_name, HealthStatus.UNKNOWN)
+        
+        service_health = self._services[service_name]
+        service_health.status = health_check.status
+        service_health.last_check = health_check.timestamp
+        service_health.uptime = time.time() - self._start_time
+        
+        # Keep last 10 checks
+        service_health.checks.append(health_check)
+        if len(service_health.checks) > 10:
+            service_health.checks.pop(0)
+        
+        # Update error tracking
+        if health_check.status == HealthStatus.UNHEALTHY:
+            service_health.error_count += 1
+            service_health.last_error = health_check.message
+    
+    async def get_overall_health(self) -> Dict[str, Any]:
+        """Get overall system health."""
+        if not self._services:
+            return {
+                "status": HealthStatus.UNKNOWN.value,
+                "message": "Health checks not initialized"
+            }
+        
+        # Determine overall status
+        statuses = [service.status for service in self._services.values()]
+        
+        if all(status == HealthStatus.HEALTHY for status in statuses):
+            overall_status = HealthStatus.HEALTHY
+            message = "All services are healthy"
+        elif any(status == HealthStatus.UNHEALTHY for status in statuses):
+            overall_status = HealthStatus.UNHEALTHY
+            unhealthy_services = [
+                name for name, service in self._services.items()
+                if service.status == HealthStatus.UNHEALTHY
+            ]
+            message = f"Unhealthy services: {', '.join(unhealthy_services)}"
+        elif any(status == HealthStatus.DEGRADED for status in statuses):
+            overall_status = HealthStatus.DEGRADED
+            degraded_services = [
+                name for name, service in self._services.items()
+                if service.status == HealthStatus.DEGRADED
+            ]
+            message = f"Degraded services: {', '.join(degraded_services)}"
+        else:
+            overall_status = HealthStatus.UNKNOWN
+            message = "System health status unknown"
+        
+        return {
+            "status": overall_status.value,
+            "message": message,
+            "timestamp": datetime.utcnow().isoformat(),
+            "uptime": time.time() - self._start_time,
+            "services": {
+                name: {
+                    "status": service.status.value,
+                    "last_check": service.last_check.isoformat() if service.last_check else None,
+                    "error_count": service.error_count,
+                    "last_error": service.last_error
+                }
+                for name, service in self._services.items()
+            }
+        }
+    
+    async def get_service_health(self, service_name: str) -> Optional[Dict[str, Any]]:
+        """Get health information for a specific service."""
+        service = self._services.get(service_name)
+        if not service:
+            return None
+        
+        return {
+            "name": service.name,
+            "status": service.status.value,
+            "last_check": service.last_check.isoformat() if service.last_check else None,
+            "uptime": service.uptime,
+            "error_count": service.error_count,
+            "last_error": service.last_error,
+            "recent_checks": [
+                {
+                    "timestamp": check.timestamp.isoformat(),
+                    "status": check.status.value,
+                    "message": check.message,
+                    "duration_ms": check.duration_ms,
+                    "details": check.details
+                }
+                for check in service.checks[-5:]  # Last 5 checks
+            ]
+        }
+    
+    async def get_status(self) -> Dict[str, Any]:
+        """Get health check service status."""
+        return {
+            "status": "healthy" if self._running else "stopped",
+            "initialized": self._initialized,
+            "running": self._running,
+            "services_monitored": len(self._services),
+            "uptime": time.time() - self._start_time
+        }

src/services/metrics.py

@@ -0,0 +1,431 @@
+"""
+Metrics collection service for WiFi-DensePose API
+"""
+
+import asyncio
+import logging
+import time
+import psutil
+from typing import Dict, Any, List, Optional
+from datetime import datetime, timedelta
+from dataclasses import dataclass, field
+from collections import defaultdict, deque
+
+from src.config.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MetricPoint:
+    """Single metric data point."""
+    timestamp: datetime
+    value: float
+    labels: Dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class MetricSeries:
+    """Time series of metric points."""
+    name: str
+    description: str
+    unit: str
+    points: deque = field(default_factory=lambda: deque(maxlen=1000))
+    
+    def add_point(self, value: float, labels: Optional[Dict[str, str]] = None):
+        """Add a metric point."""
+        point = MetricPoint(
+            timestamp=datetime.utcnow(),
+            value=value,
+            labels=labels or {}
+        )
+        self.points.append(point)
+    
+    def get_latest(self) -> Optional[MetricPoint]:
+        """Get the latest metric point."""
+        return self.points[-1] if self.points else None
+    
+    def get_average(self, duration: timedelta) -> Optional[float]:
+        """Get average value over a time duration."""
+        cutoff = datetime.utcnow() - duration
+        relevant_points = [
+            point for point in self.points
+            if point.timestamp >= cutoff
+        ]
+        
+        if not relevant_points:
+            return None
+        
+        return sum(point.value for point in relevant_points) / len(relevant_points)
+    
+    def get_max(self, duration: timedelta) -> Optional[float]:
+        """Get maximum value over a time duration."""
+        cutoff = datetime.utcnow() - duration
+        relevant_points = [
+            point for point in self.points
+            if point.timestamp >= cutoff
+        ]
+        
+        if not relevant_points:
+            return None
+        
+        return max(point.value for point in relevant_points)
+
+
+class MetricsService:
+    """Service for collecting and managing application metrics."""
+    
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        self._metrics: Dict[str, MetricSeries] = {}
+        self._counters: Dict[str, float] = defaultdict(float)
+        self._gauges: Dict[str, float] = {}
+        self._histograms: Dict[str, List[float]] = defaultdict(list)
+        self._start_time = time.time()
+        self._initialized = False
+        self._running = False
+        
+        # Initialize standard metrics
+        self._initialize_standard_metrics()
+    
+    def _initialize_standard_metrics(self):
+        """Initialize standard system and application metrics."""
+        self._metrics.update({
+            # System metrics
+            "system_cpu_usage": MetricSeries(
+                "system_cpu_usage", "System CPU usage percentage", "percent"
+            ),
+            "system_memory_usage": MetricSeries(
+                "system_memory_usage", "System memory usage percentage", "percent"
+            ),
+            "system_disk_usage": MetricSeries(
+                "system_disk_usage", "System disk usage percentage", "percent"
+            ),
+            "system_network_bytes_sent": MetricSeries(
+                "system_network_bytes_sent", "Network bytes sent", "bytes"
+            ),
+            "system_network_bytes_recv": MetricSeries(
+                "system_network_bytes_recv", "Network bytes received", "bytes"
+            ),
+            
+            # Application metrics
+            "app_requests_total": MetricSeries(
+                "app_requests_total", "Total HTTP requests", "count"
+            ),
+            "app_request_duration": MetricSeries(
+                "app_request_duration", "HTTP request duration", "seconds"
+            ),
+            "app_active_connections": MetricSeries(
+                "app_active_connections", "Active WebSocket connections", "count"
+            ),
+            "app_pose_detections": MetricSeries(
+                "app_pose_detections", "Pose detections performed", "count"
+            ),
+            "app_pose_processing_time": MetricSeries(
+                "app_pose_processing_time", "Pose processing time", "seconds"
+            ),
+            "app_csi_data_points": MetricSeries(
+                "app_csi_data_points", "CSI data points processed", "count"
+            ),
+            "app_stream_fps": MetricSeries(
+                "app_stream_fps", "Streaming frames per second", "fps"
+            ),
+            
+            # Error metrics
+            "app_errors_total": MetricSeries(
+                "app_errors_total", "Total application errors", "count"
+            ),
+            "app_http_errors": MetricSeries(
+                "app_http_errors", "HTTP errors", "count"
+            ),
+        })
+    
+    async def initialize(self):
+        """Initialize metrics service."""
+        if self._initialized:
+            return
+        
+        logger.info("Initializing metrics service")
+        self._initialized = True
+        logger.info("Metrics service initialized")
+    
+    async def start(self):
+        """Start metrics service."""
+        if not self._initialized:
+            await self.initialize()
+        
+        self._running = True
+        logger.info("Metrics service started")
+    
+    async def shutdown(self):
+        """Shutdown metrics service."""
+        self._running = False
+        logger.info("Metrics service shut down")
+    
+    async def collect_metrics(self):
+        """Collect all metrics."""
+        if not self._running:
+            return
+        
+        logger.debug("Collecting metrics")
+        
+        # Collect system metrics
+        await self._collect_system_metrics()
+        
+        # Collect application metrics
+        await self._collect_application_metrics()
+        
+        logger.debug("Metrics collection completed")
+    
+    async def _collect_system_metrics(self):
+        """Collect system-level metrics."""
+        try:
+            # CPU usage
+            cpu_percent = psutil.cpu_percent(interval=1)
+            self._metrics["system_cpu_usage"].add_point(cpu_percent)
+            
+            # Memory usage
+            memory = psutil.virtual_memory()
+            self._metrics["system_memory_usage"].add_point(memory.percent)
+            
+            # Disk usage
+            disk = psutil.disk_usage('/')
+            disk_percent = (disk.used / disk.total) * 100
+            self._metrics["system_disk_usage"].add_point(disk_percent)
+            
+            # Network I/O
+            network = psutil.net_io_counters()
+            self._metrics["system_network_bytes_sent"].add_point(network.bytes_sent)
+            self._metrics["system_network_bytes_recv"].add_point(network.bytes_recv)
+            
+        except Exception as e:
+            logger.error(f"Error collecting system metrics: {e}")
+    
+    async def _collect_application_metrics(self):
+        """Collect application-specific metrics."""
+        try:
+            # Import here to avoid circular imports
+            from src.api.websocket.connection_manager import connection_manager
+            
+            # Active connections
+            connection_stats = await connection_manager.get_connection_stats()
+            active_connections = connection_stats.get("active_connections", 0)
+            self._metrics["app_active_connections"].add_point(active_connections)
+            
+            # Update counters as metrics
+            for name, value in self._counters.items():
+                if name in self._metrics:
+                    self._metrics[name].add_point(value)
+            
+            # Update gauges as metrics
+            for name, value in self._gauges.items():
+                if name in self._metrics:
+                    self._metrics[name].add_point(value)
+            
+        except Exception as e:
+            logger.error(f"Error collecting application metrics: {e}")
+    
+    def increment_counter(self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None):
+        """Increment a counter metric."""
+        self._counters[name] += value
+        
+        if name in self._metrics:
+            self._metrics[name].add_point(self._counters[name], labels)
+    
+    def set_gauge(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
+        """Set a gauge metric value."""
+        self._gauges[name] = value
+        
+        if name in self._metrics:
+            self._metrics[name].add_point(value, labels)
+    
+    def record_histogram(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
+        """Record a histogram value."""
+        self._histograms[name].append(value)
+        
+        # Keep only last 1000 values
+        if len(self._histograms[name]) > 1000:
+            self._histograms[name] = self._histograms[name][-1000:]
+        
+        if name in self._metrics:
+            self._metrics[name].add_point(value, labels)
+    
+    def time_function(self, metric_name: str):
+        """Decorator to time function execution."""
+        def decorator(func):
+            import functools
+            
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                start_time = time.time()
+                try:
+                    result = await func(*args, **kwargs)
+                    return result
+                finally:
+                    duration = time.time() - start_time
+                    self.record_histogram(metric_name, duration)
+            
+            @functools.wraps(func)
+            def sync_wrapper(*args, **kwargs):
+                start_time = time.time()
+                try:
+                    result = func(*args, **kwargs)
+                    return result
+                finally:
+                    duration = time.time() - start_time
+                    self.record_histogram(metric_name, duration)
+            
+            return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper
+        
+        return decorator
+    
+    def get_metric(self, name: str) -> Optional[MetricSeries]:
+        """Get a metric series by name."""
+        return self._metrics.get(name)
+    
+    def get_metric_value(self, name: str) -> Optional[float]:
+        """Get the latest value of a metric."""
+        metric = self._metrics.get(name)
+        if metric:
+            latest = metric.get_latest()
+            return latest.value if latest else None
+        return None
+    
+    def get_counter_value(self, name: str) -> float:
+        """Get current counter value."""
+        return self._counters.get(name, 0.0)
+    
+    def get_gauge_value(self, name: str) -> Optional[float]:
+        """Get current gauge value."""
+        return self._gauges.get(name)
+    
+    def get_histogram_stats(self, name: str) -> Dict[str, float]:
+        """Get histogram statistics."""
+        values = self._histograms.get(name, [])
+        if not values:
+            return {}
+        
+        sorted_values = sorted(values)
+        count = len(sorted_values)
+        
+        return {
+            "count": count,
+            "sum": sum(sorted_values),
+            "min": sorted_values[0],
+            "max": sorted_values[-1],
+            "mean": sum(sorted_values) / count,
+            "p50": sorted_values[int(count * 0.5)],
+            "p90": sorted_values[int(count * 0.9)],
+            "p95": sorted_values[int(count * 0.95)],
+            "p99": sorted_values[int(count * 0.99)],
+        }
+    
+    async def get_all_metrics(self) -> Dict[str, Any]:
+        """Get all current metrics."""
+        metrics = {}
+        
+        # Current metric values
+        for name, metric_series in self._metrics.items():
+            latest = metric_series.get_latest()
+            if latest:
+                metrics[name] = {
+                    "value": latest.value,
+                    "timestamp": latest.timestamp.isoformat(),
+                    "description": metric_series.description,
+                    "unit": metric_series.unit,
+                    "labels": latest.labels
+                }
+        
+        # Counter values
+        metrics.update({
+            f"counter_{name}": value
+            for name, value in self._counters.items()
+        })
+        
+        # Gauge values
+        metrics.update({
+            f"gauge_{name}": value
+            for name, value in self._gauges.items()
+        })
+        
+        # Histogram statistics
+        for name, values in self._histograms.items():
+            if values:
+                stats = self.get_histogram_stats(name)
+                metrics[f"histogram_{name}"] = stats
+        
+        return metrics
+    
+    async def get_system_metrics(self) -> Dict[str, Any]:
+        """Get system metrics summary."""
+        return {
+            "cpu_usage": self.get_metric_value("system_cpu_usage"),
+            "memory_usage": self.get_metric_value("system_memory_usage"),
+            "disk_usage": self.get_metric_value("system_disk_usage"),
+            "network_bytes_sent": self.get_metric_value("system_network_bytes_sent"),
+            "network_bytes_recv": self.get_metric_value("system_network_bytes_recv"),
+        }
+    
+    async def get_application_metrics(self) -> Dict[str, Any]:
+        """Get application metrics summary."""
+        return {
+            "requests_total": self.get_counter_value("app_requests_total"),
+            "active_connections": self.get_metric_value("app_active_connections"),
+            "pose_detections": self.get_counter_value("app_pose_detections"),
+            "csi_data_points": self.get_counter_value("app_csi_data_points"),
+            "errors_total": self.get_counter_value("app_errors_total"),
+            "uptime_seconds": time.time() - self._start_time,
+            "request_duration_stats": self.get_histogram_stats("app_request_duration"),
+            "pose_processing_time_stats": self.get_histogram_stats("app_pose_processing_time"),
+        }
+    
+    async def get_performance_summary(self) -> Dict[str, Any]:
+        """Get performance metrics summary."""
+        one_hour = timedelta(hours=1)
+        
+        return {
+            "system": {
+                "cpu_avg_1h": self._metrics["system_cpu_usage"].get_average(one_hour),
+                "memory_avg_1h": self._metrics["system_memory_usage"].get_average(one_hour),
+                "cpu_max_1h": self._metrics["system_cpu_usage"].get_max(one_hour),
+                "memory_max_1h": self._metrics["system_memory_usage"].get_max(one_hour),
+            },
+            "application": {
+                "avg_request_duration": self.get_histogram_stats("app_request_duration").get("mean"),
+                "avg_pose_processing_time": self.get_histogram_stats("app_pose_processing_time").get("mean"),
+                "total_requests": self.get_counter_value("app_requests_total"),
+                "total_errors": self.get_counter_value("app_errors_total"),
+                "error_rate": (
+                    self.get_counter_value("app_errors_total") / 
+                    max(self.get_counter_value("app_requests_total"), 1)
+                ) * 100,
+            }
+        }
+    
+    async def get_status(self) -> Dict[str, Any]:
+        """Get metrics service status."""
+        return {
+            "status": "healthy" if self._running else "stopped",
+            "initialized": self._initialized,
+            "running": self._running,
+            "metrics_count": len(self._metrics),
+            "counters_count": len(self._counters),
+            "gauges_count": len(self._gauges),
+            "histograms_count": len(self._histograms),
+            "uptime": time.time() - self._start_time
+        }
+    
+    def reset_metrics(self):
+        """Reset all metrics."""
+        logger.info("Resetting all metrics")
+        
+        # Clear metric points but keep series definitions
+        for metric_series in self._metrics.values():
+            metric_series.points.clear()
+        
+        # Reset counters, gauges, and histograms
+        self._counters.clear()
+        self._gauges.clear()
+        self._histograms.clear()
+        
+        logger.info("All metrics reset")

src/services/orchestrator.py

@@ -0,0 +1,395 @@
+"""
+Main service orchestrator for WiFi-DensePose API
+"""
+
+import asyncio
+import logging
+from typing import Dict, Any, List, Optional
+from contextlib import asynccontextmanager
+
+from src.config.settings import Settings
+from src.services.health_check import HealthCheckService
+from src.services.metrics import MetricsService
+from src.api.dependencies import (
+    get_hardware_service,
+    get_pose_service,
+    get_stream_service
+)
+from src.api.websocket.connection_manager import connection_manager
+from src.api.websocket.pose_stream import PoseStreamHandler
+
+logger = logging.getLogger(__name__)
+
+
+class ServiceOrchestrator:
+    """Main service orchestrator that manages all application services."""
+    
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        self._services: Dict[str, Any] = {}
+        self._background_tasks: List[asyncio.Task] = []
+        self._initialized = False
+        self._started = False
+        
+        # Core services
+        self.health_service = HealthCheckService(settings)
+        self.metrics_service = MetricsService(settings)
+        
+        # Application services (will be initialized later)
+        self.hardware_service = None
+        self.pose_service = None
+        self.stream_service = None
+        self.pose_stream_handler = None
+    
+    async def initialize(self):
+        """Initialize all services."""
+        if self._initialized:
+            logger.warning("Services already initialized")
+            return
+        
+        logger.info("Initializing services...")
+        
+        try:
+            # Initialize core services
+            await self.health_service.initialize()
+            await self.metrics_service.initialize()
+            
+            # Initialize application services
+            await self._initialize_application_services()
+            
+            # Store services in registry
+            self._services = {
+                'health': self.health_service,
+                'metrics': self.metrics_service,
+                'hardware': self.hardware_service,
+                'pose': self.pose_service,
+                'stream': self.stream_service,
+                'pose_stream_handler': self.pose_stream_handler,
+                'connection_manager': connection_manager
+            }
+            
+            self._initialized = True
+            logger.info("All services initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize services: {e}")
+            await self.shutdown()
+            raise
+    
+    async def _initialize_application_services(self):
+        """Initialize application-specific services."""
+        try:
+            # Initialize hardware service
+            self.hardware_service = get_hardware_service()
+            await self.hardware_service.initialize()
+            logger.info("Hardware service initialized")
+            
+            # Initialize pose service
+            self.pose_service = get_pose_service()
+            await self.pose_service.initialize()
+            logger.info("Pose service initialized")
+            
+            # Initialize stream service
+            self.stream_service = get_stream_service()
+            await self.stream_service.initialize()
+            logger.info("Stream service initialized")
+            
+            # Initialize pose stream handler
+            self.pose_stream_handler = PoseStreamHandler(
+                connection_manager=connection_manager,
+                pose_service=self.pose_service,
+                stream_service=self.stream_service
+            )
+            logger.info("Pose stream handler initialized")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize application services: {e}")
+            raise
+    
+    async def start(self):
+        """Start all services and background tasks."""
+        if not self._initialized:
+            await self.initialize()
+        
+        if self._started:
+            logger.warning("Services already started")
+            return
+        
+        logger.info("Starting services...")
+        
+        try:
+            # Start core services
+            await self.health_service.start()
+            await self.metrics_service.start()
+            
+            # Start application services
+            await self._start_application_services()
+            
+            # Start background tasks
+            await self._start_background_tasks()
+            
+            self._started = True
+            logger.info("All services started successfully")
+            
+        except Exception as e:
+            logger.error(f"Failed to start services: {e}")
+            await self.shutdown()
+            raise
+    
+    async def _start_application_services(self):
+        """Start application-specific services."""
+        try:
+            # Start hardware service
+            if hasattr(self.hardware_service, 'start'):
+                await self.hardware_service.start()
+            
+            # Start pose service
+            if hasattr(self.pose_service, 'start'):
+                await self.pose_service.start()
+            
+            # Start stream service
+            if hasattr(self.stream_service, 'start'):
+                await self.stream_service.start()
+            
+            logger.info("Application services started")
+            
+        except Exception as e:
+            logger.error(f"Failed to start application services: {e}")
+            raise
+    
+    async def _start_background_tasks(self):
+        """Start background tasks."""
+        try:
+            # Start health check monitoring
+            if self.settings.health_check_interval > 0:
+                task = asyncio.create_task(self._health_check_loop())
+                self._background_tasks.append(task)
+            
+            # Start metrics collection
+            if self.settings.metrics_enabled:
+                task = asyncio.create_task(self._metrics_collection_loop())
+                self._background_tasks.append(task)
+            
+            # Start pose streaming if enabled
+            if self.settings.enable_real_time_processing:
+                await self.pose_stream_handler.start_streaming()
+            
+            logger.info(f"Started {len(self._background_tasks)} background tasks")
+            
+        except Exception as e:
+            logger.error(f"Failed to start background tasks: {e}")
+            raise
+    
+    async def _health_check_loop(self):
+        """Background health check loop."""
+        logger.info("Starting health check loop")
+        
+        while True:
+            try:
+                await self.health_service.perform_health_checks()
+                await asyncio.sleep(self.settings.health_check_interval)
+            except asyncio.CancelledError:
+                logger.info("Health check loop cancelled")
+                break
+            except Exception as e:
+                logger.error(f"Error in health check loop: {e}")
+                await asyncio.sleep(self.settings.health_check_interval)
+    
+    async def _metrics_collection_loop(self):
+        """Background metrics collection loop."""
+        logger.info("Starting metrics collection loop")
+        
+        while True:
+            try:
+                await self.metrics_service.collect_metrics()
+                await asyncio.sleep(60)  # Collect metrics every minute
+            except asyncio.CancelledError:
+                logger.info("Metrics collection loop cancelled")
+                break
+            except Exception as e:
+                logger.error(f"Error in metrics collection loop: {e}")
+                await asyncio.sleep(60)
+    
+    async def shutdown(self):
+        """Shutdown all services and cleanup resources."""
+        logger.info("Shutting down services...")
+        
+        try:
+            # Cancel background tasks
+            for task in self._background_tasks:
+                if not task.done():
+                    task.cancel()
+            
+            if self._background_tasks:
+                await asyncio.gather(*self._background_tasks, return_exceptions=True)
+                self._background_tasks.clear()
+            
+            # Stop pose streaming
+            if self.pose_stream_handler:
+                await self.pose_stream_handler.shutdown()
+            
+            # Shutdown connection manager
+            await connection_manager.shutdown()
+            
+            # Shutdown application services
+            await self._shutdown_application_services()
+            
+            # Shutdown core services
+            await self.health_service.shutdown()
+            await self.metrics_service.shutdown()
+            
+            self._started = False
+            self._initialized = False
+            
+            logger.info("All services shut down successfully")
+            
+        except Exception as e:
+            logger.error(f"Error during shutdown: {e}")
+    
+    async def _shutdown_application_services(self):
+        """Shutdown application-specific services."""
+        try:
+            # Shutdown services in reverse order
+            if self.stream_service and hasattr(self.stream_service, 'shutdown'):
+                await self.stream_service.shutdown()
+            
+            if self.pose_service and hasattr(self.pose_service, 'shutdown'):
+                await self.pose_service.shutdown()
+            
+            if self.hardware_service and hasattr(self.hardware_service, 'shutdown'):
+                await self.hardware_service.shutdown()
+            
+            logger.info("Application services shut down")
+            
+        except Exception as e:
+            logger.error(f"Error shutting down application services: {e}")
+    
+    async def restart_service(self, service_name: str):
+        """Restart a specific service."""
+        logger.info(f"Restarting service: {service_name}")
+        
+        service = self._services.get(service_name)
+        if not service:
+            raise ValueError(f"Service not found: {service_name}")
+        
+        try:
+            # Stop service
+            if hasattr(service, 'stop'):
+                await service.stop()
+            elif hasattr(service, 'shutdown'):
+                await service.shutdown()
+            
+            # Reinitialize service
+            if hasattr(service, 'initialize'):
+                await service.initialize()
+            
+            # Start service
+            if hasattr(service, 'start'):
+                await service.start()
+            
+            logger.info(f"Service restarted successfully: {service_name}")
+            
+        except Exception as e:
+            logger.error(f"Failed to restart service {service_name}: {e}")
+            raise
+    
+    async def reset_services(self):
+        """Reset all services to initial state."""
+        logger.info("Resetting all services")
+        
+        try:
+            # Reset application services
+            if self.hardware_service and hasattr(self.hardware_service, 'reset'):
+                await self.hardware_service.reset()
+            
+            if self.pose_service and hasattr(self.pose_service, 'reset'):
+                await self.pose_service.reset()
+            
+            if self.stream_service and hasattr(self.stream_service, 'reset'):
+                await self.stream_service.reset()
+            
+            # Reset connection manager
+            await connection_manager.reset()
+            
+            logger.info("All services reset successfully")
+            
+        except Exception as e:
+            logger.error(f"Failed to reset services: {e}")
+            raise
+    
+    async def get_service_status(self) -> Dict[str, Any]:
+        """Get status of all services."""
+        status = {}
+        
+        for name, service in self._services.items():
+            try:
+                if hasattr(service, 'get_status'):
+                    status[name] = await service.get_status()
+                else:
+                    status[name] = {"status": "unknown"}
+            except Exception as e:
+                status[name] = {"status": "error", "error": str(e)}
+        
+        return status
+    
+    async def get_service_metrics(self) -> Dict[str, Any]:
+        """Get metrics from all services."""
+        metrics = {}
+        
+        for name, service in self._services.items():
+            try:
+                if hasattr(service, 'get_metrics'):
+                    metrics[name] = await service.get_metrics()
+                elif hasattr(service, 'get_performance_metrics'):
+                    metrics[name] = await service.get_performance_metrics()
+            except Exception as e:
+                logger.error(f"Failed to get metrics from {name}: {e}")
+                metrics[name] = {"error": str(e)}
+        
+        return metrics
+    
+    async def get_service_info(self) -> Dict[str, Any]:
+        """Get information about all services."""
+        info = {
+            "total_services": len(self._services),
+            "initialized": self._initialized,
+            "started": self._started,
+            "background_tasks": len(self._background_tasks),
+            "services": {}
+        }
+        
+        for name, service in self._services.items():
+            service_info = {
+                "type": type(service).__name__,
+                "module": type(service).__module__
+            }
+            
+            # Add service-specific info if available
+            if hasattr(service, 'get_info'):
+                try:
+                    service_info.update(await service.get_info())
+                except Exception as e:
+                    service_info["error"] = str(e)
+            
+            info["services"][name] = service_info
+        
+        return info
+    
+    def get_service(self, name: str) -> Optional[Any]:
+        """Get a specific service by name."""
+        return self._services.get(name)
+    
+    @property
+    def is_healthy(self) -> bool:
+        """Check if all services are healthy."""
+        return self._initialized and self._started
+    
+    @asynccontextmanager
+    async def service_context(self):
+        """Context manager for service lifecycle."""
+        try:
+            await self.initialize()
+            await self.start()
+            yield self
+        finally:
+            await self.shutdown()