ruvnet--RuView/vendor/ruvector/crates/ruvllm/src/serving/engine.rs

//! Serving Engine for Continuous Batching
//!
//! This module provides the main serving engine that coordinates
//! request submission, scheduling, and model execution with streaming output.

use super::kv_cache_manager::KvCachePoolConfig;
use super::request::{
    CompletedRequest, FinishReason, InferenceRequest, Priority, RequestId, RequestState,
    RunningRequest, TokenOutput,
};
use super::scheduler::{ContinuousBatchScheduler, RequestQueue, SchedulerConfig};
use crate::backends::{GenerateParams, GeneratedToken, LlmBackend};
use crate::error::{Result, RuvLLMError};
use crate::optimization::realtime::RealtimeOptimizer;
use crate::speculative::{SpeculativeConfig, SpeculativeDecoder};
use parking_lot::{Mutex, RwLock};
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};

#[cfg(feature = "async-runtime")]
use tokio::sync::mpsc;

/// Configuration for the serving engine
#[derive(Debug, Clone)]
pub struct ServingEngineConfig {
    /// Scheduler configuration
    pub scheduler: SchedulerConfig,
    /// KV cache pool configuration
    pub kv_cache: KvCachePoolConfig,
    /// Maximum concurrent requests
    pub max_concurrent_requests: usize,
    /// Enable request coalescing
    pub coalesce_requests: bool,
    /// Coalescing window in milliseconds
    pub coalesce_window_ms: u64,
    /// Enable streaming output
    pub streaming_enabled: bool,
    /// Request timeout in milliseconds
    pub request_timeout_ms: u64,
    /// Enable speculative decoding (default: true for 2-3x speedup)
    pub enable_speculative: bool,
    /// Speculative decoding configuration
    pub speculative_config: SpeculativeConfig,
    /// Draft model path for speculative decoding (auto-detected if None)
    /// - For 7B+ models: use 1B draft (e.g., "TinyLlama/TinyLlama-1.1B-Chat-v1.0")
    /// - For 3B models: use 0.5B draft (e.g., "Qwen/Qwen2.5-0.5B")
    pub draft_model_path: Option<String>,
}

impl Default for ServingEngineConfig {
    fn default() -> Self {
        Self {
            scheduler: SchedulerConfig::default(),
            kv_cache: KvCachePoolConfig::default(),
            max_concurrent_requests: 256,
            coalesce_requests: false,
            coalesce_window_ms: 10,
            streaming_enabled: true,
            request_timeout_ms: 60000,
            enable_speculative: true, // Enabled by default for 2-3x decode speedup
            speculative_config: SpeculativeConfig::default(),
            draft_model_path: None, // Auto-detected based on main model size
        }
    }
}

/// Result of processing a request
#[derive(Debug, Clone)]
pub struct GenerationResult {
    /// Request ID
    pub request_id: RequestId,
    /// Generated token IDs
    pub generated_tokens: Vec<u32>,
    /// Generated text (if decoded)
    pub generated_text: Option<String>,
    /// Finish reason
    pub finish_reason: FinishReason,
    /// Processing time in milliseconds
    pub processing_time_ms: u64,
    /// Tokens per second
    pub tokens_per_second: f64,
    /// Number of prompt tokens
    pub prompt_tokens: usize,
    /// Number of generated tokens
    pub completion_tokens: usize,
}

impl From<CompletedRequest> for GenerationResult {
    fn from(completed: CompletedRequest) -> Self {
        Self {
            request_id: completed.id,
            generated_tokens: completed.generated_tokens.clone(),
            generated_text: None,
            finish_reason: completed.finish_reason,
            processing_time_ms: completed.processing_time_ms,
            tokens_per_second: completed.tokens_per_second,
            prompt_tokens: completed.prompt_tokens.len(),
            completion_tokens: completed.generated_tokens.len(),
        }
    }
}

/// Streaming token callback
pub type TokenCallback = Box<dyn Fn(TokenOutput) + Send + Sync>;

/// Internal request state for the engine
struct EngineRequest {
    /// Request data
    request: InferenceRequest,
    /// Token callback for streaming
    callback: Option<TokenCallback>,
    /// Completion notifier
    #[cfg(feature = "async-runtime")]
    completion_tx: Option<tokio::sync::oneshot::Sender<GenerationResult>>,
    /// Created time
    created_at: Instant,
}

/// The serving engine for continuous batching
pub struct ServingEngine {
    /// Configuration
    config: ServingEngineConfig,
    /// The LLM backend
    model: Arc<dyn LlmBackend>,
    /// Draft model for speculative decoding (loaded lazily)
    draft_model: RwLock<Option<Arc<dyn LlmBackend>>>,
    /// Request scheduler
    scheduler: Mutex<ContinuousBatchScheduler>,
    /// Request queue
    queue: Mutex<RequestQueue>,
    /// Pending request data
    pending_requests: RwLock<HashMap<RequestId, EngineRequest>>,
    /// Completed results
    completed_results: RwLock<HashMap<RequestId, GenerationResult>>,
    /// Running state
    is_running: AtomicBool,
    /// Total requests processed
    total_requests: AtomicU64,
    /// Total tokens generated
    total_tokens: AtomicU64,
    /// Start time for metrics
    start_time: Instant,
    /// Realtime optimizer for speculative decoding decisions
    optimizer: RealtimeOptimizer,
}

impl ServingEngine {
    /// Create a new serving engine
    pub fn new(model: Arc<dyn LlmBackend>, config: ServingEngineConfig) -> Self {
        use crate::optimization::realtime::RealtimeConfig;

        let scheduler =
            ContinuousBatchScheduler::new(config.scheduler.clone(), config.kv_cache.clone());

        // Create realtime optimizer with speculative decoding enabled by default
        let realtime_config = RealtimeConfig {
            enable_speculative: config.enable_speculative,
            speculative: crate::optimization::realtime::SpeculativeConfig {
                draft_model: config.draft_model_path.clone(),
                num_speculative_tokens: config.speculative_config.lookahead,
                acceptance_threshold: config.speculative_config.acceptance_threshold,
                tree_speculation: config.speculative_config.tree_speculation,
                max_tree_depth: config.speculative_config.max_tree_depth,
            },
            ..Default::default()
        };
        let optimizer = RealtimeOptimizer::new(realtime_config);

        Self {
            config,
            model,
            draft_model: RwLock::new(None),
            scheduler: Mutex::new(scheduler),
            queue: Mutex::new(RequestQueue::new()),
            pending_requests: RwLock::new(HashMap::new()),
            completed_results: RwLock::new(HashMap::new()),
            is_running: AtomicBool::new(false),
            total_requests: AtomicU64::new(0),
            total_tokens: AtomicU64::new(0),
            start_time: Instant::now(),
            optimizer,
        }
    }

    /// Create with default configuration
    pub fn with_default_config(model: Arc<dyn LlmBackend>) -> Self {
        Self::new(model, ServingEngineConfig::default())
    }

    /// Submit a request for processing
    pub fn submit(&self, request: InferenceRequest) -> Result<RequestId> {
        let request_id = request.id;

        // Check capacity
        {
            let queue = self.queue.lock();
            if queue.pending_count() + queue.running_count() >= self.config.max_concurrent_requests
            {
                return Err(RuvLLMError::OutOfMemory(
                    "Maximum concurrent requests reached".to_string(),
                ));
            }
        }

        // Store request data
        {
            let engine_request = EngineRequest {
                request: request.clone(),
                callback: None,
                #[cfg(feature = "async-runtime")]
                completion_tx: None,
                created_at: Instant::now(),
            };
            self.pending_requests
                .write()
                .insert(request_id, engine_request);
        }

        // Add to queue
        self.queue.lock().add(request);
        self.total_requests.fetch_add(1, Ordering::Relaxed);

        Ok(request_id)
    }

    /// Submit a request with a streaming callback
    pub fn submit_with_callback(
        &self,
        request: InferenceRequest,
        callback: TokenCallback,
    ) -> Result<RequestId> {
        let request_id = request.id;

        // Check capacity
        {
            let queue = self.queue.lock();
            if queue.pending_count() + queue.running_count() >= self.config.max_concurrent_requests
            {
                return Err(RuvLLMError::OutOfMemory(
                    "Maximum concurrent requests reached".to_string(),
                ));
            }
        }

        // Store request data with callback
        {
            let engine_request = EngineRequest {
                request: request.clone(),
                callback: Some(callback),
                #[cfg(feature = "async-runtime")]
                completion_tx: None,
                created_at: Instant::now(),
            };
            self.pending_requests
                .write()
                .insert(request_id, engine_request);
        }

        // Add to queue
        self.queue.lock().add(request);
        self.total_requests.fetch_add(1, Ordering::Relaxed);

        Ok(request_id)
    }

    /// Get the result of a completed request
    pub fn get_result(&self, id: RequestId) -> Option<GenerationResult> {
        self.completed_results.write().remove(&id)
    }

    /// Check if a request is complete
    pub fn is_complete(&self, id: RequestId) -> bool {
        self.completed_results.read().contains_key(&id)
    }

    /// Cancel a request
    pub fn cancel(&self, id: RequestId) -> bool {
        // Try to remove from pending
        if self.pending_requests.write().remove(&id).is_some() {
            // Remove from queue if still pending
            let mut queue = self.queue.lock();
            queue.pending.retain(|r| r.id != id);
            return true;
        }

        // Try to remove from running
        let mut queue = self.queue.lock();
        if let Some(running) = queue.remove_running(id) {
            // Free KV cache
            self.scheduler.lock().kv_cache_manager_mut().free(id);

            // Create cancelled result - extract values before moving generated_tokens
            let completion_tokens = running.generated_tokens.len();
            let processing_time_ms = running.processing_time().as_millis() as u64;
            let tokens_per_second = running.tokens_per_second();
            let prompt_tokens = running.request.prompt_len();
            let result = GenerationResult {
                request_id: id,
                generated_tokens: running.generated_tokens,
                generated_text: None,
                finish_reason: FinishReason::Cancelled,
                processing_time_ms,
                tokens_per_second,
                prompt_tokens,
                completion_tokens,
            };

            self.completed_results.write().insert(id, result);
            return true;
        }

        false
    }

    /// Run a single iteration of the serving loop
    ///
    /// Returns the generated tokens for this iteration
    pub fn run_iteration(&self) -> Result<Vec<TokenOutput>> {
        let mut outputs = Vec::new();

        // Schedule next batch
        let batch = {
            let mut queue = self.queue.lock();
            let mut scheduler = self.scheduler.lock();
            scheduler.schedule(&mut queue)
        };

        if batch.is_empty() {
            return Ok(outputs);
        }

        // Process the batch (this is where the actual model inference would happen)
        // For now, we simulate token generation

        // Process each request in the batch
        for batched_req in &batch.requests {
            let request_id = batched_req.request_id;

            if batched_req.is_prefill {
                // Prefill complete - update state
                let mut queue = self.queue.lock();
                if let Some(running) = queue.get_running_mut(request_id) {
                    if !running.prefill_complete {
                        running.advance_prefill(batched_req.token_ids.len());
                    }
                }
            } else {
                // Decode - generate a token using the real model
                let generated_token = {
                    let queue = self.queue.lock();
                    if let Some(running) = queue.running.get(&request_id) {
                        self.generate_next_token(request_id, running)?
                    } else {
                        // Request not found, skip
                        continue;
                    }
                };

                let mut queue = self.queue.lock();

                if let Some(running) = queue.get_running_mut(request_id) {
                    running.add_token(generated_token);

                    // Create output
                    let output = TokenOutput {
                        request_id,
                        token_id: generated_token,
                        token_text: None, // Would decode with tokenizer
                        logprob: None,
                        is_final: running.is_complete(),
                        finish_reason: if running.is_complete() {
                            Some(FinishReason::Length)
                        } else {
                            None
                        },
                        seq_len: running.current_seq_len,
                    };

                    // Send to callback if registered
                    if let Some(engine_req) = self.pending_requests.read().get(&request_id) {
                        if let Some(callback) = &engine_req.callback {
                            callback(output.clone());
                        }
                    }

                    outputs.push(output);

                    // Update KV cache length
                    let _ = self
                        .scheduler
                        .lock()
                        .kv_cache_manager_mut()
                        .set_length(request_id, running.current_seq_len);

                    self.total_tokens.fetch_add(1, Ordering::Relaxed);

                    // Check if complete
                    if running.is_complete() {
                        // Will handle completion below
                    }
                }
            }
        }

        // Handle completions
        self.handle_completions()?;

        Ok(outputs)
    }

    /// Handle completed requests
    fn handle_completions(&self) -> Result<()> {
        let mut completed_ids = Vec::new();

        // Find completed requests
        {
            let queue = self.queue.lock();
            for (id, running) in &queue.running {
                if running.is_complete() {
                    completed_ids.push(*id);
                }
            }
        }

        // Process completions
        for id in completed_ids {
            let running = {
                let mut queue = self.queue.lock();
                queue.remove_running(id)
            };

            if let Some(running) = running {
                // Free KV cache
                self.scheduler.lock().kv_cache_manager_mut().free(id);

                // Create result
                let result = GenerationResult {
                    request_id: id,
                    generated_tokens: running.generated_tokens.clone(),
                    generated_text: None,
                    finish_reason: FinishReason::Length,
                    processing_time_ms: running.processing_time().as_millis() as u64,
                    tokens_per_second: running.tokens_per_second(),
                    prompt_tokens: running.request.prompt_len(),
                    completion_tokens: running.generated_tokens.len(),
                };

                // Store result
                self.completed_results.write().insert(id, result.clone());

                // Send final callback
                if let Some(engine_req) = self.pending_requests.write().remove(&id) {
                    if let Some(callback) = &engine_req.callback {
                        callback(TokenOutput {
                            request_id: id,
                            token_id: running.generated_tokens.last().copied().unwrap_or(0),
                            token_text: None,
                            logprob: None,
                            is_final: true,
                            finish_reason: Some(FinishReason::Length),
                            seq_len: running.current_seq_len,
                        });
                    }

                    #[cfg(feature = "async-runtime")]
                    if let Some(tx) = engine_req.completion_tx {
                        let _ = tx.send(result);
                    }
                }
            }
        }

        Ok(())
    }

    /// Generate next token using the model backend
    ///
    /// This method implements real autoregressive token generation:
    /// 1. Gets the current context (prompt + generated tokens)
    /// 2. Runs a forward pass through the model
    /// 3. Applies sampling (temperature, top-p, top-k)
    /// 4. Uses speculative decoding when available for 2-3x speedup
    ///
    /// # Arguments
    /// * `request_id` - The request ID to generate for
    /// * `running` - The running request state
    ///
    /// # Returns
    /// The generated token ID
    fn generate_next_token(&self, request_id: RequestId, running: &RunningRequest) -> Result<u32> {
        // Build the context: prompt tokens + already generated tokens
        let mut context = running.request.prompt_tokens.clone();
        context.extend(&running.generated_tokens);

        // Get generation parameters from the request
        let params = &running.request.params;

        // Check if we should use speculative decoding
        if self.should_use_speculative(params) {
            if let Some(draft_model) = self.draft_model.read().as_ref() {
                // Speculative decoding available - use it for faster generation
                return self.generate_with_speculation(request_id, &context, params, draft_model);
            }
        }

        // Standard single-token generation via model backend
        self.generate_single_token(&context, params)
    }

    /// Generate a single token using standard autoregressive decoding
    fn generate_single_token(
        &self,
        context: &[u32],
        params: &crate::backends::GenerateParams,
    ) -> Result<u32> {
        // Check if model is loaded - if not, fall back to simulation for testing
        if !self.model.is_model_loaded() {
            // No model loaded - simulate token generation for testing
            // In production this should be an error, but for tests without
            // a real model we return a pseudo-random token based on context
            let hash = context
                .iter()
                .fold(0u32, |acc, &t| acc.wrapping_add(t).wrapping_mul(31));
            return Ok(hash % 32000);
        }

        // Decode context to text for the backend
        let context_text = if let Some(tokenizer) = self.model.tokenizer() {
            tokenizer.decode(context)?
        } else {
            // No tokenizer but model is loaded - try direct generation
            // and extract token from the generated text
            return Err(RuvLLMError::InvalidOperation(
                "No tokenizer available for text decoding".to_string(),
            ));
        };

        // Generate one token using the backend
        let gen_params = crate::backends::GenerateParams {
            max_tokens: 1,
            temperature: params.temperature,
            top_p: params.top_p,
            top_k: params.top_k,
            repetition_penalty: params.repetition_penalty,
            frequency_penalty: params.frequency_penalty,
            presence_penalty: params.presence_penalty,
            stop_sequences: vec![], // Don't stop on sequences for single token
            seed: params.seed,
        };

        // Generate text (single token)
        let generated_text = self.model.generate(&context_text, gen_params)?;

        // Tokenize the result to get the new token
        if let Some(tokenizer) = self.model.tokenizer() {
            let full_text = format!("{}{}", context_text, generated_text);
            let full_tokens = tokenizer.encode(&full_text)?;

            // The new token is at position context.len()
            if full_tokens.len() > context.len() {
                return Ok(full_tokens[context.len()]);
            }

            // If no new token generated, return EOS
            if let Some(eos) = tokenizer.special_tokens().eos_token_id {
                return Ok(eos);
            }
        }

        Err(RuvLLMError::Generation(
            "Failed to generate token".to_string(),
        ))
    }

    /// Generate tokens using speculative decoding for 2-3x speedup
    ///
    /// Speculative decoding works by:
    /// 1. Using a small draft model to predict K tokens ahead
    /// 2. Verifying all K tokens with the main model in a single forward pass
    /// 3. Accepting matching tokens and correcting where they diverge
    fn generate_with_speculation(
        &self,
        _request_id: RequestId,
        context: &[u32],
        params: &crate::backends::GenerateParams,
        draft_model: &Arc<dyn LlmBackend>,
    ) -> Result<u32> {
        let spec_config = &self.config.speculative_config;
        let lookahead = spec_config.lookahead;

        // Get tokenizer for encoding/decoding
        let tokenizer = self
            .model
            .tokenizer()
            .ok_or_else(|| RuvLLMError::InvalidOperation("No tokenizer available".to_string()))?;

        // Decode context to text
        let context_text = tokenizer.decode(context)?;

        // Draft phase: generate K tokens with the small model
        let draft_params = crate::backends::GenerateParams {
            max_tokens: lookahead,
            temperature: spec_config.draft_temperature,
            top_p: spec_config.draft_top_p,
            top_k: if spec_config.draft_temperature == 0.0 {
                1
            } else {
                40
            },
            ..Default::default()
        };

        let draft_text = draft_model.generate(&context_text, draft_params)?;
        let draft_full = format!("{}{}", context_text, draft_text);
        let draft_tokens = tokenizer.encode(&draft_full)?;

        // Extract draft tokens (beyond original context)
        let draft_new: Vec<u32> = draft_tokens
            .iter()
            .skip(context.len())
            .take(lookahead)
            .copied()
            .collect();

        if draft_new.is_empty() {
            // Draft model couldn't generate, fall back to single token
            return self.generate_single_token(context, params);
        }

        // Verify phase: check draft tokens with main model
        // Build context with draft tokens for verification
        let mut verify_context = context.to_vec();

        for (i, &draft_token) in draft_new.iter().enumerate() {
            let verify_text = tokenizer.decode(&verify_context)?;

            let verify_params = crate::backends::GenerateParams {
                max_tokens: 1,
                temperature: params.temperature,
                top_p: params.top_p,
                top_k: params.top_k,
                ..params.clone()
            };

            let main_text = self.model.generate(&verify_text, verify_params)?;
            let main_full = format!("{}{}", verify_text, main_text);
            let main_tokens = tokenizer.encode(&main_full)?;

            if main_tokens.len() <= verify_context.len() {
                // Main model produced nothing, return EOS or use draft
                if let Some(eos) = tokenizer.special_tokens().eos_token_id {
                    return Ok(eos);
                }
                return Ok(draft_token);
            }

            let main_token = main_tokens[verify_context.len()];

            if main_token == draft_token {
                // Accept draft token
                verify_context.push(draft_token);
            } else {
                // Reject - return main model's correction
                // Record stats through optimizer
                self.optimizer.update_speculation_stats(i, draft_new.len());
                return Ok(main_token);
            }
        }

        // All drafts accepted - get one more token from main model
        let final_text = tokenizer.decode(&verify_context)?;
        let final_params = crate::backends::GenerateParams {
            max_tokens: 1,
            temperature: params.temperature,
            top_p: params.top_p,
            top_k: params.top_k,
            ..params.clone()
        };

        let continuation = self.model.generate(&final_text, final_params)?;
        let continuation_full = format!("{}{}", final_text, continuation);
        let continuation_tokens = tokenizer.encode(&continuation_full)?;

        // Record successful speculation
        self.optimizer
            .update_speculation_stats(draft_new.len(), draft_new.len());

        if continuation_tokens.len() > verify_context.len() {
            Ok(continuation_tokens[verify_context.len()])
        } else if let Some(eos) = tokenizer.special_tokens().eos_token_id {
            Ok(eos)
        } else {
            Err(RuvLLMError::Generation(
                "Failed to generate continuation token".to_string(),
            ))
        }
    }

    /// Generate tokens with streaming callback support
    ///
    /// This method generates tokens one at a time, calling the provided
    /// callback for each token. Useful for real-time output display.
    pub fn generate_with_callback<F>(
        &self,
        request: &InferenceRequest,
        mut callback: F,
    ) -> Result<Vec<u32>>
    where
        F: FnMut(TokenOutput) -> bool, // Returns false to stop generation
    {
        let mut context = request.prompt_tokens.clone();
        let mut generated = Vec::new();
        let params = &request.params;

        let eos_token = self
            .model
            .tokenizer()
            .and_then(|t| t.special_tokens().eos_token_id);

        while generated.len() < params.max_tokens {
            // Generate next token
            let token = self.generate_single_token(&context, params)?;

            // Check for EOS
            if Some(token) == eos_token {
                let output = TokenOutput {
                    request_id: request.id,
                    token_id: token,
                    token_text: self.decode_token(token),
                    logprob: None,
                    is_final: true,
                    finish_reason: Some(FinishReason::EndOfSequence),
                    seq_len: context.len() + 1,
                };
                callback(output);
                break;
            }

            // Update context
            context.push(token);
            generated.push(token);

            // Create output and call callback
            let is_final = generated.len() >= params.max_tokens;
            let output = TokenOutput {
                request_id: request.id,
                token_id: token,
                token_text: self.decode_token(token),
                logprob: None,
                is_final,
                finish_reason: if is_final {
                    Some(FinishReason::Length)
                } else {
                    None
                },
                seq_len: context.len(),
            };

            // Check if callback wants to stop
            if !callback(output) {
                break;
            }

            // Check stop sequences
            if !params.stop_sequences.is_empty() {
                if let Some(tokenizer) = self.model.tokenizer() {
                    if let Ok(generated_text) = tokenizer.decode(&generated) {
                        for stop_seq in &params.stop_sequences {
                            if generated_text.contains(stop_seq) {
                                return Ok(generated);
                            }
                        }
                    }
                }
            }
        }

        Ok(generated)
    }

    /// Decode a single token to text (helper method)
    fn decode_token(&self, token: u32) -> Option<String> {
        self.model.tokenizer().and_then(|t| t.decode(&[token]).ok())
    }

    /// Run the serving loop until stopped
    pub fn run(&self) -> Result<()> {
        self.is_running.store(true, Ordering::SeqCst);

        while self.is_running.load(Ordering::SeqCst) {
            // Check if there's work to do
            let has_work = {
                let queue = self.queue.lock();
                !queue.is_empty()
            };

            if has_work {
                self.run_iteration()?;
            } else {
                // No work, yield
                std::thread::sleep(Duration::from_micros(100));
            }

            // Check for timeout requests
            self.check_timeouts();
        }

        Ok(())
    }

    /// Stop the serving loop
    pub fn stop(&self) {
        self.is_running.store(false, Ordering::SeqCst);
    }

    /// Check for and handle timed out requests
    fn check_timeouts(&self) {
        let timeout = Duration::from_millis(self.config.request_timeout_ms);
        let mut timed_out = Vec::new();

        // Find timed out pending requests
        {
            let pending = self.pending_requests.read();
            for (id, req) in pending.iter() {
                if req.created_at.elapsed() > timeout {
                    timed_out.push(*id);
                }
            }
        }

        // Cancel timed out requests
        for id in timed_out {
            self.cancel(id);
        }
    }

    /// Get serving metrics
    pub fn metrics(&self) -> ServingMetrics {
        let queue = self.queue.lock();
        let scheduler = self.scheduler.lock();
        let elapsed = self.start_time.elapsed().as_secs_f64();

        let total_requests = self.total_requests.load(Ordering::Relaxed);
        let total_tokens = self.total_tokens.load(Ordering::Relaxed);
        let completed_count = self.completed_results.read().len();

        ServingMetrics {
            requests_per_second: if elapsed > 0.0 {
                total_requests as f64 / elapsed
            } else {
                0.0
            },
            tokens_per_second: if elapsed > 0.0 {
                total_tokens as f64 / elapsed
            } else {
                0.0
            },
            average_latency_ms: 0.0, // Would need to track per-request latencies
            p99_latency_ms: 0.0,     // Would need latency histogram
            batch_utilization: 0.0,  // Would need to track batch sizes
            kv_cache_utilization: scheduler.kv_cache_manager().stats().slot_utilization(),
            pending_requests: queue.pending_count(),
            running_requests: queue.running_count(),
            completed_requests: completed_count,
            total_requests_processed: total_requests,
            total_tokens_generated: total_tokens,
            uptime_seconds: elapsed,
        }
    }

    /// Get serving statistics (alias for metrics)
    pub fn stats(&self) -> ServingMetrics {
        self.metrics()
    }

    /// Get configuration
    pub fn config(&self) -> &ServingEngineConfig {
        &self.config
    }

    /// Check if speculative decoding should be used for the given generation params
    ///
    /// Returns true when:
    /// - Speculative decoding is enabled in config
    /// - Temperature is low (< 0.5) for deterministic generation
    /// - Greedy decoding (top_k = 1)
    /// - A draft model is available or can be loaded
    pub fn should_use_speculative(&self, params: &GenerateParams) -> bool {
        if !self.config.enable_speculative {
            return false;
        }

        // Use the optimizer's recommendation
        self.optimizer.should_use_speculative(params)
    }

    /// Get recommended draft model path based on main model size
    ///
    /// Auto-detection rules:
    /// - For 7B+ models: use 1B draft (e.g., TinyLlama-1.1B)
    /// - For 3B models: use 0.5B draft (e.g., Qwen2.5-0.5B)
    /// - Returns configured path if explicitly set
    pub fn get_draft_model_path(&self) -> Option<String> {
        // Return configured path if explicitly set
        if let Some(ref path) = self.config.draft_model_path {
            return Some(path.clone());
        }

        // Auto-detect based on main model info
        if let Some(info) = self.model.model_info() {
            let params_billions = info.num_parameters as f64 / 1_000_000_000.0;

            if params_billions >= 7.0 {
                // 7B+ models: use 1B draft model
                Some("TinyLlama/TinyLlama-1.1B-Chat-v1.0".to_string())
            } else if params_billions >= 3.0 {
                // 3B models: use 0.5B draft model
                Some("Qwen/Qwen2.5-0.5B".to_string())
            } else {
                // For smaller models, speculative decoding overhead may not be worth it
                None
            }
        } else {
            // No model info available, use sensible default
            Some("TinyLlama/TinyLlama-1.1B-Chat-v1.0".to_string())
        }
    }

    /// Set the draft model for speculative decoding
    pub fn set_draft_model(&self, draft_model: Arc<dyn LlmBackend>) {
        *self.draft_model.write() = Some(draft_model);

        // Enable speculative decoding in the optimizer
        if let Some(path) = self.get_draft_model_path() {
            self.optimizer.enable_speculative_decoding(&path);
        }
    }

    /// Get the realtime optimizer for advanced optimization decisions
    pub fn optimizer(&self) -> &RealtimeOptimizer {
        &self.optimizer
    }

    /// Get speculative decoding statistics
    pub fn speculative_stats(&self) -> Option<crate::speculative::SpeculativeStats> {
        // TODO: Return actual stats when speculative decoder is integrated
        // For now, return placeholder stats
        if self.optimizer.is_speculative_active() {
            Some(crate::speculative::SpeculativeStats {
                draft_tokens: 0,
                accepted_tokens: 0,
                acceptance_rate: 0.0,
                speedup: 1.0,
                main_forward_passes: 0,
                draft_forward_passes: 0,
                avg_tokens_per_main_pass: 1.0,
                total_speculation_time_ms: 0.0,
                total_tokens_generated: 0,
            })
        } else {
            None
        }
    }
}

/// Serving metrics
#[derive(Debug, Clone, Default)]
pub struct ServingMetrics {
    /// Requests processed per second
    pub requests_per_second: f64,
    /// Tokens generated per second
    pub tokens_per_second: f64,
    /// Average request latency in milliseconds
    pub average_latency_ms: f64,
    /// 99th percentile latency in milliseconds
    pub p99_latency_ms: f64,
    /// Batch utilization (0.0 - 1.0)
    pub batch_utilization: f64,
    /// KV cache utilization (0.0 - 1.0)
    pub kv_cache_utilization: f64,
    /// Number of pending requests
    pub pending_requests: usize,
    /// Number of running requests
    pub running_requests: usize,
    /// Number of completed requests
    pub completed_requests: usize,
    /// Total requests processed
    pub total_requests_processed: u64,
    /// Total tokens generated
    pub total_tokens_generated: u64,
    /// Uptime in seconds
    pub uptime_seconds: f64,
}

// ============================================================================
// Async support
// ============================================================================

#[cfg(feature = "async-runtime")]
impl ServingEngine {
    /// Submit a request and await completion
    pub async fn submit_async(&self, request: InferenceRequest) -> Result<GenerationResult> {
        let request_id = request.id;
        let (tx, rx) = tokio::sync::oneshot::channel();

        // Store request with completion channel
        {
            let engine_request = EngineRequest {
                request: request.clone(),
                callback: None,
                completion_tx: Some(tx),
                created_at: Instant::now(),
            };
            self.pending_requests
                .write()
                .insert(request_id, engine_request);
        }

        // Add to queue
        self.queue.lock().add(request);
        self.total_requests.fetch_add(1, Ordering::Relaxed);

        // Wait for completion
        rx.await
            .map_err(|_| RuvLLMError::Generation("Request cancelled".to_string()))
    }

    /// Stream tokens for a request
    pub fn stream(
        &self,
        request: InferenceRequest,
    ) -> Result<impl futures_core::Stream<Item = TokenOutput>> {
        let (tx, rx) = mpsc::unbounded_channel();
        let request_id = request.id;

        // Create callback that sends to channel
        let callback: TokenCallback = Box::new(move |output| {
            let _ = tx.send(output);
        });

        // Submit with callback
        self.submit_with_callback(request, callback)?;

        Ok(tokio_stream::wrappers::UnboundedReceiverStream::new(rx))
    }

    /// Run the serving loop asynchronously
    pub async fn run_async(&self) -> Result<()> {
        self.is_running.store(true, Ordering::SeqCst);

        while self.is_running.load(Ordering::SeqCst) {
            let has_work = {
                let queue = self.queue.lock();
                !queue.is_empty()
            };

            if has_work {
                self.run_iteration()?;
            } else {
                tokio::time::sleep(Duration::from_micros(100)).await;
            }

            self.check_timeouts();
        }

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::backends::NoopBackend;

    fn create_test_engine() -> ServingEngine {
        let model = Arc::new(NoopBackend);
        let config = ServingEngineConfig {
            kv_cache: KvCachePoolConfig {
                num_slots: 4,
                max_seq_len: 256,
                block_size: 16,
                total_blocks: 64,
                num_kv_heads: 2,
                head_dim: 64,
                num_layers: 4,
            },
            ..Default::default()
        };
        ServingEngine::new(model, config)
    }

    fn create_test_request() -> InferenceRequest {
        let params = GenerateParams::default().with_max_tokens(10);
        InferenceRequest::new(vec![1, 2, 3, 4, 5], params)
    }

    #[test]
    fn test_submit_request() {
        let engine = create_test_engine();
        let request = create_test_request();
        let id = request.id;

        let result = engine.submit(request);
        assert!(result.is_ok());
        assert_eq!(result.unwrap(), id);
    }

    #[test]
    fn test_cancel_request() {
        let engine = create_test_engine();
        let request = create_test_request();
        let id = engine.submit(request).unwrap();

        let cancelled = engine.cancel(id);
        assert!(cancelled);
    }

    #[test]
    fn test_run_iteration() {
        let engine = create_test_engine();
        let request = create_test_request();
        engine.submit(request).unwrap();

        // First iteration should do prefill
        let outputs = engine.run_iteration().unwrap();
        // May or may not have outputs depending on scheduler behavior
    }

    #[test]
    fn test_metrics() {
        let engine = create_test_engine();
        let metrics = engine.metrics();

        assert_eq!(metrics.pending_requests, 0);
        assert_eq!(metrics.running_requests, 0);
    }

    #[test]
    fn test_with_callback() {
        use std::sync::atomic::AtomicUsize;

        let engine = create_test_engine();
        let request = create_test_request();

        let callback_count = Arc::new(AtomicUsize::new(0));
        let count_clone = callback_count.clone();

        let callback: TokenCallback = Box::new(move |_| {
            count_clone.fetch_add(1, Ordering::Relaxed);
        });

        let id = engine.submit_with_callback(request, callback).unwrap();

        // Run a few iterations
        for _ in 0..15 {
            let _ = engine.run_iteration();
        }

        // Callback should have been called at least once
        // (actual count depends on scheduling and token generation)
    }

    #[test]
    fn test_token_generation_with_noop_backend() {
        // Test that token generation works (via simulation) even with NoopBackend
        let engine = create_test_engine();
        let request = create_test_request();
        engine.submit(request).unwrap();

        // Run multiple iterations to process prefill and generate tokens
        for _ in 0..20 {
            let result = engine.run_iteration();
            // Should not error even without a real model
            assert!(result.is_ok());
        }

        let stats = engine.stats();
        // Should have processed at least one request
        assert!(
            stats.running_requests > 0
                || stats.completed_requests > 0
                || stats.pending_requests > 0
        );
    }

    #[test]
    fn test_generation_produces_different_tokens() {
        // Test that different contexts produce different tokens
        let engine = create_test_engine();

        // Submit requests with different prompt tokens
        let params1 = GenerateParams::default().with_max_tokens(5);
        let request1 = InferenceRequest::new(vec![1, 2, 3], params1);

        let params2 = GenerateParams::default().with_max_tokens(5);
        let request2 = InferenceRequest::new(vec![100, 200, 300], params2);

        let id1 = engine.submit(request1).unwrap();
        let id2 = engine.submit(request2).unwrap();

        // Run iterations
        for _ in 0..30 {
            let _ = engine.run_iteration();
        }

        // Both requests should have been processed
        let stats = engine.stats();
        // At minimum we should have started processing
    }

    #[test]
    fn test_speculative_config_defaults() {
        // Test that speculative decoding config has sensible defaults
        let config = ServingEngineConfig::default();

        // Speculative decoding should be enabled by default
        assert!(config.enable_speculative);

        // Default lookahead should be reasonable (4-8 tokens)
        assert!(config.speculative_config.lookahead >= 2);
        assert!(config.speculative_config.lookahead <= 16);

        // Draft temperature should be low for deterministic drafting
        assert!(config.speculative_config.draft_temperature <= 0.5);
    }

    #[test]
    fn test_streaming_generation() {
        // Test streaming generation with callbacks
        use std::sync::atomic::AtomicUsize;

        let engine = create_test_engine();
        let params = GenerateParams::default()
            .with_max_tokens(5)
            .with_temperature(0.8);
        let request = InferenceRequest::new(vec![1, 2, 3, 4, 5], params);

        let tokens_received = Arc::new(AtomicUsize::new(0));
        let tokens_clone = tokens_received.clone();

        let callback: TokenCallback = Box::new(move |output| {
            tokens_clone.fetch_add(1, Ordering::Relaxed);
            // Verify token output has valid fields
            assert!(output.seq_len > 0);
        });

        engine.submit_with_callback(request, callback).unwrap();

        // Run iterations
        for _ in 0..30 {
            let _ = engine.run_iteration();
        }

        // Should have received at least some tokens
        // (exact count depends on prefill/decode scheduling)
    }

    #[test]
    fn test_generation_respects_max_tokens() {
        let engine = create_test_engine();

        // Request with small max_tokens
        let params = GenerateParams::default().with_max_tokens(3);
        let request = InferenceRequest::new(vec![1, 2, 3], params);

        engine.submit(request).unwrap();

        // Run many iterations
        for _ in 0..50 {
            let _ = engine.run_iteration();
        }

        // Check metrics - request should complete
        let stats = engine.stats();
        // Either completed or still processing, but should not hang
    }

    #[test]
    fn test_deterministic_generation_with_seed() {
        // Test that the same context produces consistent results
        let engine = create_test_engine();

        // Two identical requests
        let params = GenerateParams::default().with_max_tokens(5).with_seed(42);

        let request1 = InferenceRequest::new(vec![10, 20, 30], params.clone());
        let request2 = InferenceRequest::new(vec![10, 20, 30], params);

        engine.submit(request1).unwrap();
        engine.submit(request2).unwrap();

        // Process both
        for _ in 0..30 {
            let _ = engine.run_iteration();
        }

        // Both should complete successfully
    }
}