Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'

vendor/ruvector/crates/ruvector-attention/src/sparse/flash.rs

@@ -0,0 +1,227 @@
+//! Flash attention - memory-efficient attention with tiled computation
+//!
+//! Memory: O(block_size) for attention matrix instead of O(n²)
+
+use crate::error::{AttentionError, AttentionResult};
+use crate::traits::Attention;
+
+/// Flash attention with block-wise computation
+///
+/// Computes attention in tiles to minimize memory usage while maintaining numerical stability.
+pub struct FlashAttention {
+    dim: usize,
+    block_size: usize,
+    scale: f32,
+    causal: bool,
+}
+
+impl FlashAttention {
+    /// Create new flash attention
+    pub fn new(dim: usize, block_size: usize) -> Self {
+        Self {
+            dim,
+            block_size,
+            scale: 1.0 / (dim as f32).sqrt(),
+            causal: false,
+        }
+    }
+
+    /// Create with causal masking
+    pub fn causal(dim: usize, block_size: usize) -> Self {
+        Self {
+            dim,
+            block_size,
+            scale: 1.0 / (dim as f32).sqrt(),
+            causal: true,
+        }
+    }
+
+    /// Compute attention scores for a block
+    fn compute_block_scores(&self, query: &[f32], keys: &[&[f32]], start_idx: usize) -> Vec<f32> {
+        keys.iter()
+            .enumerate()
+            .map(|(j, key)| {
+                if self.causal && start_idx + j > 0 {
+                    // Simplified causal: assuming query is at position 0
+                    f32::NEG_INFINITY
+                } else {
+                    query
+                        .iter()
+                        .zip(key.iter())
+                        .map(|(q, k)| q * k)
+                        .sum::<f32>()
+                        * self.scale
+                }
+            })
+            .collect()
+    }
+}
+
+impl Attention for FlashAttention {
+    fn compute(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+    ) -> AttentionResult<Vec<f32>> {
+        if keys.is_empty() {
+            return Err(AttentionError::InvalidConfig("Empty keys".to_string()));
+        }
+        if keys.len() != values.len() {
+            return Err(AttentionError::DimensionMismatch {
+                expected: keys.len(),
+                actual: values.len(),
+            });
+        }
+        if query.len() != self.dim {
+            return Err(AttentionError::DimensionMismatch {
+                expected: self.dim,
+                actual: query.len(),
+            });
+        }
+
+        let n = keys.len();
+        let value_dim = values[0].len();
+
+        // Online softmax with tiled computation
+        let mut output = vec![0.0f32; value_dim];
+        let mut max_so_far = f32::NEG_INFINITY;
+        let mut sum_exp = 0.0f32;
+
+        // Process in blocks
+        for block_start in (0..n).step_by(self.block_size) {
+            let block_end = (block_start + self.block_size).min(n);
+            let block_keys: Vec<&[f32]> = keys[block_start..block_end].to_vec();
+
+            // Compute attention scores for this block
+            let block_scores = self.compute_block_scores(query, &block_keys, block_start);
+
+            // Find block maximum
+            let block_max = block_scores
+                .iter()
+                .copied()
+                .filter(|x| x.is_finite())
+                .fold(f32::NEG_INFINITY, f32::max);
+
+            if !block_max.is_finite() {
+                continue; // Skip fully masked blocks
+            }
+
+            // New maximum
+            let new_max = max_so_far.max(block_max);
+
+            // Rescale previous accumulations
+            if max_so_far.is_finite() {
+                let rescale = (max_so_far - new_max).exp();
+                sum_exp *= rescale;
+                output.iter_mut().for_each(|o| *o *= rescale);
+            }
+
+            // Add contribution from this block
+            for (local_idx, &score) in block_scores.iter().enumerate() {
+                if score.is_finite() {
+                    let exp_score = (score - new_max).exp();
+                    sum_exp += exp_score;
+
+                    let global_idx = block_start + local_idx;
+                    for (j, &vj) in values[global_idx].iter().enumerate() {
+                        output[j] += exp_score * vj;
+                    }
+                }
+            }
+
+            max_so_far = new_max;
+        }
+
+        // Final normalization
+        if sum_exp > 1e-8 {
+            output.iter_mut().for_each(|o| *o /= sum_exp);
+        }
+
+        Ok(output)
+    }
+
+    fn compute_with_mask(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+        mask: Option<&[bool]>,
+    ) -> AttentionResult<Vec<f32>> {
+        if let Some(m) = mask {
+            let filtered: Vec<(usize, bool)> = m
+                .iter()
+                .copied()
+                .enumerate()
+                .filter(|(_, keep)| *keep)
+                .collect();
+            let filtered_keys: Vec<&[f32]> = filtered.iter().map(|(i, _)| keys[*i]).collect();
+            let filtered_values: Vec<&[f32]> = filtered.iter().map(|(i, _)| values[*i]).collect();
+            self.compute(query, &filtered_keys, &filtered_values)
+        } else {
+            self.compute(query, keys, values)
+        }
+    }
+
+    fn dim(&self) -> usize {
+        self.dim
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::attention::ScaledDotProductAttention;
+
+    #[test]
+    fn test_flash_attention() {
+        let attention = FlashAttention::new(64, 16);
+
+        let query = vec![0.5; 64];
+        let keys: Vec<Vec<f32>> = (0..256).map(|_| vec![0.3; 64]).collect();
+        let values: Vec<Vec<f32>> = (0..256).map(|_| vec![1.0; 64]).collect();
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+        assert_eq!(result.len(), 64);
+    }
+
+    #[test]
+    fn test_flash_matches_standard() {
+        let dim = 32;
+        let flash = FlashAttention::new(dim, 8);
+        let standard = ScaledDotProductAttention::new(dim);
+
+        let query = vec![0.5; dim];
+        let keys: Vec<Vec<f32>> = (0..16).map(|i| vec![(i as f32) * 0.1; dim]).collect();
+        let values: Vec<Vec<f32>> = (0..16).map(|i| vec![(i as f32) * 0.2; dim]).collect();
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let flash_result = flash.compute(&query, &keys_refs, &values_refs).unwrap();
+        let standard_result = standard.compute(&query, &keys_refs, &values_refs).unwrap();
+
+        // Results should be approximately equal
+        for (f, s) in flash_result.iter().zip(standard_result.iter()) {
+            assert!((f - s).abs() < 1e-4, "Flash: {}, Standard: {}", f, s);
+        }
+    }
+
+    #[test]
+    fn test_causal_flash() {
+        let attention = FlashAttention::causal(32, 8);
+
+        let query = vec![1.0; 32];
+        let keys = vec![vec![0.5; 32]; 20];
+        let values = vec![vec![1.0; 32]; 20];
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+        assert_eq!(result.len(), 32);
+    }
+}

vendor/ruvector/crates/ruvector-attention/src/sparse/linear.rs

@@ -0,0 +1,237 @@
+//! Linear attention using random feature approximation (Performer-style)
+//!
+//! Complexity: O(n * k * d) where k = number of random features
+
+use crate::error::{AttentionError, AttentionResult};
+use crate::traits::Attention;
+
+/// Kernel type for linear attention
+#[derive(Clone, Debug)]
+pub enum KernelType {
+    /// FAVOR+ softmax approximation
+    Softmax,
+    /// ReLU kernel
+    ReLU,
+    /// ELU kernel
+    ELU,
+}
+
+/// Linear attention with random feature maps
+///
+/// Uses kernel trick to achieve O(n * k * d) complexity instead of O(n² * d).
+pub struct LinearAttention {
+    dim: usize,
+    num_features: usize,
+    kernel: KernelType,
+    /// Random projection matrix [num_features x dim]
+    random_features: Vec<f32>,
+}
+
+impl LinearAttention {
+    /// Create new linear attention
+    pub fn new(dim: usize, num_features: usize) -> Self {
+        Self::with_kernel(dim, num_features, KernelType::Softmax)
+    }
+
+    /// Create with specific kernel type
+    pub fn with_kernel(dim: usize, num_features: usize, kernel: KernelType) -> Self {
+        // Initialize random features using Box-Muller for Gaussian
+        let random_features = Self::generate_random_features(dim, num_features);
+
+        Self {
+            dim,
+            num_features,
+            kernel,
+            random_features,
+        }
+    }
+
+    fn generate_random_features(dim: usize, num_features: usize) -> Vec<f32> {
+        use std::f32::consts::PI;
+
+        let mut features = Vec::with_capacity(num_features * dim);
+        let mut seed = 42u64;
+
+        for _ in 0..((num_features * dim + 1) / 2) {
+            // Simple LCG for reproducibility
+            seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
+            let u1 = (seed as f32) / (u64::MAX as f32);
+            seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
+            let u2 = (seed as f32) / (u64::MAX as f32);
+
+            // Box-Muller transform
+            let r = (-2.0 * u1.max(1e-10).ln()).sqrt();
+            let theta = 2.0 * PI * u2;
+
+            features.push(r * theta.cos());
+            if features.len() < num_features * dim {
+                features.push(r * theta.sin());
+            }
+        }
+
+        features.truncate(num_features * dim);
+
+        // Normalize columns
+        let scale = 1.0 / (dim as f32).sqrt();
+        features.iter_mut().for_each(|x| *x *= scale);
+
+        features
+    }
+
+    /// Apply feature map to input
+    fn feature_map(&self, x: &[f32]) -> Vec<f32> {
+        let mut phi = vec![0.0f32; self.num_features];
+
+        for (i, phi_i) in phi.iter_mut().enumerate() {
+            let projection: f32 = x
+                .iter()
+                .enumerate()
+                .map(|(j, &xj)| xj * self.random_features[i * self.dim + j])
+                .sum();
+
+            *phi_i = match self.kernel {
+                KernelType::Softmax => {
+                    // FAVOR+: exp(projection - ||x||²/2) / sqrt(num_features)
+                    let norm_sq: f32 = x.iter().map(|xi| xi * xi).sum();
+                    (projection - norm_sq / 2.0).exp() / (self.num_features as f32).sqrt()
+                }
+                KernelType::ReLU => projection.max(0.0),
+                KernelType::ELU => {
+                    if projection >= 0.0 {
+                        projection
+                    } else {
+                        projection.exp() - 1.0
+                    }
+                }
+            };
+        }
+
+        phi
+    }
+}
+
+impl Attention for LinearAttention {
+    fn compute(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+    ) -> AttentionResult<Vec<f32>> {
+        if keys.is_empty() {
+            return Err(AttentionError::InvalidConfig("Empty keys".to_string()));
+        }
+        if keys.len() != values.len() {
+            return Err(AttentionError::DimensionMismatch {
+                expected: keys.len(),
+                actual: values.len(),
+            });
+        }
+        if query.len() != self.dim {
+            return Err(AttentionError::DimensionMismatch {
+                expected: self.dim,
+                actual: query.len(),
+            });
+        }
+
+        // Compute phi(Q)
+        let phi_q = self.feature_map(query);
+
+        // Compute sum_i phi(K_i)^T * V_i  and  sum_i phi(K_i)
+        let value_dim = values[0].len();
+        let mut kv_sum = vec![0.0f32; self.num_features * value_dim]; // [num_features x value_dim]
+        let mut k_sum = vec![0.0f32; self.num_features];
+
+        for (key, value) in keys.iter().zip(values.iter()) {
+            let phi_k = self.feature_map(key);
+
+            // Accumulate phi(K)^T * V (outer product contribution)
+            for (i, &phi_ki) in phi_k.iter().enumerate() {
+                for (j, &vj) in value.iter().enumerate() {
+                    kv_sum[i * value_dim + j] += phi_ki * vj;
+                }
+                k_sum[i] += phi_ki;
+            }
+        }
+
+        // Compute output: (phi(Q)^T * KV_sum) / (phi(Q)^T * K_sum)
+        let mut output = vec![0.0f32; value_dim];
+        let mut normalizer = 0.0f32;
+
+        for (i, &phi_qi) in phi_q.iter().enumerate() {
+            for (j, out_j) in output.iter_mut().enumerate() {
+                *out_j += phi_qi * kv_sum[i * value_dim + j];
+            }
+            normalizer += phi_qi * k_sum[i];
+        }
+
+        // Normalize
+        if normalizer.abs() > 1e-8 {
+            output.iter_mut().for_each(|x| *x /= normalizer);
+        }
+
+        Ok(output)
+    }
+
+    fn compute_with_mask(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+        mask: Option<&[bool]>,
+    ) -> AttentionResult<Vec<f32>> {
+        if let Some(m) = mask {
+            let filtered: Vec<(usize, bool)> = m
+                .iter()
+                .copied()
+                .enumerate()
+                .filter(|(_, keep)| *keep)
+                .collect();
+            let filtered_keys: Vec<&[f32]> = filtered.iter().map(|(i, _)| keys[*i]).collect();
+            let filtered_values: Vec<&[f32]> = filtered.iter().map(|(i, _)| values[*i]).collect();
+            self.compute(query, &filtered_keys, &filtered_values)
+        } else {
+            self.compute(query, keys, values)
+        }
+    }
+
+    fn dim(&self) -> usize {
+        self.dim
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_linear_attention() {
+        let attention = LinearAttention::new(64, 32);
+
+        let query = vec![0.5; 64];
+        let keys: Vec<Vec<f32>> = (0..100).map(|_| vec![0.3; 64]).collect();
+        let values: Vec<Vec<f32>> = (0..100).map(|_| vec![1.0; 64]).collect();
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+        assert_eq!(result.len(), 64);
+    }
+
+    #[test]
+    fn test_kernel_types() {
+        for kernel in [KernelType::Softmax, KernelType::ReLU, KernelType::ELU] {
+            let attention = LinearAttention::with_kernel(32, 16, kernel);
+
+            let query = vec![1.0; 32];
+            let keys = vec![vec![0.5; 32]; 10];
+            let values = vec![vec![1.0; 32]; 10];
+
+            let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+            let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+            let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+            assert_eq!(result.len(), 32);
+        }
+    }
+}

vendor/ruvector/crates/ruvector-attention/src/sparse/local_global.rs

@@ -0,0 +1,193 @@
+//! Local-Global attention for efficient long-range dependencies
+//!
+//! Complexity: O(n * (w + g)) where w = window size, g = global tokens
+
+use crate::error::{AttentionError, AttentionResult};
+use crate::traits::Attention;
+use crate::utils::stable_softmax;
+
+/// Local-Global attention mechanism
+///
+/// Combines local windowed attention with global tokens for O(n*(w+g)) complexity.
+pub struct LocalGlobalAttention {
+    dim: usize,
+    local_window: usize,
+    num_global_tokens: usize,
+    scale: f32,
+}
+
+impl LocalGlobalAttention {
+    /// Create new local-global attention
+    pub fn new(dim: usize, local_window: usize, num_global_tokens: usize) -> Self {
+        Self {
+            dim,
+            local_window,
+            num_global_tokens,
+            scale: 1.0 / (dim as f32).sqrt(),
+        }
+    }
+
+    /// Compute attention scores for local window
+    fn compute_local_scores(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        position: usize,
+    ) -> Vec<(usize, f32)> {
+        let n = keys.len();
+        let half_window = self.local_window / 2;
+        let start = position.saturating_sub(half_window);
+        let end = (position + half_window + 1).min(n);
+
+        (start..end)
+            .map(|j| {
+                let score: f32 = query
+                    .iter()
+                    .zip(keys[j].iter())
+                    .map(|(q, k)| q * k)
+                    .sum::<f32>()
+                    * self.scale;
+                (j, score)
+            })
+            .collect()
+    }
+
+    /// Compute attention scores for global tokens
+    fn compute_global_scores(&self, query: &[f32], keys: &[&[f32]]) -> Vec<(usize, f32)> {
+        let num_global = self.num_global_tokens.min(keys.len());
+
+        (0..num_global)
+            .map(|j| {
+                let score: f32 = query
+                    .iter()
+                    .zip(keys[j].iter())
+                    .map(|(q, k)| q * k)
+                    .sum::<f32>()
+                    * self.scale;
+                (j, score)
+            })
+            .collect()
+    }
+}
+
+impl Attention for LocalGlobalAttention {
+    fn compute(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+    ) -> AttentionResult<Vec<f32>> {
+        if keys.is_empty() {
+            return Err(AttentionError::InvalidConfig("Empty keys".to_string()));
+        }
+        if keys.len() != values.len() {
+            return Err(AttentionError::DimensionMismatch {
+                expected: keys.len(),
+                actual: values.len(),
+            });
+        }
+        if query.len() != self.dim {
+            return Err(AttentionError::DimensionMismatch {
+                expected: self.dim,
+                actual: query.len(),
+            });
+        }
+
+        // For simplicity, compute at position 0 (middle of sequence would be typical)
+        let position = keys.len() / 2;
+
+        // Collect all attended positions and scores
+        let mut attended: Vec<(usize, f32)> = Vec::new();
+
+        // Add global scores
+        attended.extend(self.compute_global_scores(query, keys));
+
+        // Add local scores
+        for (idx, score) in self.compute_local_scores(query, keys, position) {
+            if !attended.iter().any(|(i, _)| *i == idx) {
+                attended.push((idx, score));
+            }
+        }
+
+        if attended.is_empty() {
+            return Err(AttentionError::ComputationError(
+                "No attended positions".to_string(),
+            ));
+        }
+
+        // Softmax over attended positions
+        let scores: Vec<f32> = attended.iter().map(|(_, s)| *s).collect();
+        let weights = stable_softmax(&scores);
+
+        // Weighted sum of values
+        let mut output = vec![0.0f32; self.dim];
+        for ((idx, _), weight) in attended.iter().zip(weights.iter()) {
+            for (o, v) in output.iter_mut().zip(values[*idx].iter()) {
+                *o += weight * v;
+            }
+        }
+
+        Ok(output)
+    }
+
+    fn compute_with_mask(
+        &self,
+        query: &[f32],
+        keys: &[&[f32]],
+        values: &[&[f32]],
+        mask: Option<&[bool]>,
+    ) -> AttentionResult<Vec<f32>> {
+        if let Some(m) = mask {
+            let filtered: Vec<(usize, bool)> = m
+                .iter()
+                .copied()
+                .enumerate()
+                .filter(|(_, keep)| *keep)
+                .collect();
+            let filtered_keys: Vec<&[f32]> = filtered.iter().map(|(i, _)| keys[*i]).collect();
+            let filtered_values: Vec<&[f32]> = filtered.iter().map(|(i, _)| values[*i]).collect();
+            self.compute(query, &filtered_keys, &filtered_values)
+        } else {
+            self.compute(query, keys, values)
+        }
+    }
+
+    fn dim(&self) -> usize {
+        self.dim
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_local_global_attention() {
+        let attention = LocalGlobalAttention::new(64, 8, 2);
+
+        let query = vec![0.5; 64];
+        let keys: Vec<Vec<f32>> = (0..100).map(|_| vec![0.3; 64]).collect();
+        let values: Vec<Vec<f32>> = (0..100).map(|i| vec![i as f32; 64]).collect();
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+        assert_eq!(result.len(), 64);
+    }
+
+    #[test]
+    fn test_small_sequence() {
+        let attention = LocalGlobalAttention::new(32, 4, 1);
+
+        let query = vec![1.0; 32];
+        let keys = vec![vec![0.5; 32]; 5];
+        let values = vec![vec![1.0; 32]; 5];
+
+        let keys_refs: Vec<&[f32]> = keys.iter().map(|k| k.as_slice()).collect();
+        let values_refs: Vec<&[f32]> = values.iter().map(|v| v.as_slice()).collect();
+
+        let result = attention.compute(&query, &keys_refs, &values_refs).unwrap();
+        assert_eq!(result.len(), 32);
+    }
+}

vendor/ruvector/crates/ruvector-attention/src/sparse/mask.rs

@@ -0,0 +1,207 @@
+//! Sparse mask utilities for attention patterns
+
+use std::collections::HashSet;
+
+/// Sparse mask for attention patterns
+#[derive(Clone, Debug)]
+pub struct AttentionMask {
+    /// Sparse indices as (row, col) pairs
+    pub indices: Vec<(usize, usize)>,
+    /// Shape of the full attention matrix
+    pub shape: (usize, usize),
+    /// Set for O(1) lookup
+    lookup: HashSet<(usize, usize)>,
+}
+
+impl AttentionMask {
+    /// Create a new sparse mask from indices
+    pub fn new(indices: Vec<(usize, usize)>, shape: (usize, usize)) -> Self {
+        let lookup: HashSet<_> = indices.iter().copied().collect();
+        Self {
+            indices,
+            shape,
+            lookup,
+        }
+    }
+
+    /// Check if position is masked (should attend)
+    #[inline]
+    pub fn is_attended(&self, row: usize, col: usize) -> bool {
+        self.lookup.contains(&(row, col))
+    }
+
+    /// Apply mask to attention scores (set non-attended to -inf)
+    pub fn apply(&self, scores: &mut [f32], seq_len: usize) {
+        for i in 0..seq_len {
+            for j in 0..seq_len {
+                if !self.is_attended(i, j) {
+                    scores[i * seq_len + j] = f32::NEG_INFINITY;
+                }
+            }
+        }
+    }
+
+    /// Create a local window mask
+    pub fn local_window(n: usize, window_size: usize) -> Self {
+        let mut indices = Vec::new();
+        let half_window = window_size / 2;
+
+        for i in 0..n {
+            let start = i.saturating_sub(half_window);
+            let end = (i + half_window + 1).min(n);
+            for j in start..end {
+                indices.push((i, j));
+            }
+        }
+
+        Self::new(indices, (n, n))
+    }
+
+    /// Create a causal mask (lower triangular)
+    pub fn causal(n: usize) -> Self {
+        let mut indices = Vec::new();
+        for i in 0..n {
+            for j in 0..=i {
+                indices.push((i, j));
+            }
+        }
+        Self::new(indices, (n, n))
+    }
+
+    /// Create a strided mask
+    pub fn strided(n: usize, stride: usize) -> Self {
+        let mut indices = Vec::new();
+        for i in 0..n {
+            for j in (0..n).step_by(stride) {
+                indices.push((i, j));
+            }
+            // Always attend to self
+            indices.push((i, i));
+        }
+        let mut indices: Vec<_> = indices
+            .into_iter()
+            .collect::<HashSet<_>>()
+            .into_iter()
+            .collect();
+        indices.sort();
+        Self::new(indices, (n, n))
+    }
+
+    /// Number of non-zero entries
+    pub fn nnz(&self) -> usize {
+        self.indices.len()
+    }
+
+    /// Sparsity ratio (0 = all zeros, 1 = all ones)
+    pub fn density(&self) -> f32 {
+        self.nnz() as f32 / (self.shape.0 * self.shape.1) as f32
+    }
+}
+
+/// Builder for creating sparse masks
+pub struct SparseMaskBuilder {
+    n: usize,
+    indices: Vec<(usize, usize)>,
+}
+
+impl SparseMaskBuilder {
+    pub fn new(n: usize) -> Self {
+        Self {
+            n,
+            indices: Vec::new(),
+        }
+    }
+
+    /// Add local window pattern
+    pub fn with_local_window(mut self, window_size: usize) -> Self {
+        let half_window = window_size / 2;
+        for i in 0..self.n {
+            let start = i.saturating_sub(half_window);
+            let end = (i + half_window + 1).min(self.n);
+            for j in start..end {
+                self.indices.push((i, j));
+            }
+        }
+        self
+    }
+
+    /// Add global tokens (all positions attend to these)
+    pub fn with_global_tokens(mut self, global_indices: &[usize]) -> Self {
+        for i in 0..self.n {
+            for &g in global_indices {
+                if g < self.n {
+                    self.indices.push((i, g));
+                    self.indices.push((g, i));
+                }
+            }
+        }
+        self
+    }
+
+    /// Add causal masking
+    pub fn with_causal(mut self) -> Self {
+        for i in 0..self.n {
+            for j in 0..=i {
+                self.indices.push((i, j));
+            }
+        }
+        self
+    }
+
+    /// Build the mask
+    pub fn build(self) -> AttentionMask {
+        let mut indices: Vec<_> = self
+            .indices
+            .into_iter()
+            .collect::<HashSet<_>>()
+            .into_iter()
+            .collect();
+        indices.sort();
+        AttentionMask::new(indices, (self.n, self.n))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_local_window_mask() {
+        let mask = AttentionMask::local_window(10, 3);
+
+        // Position 5 should attend to positions 4, 5, 6
+        assert!(mask.is_attended(5, 4));
+        assert!(mask.is_attended(5, 5));
+        assert!(mask.is_attended(5, 6));
+
+        // Position 5 should not attend to position 0
+        assert!(!mask.is_attended(5, 0));
+    }
+
+    #[test]
+    fn test_causal_mask() {
+        let mask = AttentionMask::causal(5);
+
+        // Lower triangle should be attended
+        assert!(mask.is_attended(2, 0));
+        assert!(mask.is_attended(2, 1));
+        assert!(mask.is_attended(2, 2));
+
+        // Upper triangle should not
+        assert!(!mask.is_attended(2, 3));
+        assert!(!mask.is_attended(2, 4));
+    }
+
+    #[test]
+    fn test_builder() {
+        let mask = SparseMaskBuilder::new(10)
+            .with_local_window(3)
+            .with_global_tokens(&[0])
+            .build();
+
+        // All positions should attend to global token 0
+        for i in 0..10 {
+            assert!(mask.is_attended(i, 0));
+        }
+    }
+}

vendor/ruvector/crates/ruvector-attention/src/sparse/mod.rs

@@ -0,0 +1,13 @@
+//! Sparse attention mechanisms for efficient computation on long sequences
+//!
+//! This module provides sparse attention patterns that reduce complexity from O(n²) to sub-quadratic.
+
+pub mod flash;
+pub mod linear;
+pub mod local_global;
+pub mod mask;
+
+pub use flash::FlashAttention;
+pub use linear::LinearAttention;
+pub use local_global::LocalGlobalAttention;
+pub use mask::{AttentionMask, SparseMaskBuilder};