mirror of
https://github.com/ruvnet/RuView
synced 2026-07-30 18:41:42 +00:00
ruv
525 lines
14 KiB
JavaScript
525 lines
14 KiB
JavaScript
/**
|
|
* RuVector Native Storage for Intelligence Layer
|
|
*
|
|
* Replaces JSON file storage with:
|
|
* - @ruvector/core: Native HNSW vector storage (150x faster)
|
|
* - @ruvector/sona: ReasoningBank for Q-learning and patterns
|
|
* - redb: Embedded database for metadata
|
|
*/
|
|
|
|
import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs';
|
|
import { dirname, join } from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const DATA_DIR = join(__dirname, 'data');
|
|
const DB_PATH = join(DATA_DIR, 'intelligence.db');
|
|
|
|
// Legacy JSON paths for migration
|
|
const LEGACY_PATTERNS = join(DATA_DIR, 'patterns.json');
|
|
const LEGACY_TRAJECTORIES = join(DATA_DIR, 'trajectories.json');
|
|
const LEGACY_MEMORY = join(DATA_DIR, 'memory.json');
|
|
const LEGACY_FEEDBACK = join(DATA_DIR, 'feedback.json');
|
|
const LEGACY_SEQUENCES = join(DATA_DIR, 'sequences.json');
|
|
|
|
// Try to load native modules
|
|
let ruvectorCore = null;
|
|
let sona = null;
|
|
|
|
try {
|
|
ruvectorCore = await import('@ruvector/core');
|
|
console.log('✅ @ruvector/core loaded - using native HNSW');
|
|
} catch (e) {
|
|
console.log('⚠️ @ruvector/core not available - using fallback');
|
|
}
|
|
|
|
try {
|
|
sona = await import('@ruvector/sona');
|
|
console.log('✅ @ruvector/sona loaded - using native ReasoningBank');
|
|
} catch (e) {
|
|
console.log('⚠️ @ruvector/sona not available - using fallback');
|
|
}
|
|
|
|
/**
|
|
* Native Vector Storage using @ruvector/core
|
|
*/
|
|
export class NativeVectorStorage {
|
|
constructor(options = {}) {
|
|
this.dimensions = options.dimensions || 128;
|
|
this.dbPath = options.dbPath || DB_PATH;
|
|
this.useNative = !!ruvectorCore;
|
|
this.db = null;
|
|
this.fallbackData = [];
|
|
}
|
|
|
|
async init() {
|
|
if (this.useNative && ruvectorCore) {
|
|
try {
|
|
// Use native VectorDB
|
|
this.db = new ruvectorCore.VectorDB({
|
|
dimensions: this.dimensions,
|
|
storagePath: this.dbPath,
|
|
efConstruction: 200,
|
|
maxNeighbors: 32,
|
|
efSearch: 100
|
|
});
|
|
return true;
|
|
} catch (e) {
|
|
console.warn('Native VectorDB init failed:', e.message);
|
|
this.useNative = false;
|
|
}
|
|
}
|
|
|
|
// Fallback: load from JSON
|
|
if (existsSync(LEGACY_MEMORY)) {
|
|
try {
|
|
this.fallbackData = JSON.parse(readFileSync(LEGACY_MEMORY, 'utf-8'));
|
|
} catch (e) {
|
|
this.fallbackData = [];
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async insert(id, vector, metadata = {}) {
|
|
if (this.useNative && this.db) {
|
|
// Native module requires Float32Array
|
|
const typedVector = vector instanceof Float32Array
|
|
? vector
|
|
: new Float32Array(vector);
|
|
return this.db.insert({
|
|
id,
|
|
vector: typedVector
|
|
});
|
|
}
|
|
|
|
// Fallback
|
|
this.fallbackData.push({ id, vector: Array.from(vector), metadata });
|
|
return id;
|
|
}
|
|
|
|
async search(query, k = 5) {
|
|
if (this.useNative && this.db) {
|
|
const typedQuery = query instanceof Float32Array
|
|
? query
|
|
: new Float32Array(query);
|
|
return this.db.search({
|
|
vector: typedQuery,
|
|
k,
|
|
efSearch: 100
|
|
});
|
|
}
|
|
|
|
// Fallback: brute force cosine similarity
|
|
const results = this.fallbackData.map(item => {
|
|
const score = this.cosineSimilarity(query, item.vector);
|
|
return { ...item, score };
|
|
});
|
|
|
|
return results
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, k);
|
|
}
|
|
|
|
cosineSimilarity(a, b) {
|
|
let dot = 0, normA = 0, normB = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
dot += a[i] * b[i];
|
|
normA += a[i] * a[i];
|
|
normB += b[i] * b[i];
|
|
}
|
|
return dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-8);
|
|
}
|
|
|
|
async count() {
|
|
if (this.useNative && this.db) {
|
|
return this.db.len();
|
|
}
|
|
return this.fallbackData.length;
|
|
}
|
|
|
|
async save() {
|
|
if (!this.useNative) {
|
|
writeFileSync(LEGACY_MEMORY, JSON.stringify(this.fallbackData, null, 2));
|
|
}
|
|
// Native storage is already persistent
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Native ReasoningBank using @ruvector/sona
|
|
*/
|
|
export class NativeReasoningBank {
|
|
constructor(options = {}) {
|
|
this.useNative = !!sona;
|
|
this.engine = null;
|
|
this.alpha = options.alpha || 0.1;
|
|
this.gamma = options.gamma || 0.9;
|
|
this.epsilon = options.epsilon || 0.1;
|
|
|
|
// Fallback Q-table
|
|
this.qTable = {};
|
|
this.trajectories = [];
|
|
this.abTestGroup = process.env.INTELLIGENCE_MODE || 'treatment';
|
|
}
|
|
|
|
async init() {
|
|
if (this.useNative && sona) {
|
|
try {
|
|
this.engine = new sona.SonaEngine(256);
|
|
return true;
|
|
} catch (e) {
|
|
console.warn('Native SonaEngine init failed:', e.message);
|
|
this.useNative = false;
|
|
}
|
|
}
|
|
|
|
// Fallback: load from JSON
|
|
if (existsSync(LEGACY_PATTERNS)) {
|
|
try {
|
|
this.qTable = JSON.parse(readFileSync(LEGACY_PATTERNS, 'utf-8'));
|
|
} catch (e) {
|
|
this.qTable = {};
|
|
}
|
|
}
|
|
if (existsSync(LEGACY_TRAJECTORIES)) {
|
|
try {
|
|
this.trajectories = JSON.parse(readFileSync(LEGACY_TRAJECTORIES, 'utf-8'));
|
|
} catch (e) {
|
|
this.trajectories = [];
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
stateKey(state) {
|
|
return state.toLowerCase().replace(/[^a-z0-9-]+/g, '_').slice(0, 80);
|
|
}
|
|
|
|
recordTrajectory(state, action, outcome, reward) {
|
|
const stateKey = this.stateKey(state);
|
|
|
|
if (this.useNative && this.engine) {
|
|
// Use native trajectory recording
|
|
const embedding = this.stateToEmbedding(stateKey);
|
|
const builder = this.engine.beginTrajectory(embedding);
|
|
// Add step with reward
|
|
builder.addStep([reward], [1.0], reward);
|
|
this.engine.endTrajectory(builder, Math.max(0, reward));
|
|
return `traj-native-${Date.now()}`;
|
|
}
|
|
|
|
// Fallback Q-learning
|
|
const trajectory = {
|
|
id: `traj-${Date.now()}`,
|
|
state: stateKey,
|
|
action, outcome, reward,
|
|
timestamp: new Date().toISOString(),
|
|
abGroup: this.abTestGroup
|
|
};
|
|
this.trajectories.push(trajectory);
|
|
|
|
// Q-learning update
|
|
if (!this.qTable[stateKey]) {
|
|
this.qTable[stateKey] = { _meta: { lastUpdate: null, updateCount: 0 } };
|
|
}
|
|
|
|
const currentQ = this.qTable[stateKey][action] || 0;
|
|
const updateCount = (this.qTable[stateKey]._meta?.updateCount || 0) + 1;
|
|
const adaptiveLR = Math.max(0.01, this.alpha / Math.sqrt(updateCount));
|
|
|
|
this.qTable[stateKey][action] = Math.min(0.8, Math.max(-0.5,
|
|
currentQ + adaptiveLR * (reward - currentQ)
|
|
));
|
|
|
|
this.qTable[stateKey]._meta = {
|
|
lastUpdate: new Date().toISOString(),
|
|
updateCount
|
|
};
|
|
|
|
return trajectory.id;
|
|
}
|
|
|
|
getBestAction(state, availableActions) {
|
|
const stateKey = this.stateKey(state);
|
|
|
|
if (this.useNative && this.engine) {
|
|
// Use native pattern matching
|
|
const embedding = this.stateToEmbedding(stateKey);
|
|
const patterns = this.engine.findPatterns(embedding, 3);
|
|
|
|
if (patterns.length > 0) {
|
|
// Map pattern to action based on quality
|
|
const bestPattern = patterns[0];
|
|
const confidence = bestPattern.avgQuality || 0;
|
|
|
|
// Select action based on pattern cluster
|
|
const actionIdx = Math.floor(bestPattern.id % availableActions.length);
|
|
return {
|
|
action: availableActions[actionIdx],
|
|
confidence,
|
|
reason: 'native-pattern',
|
|
abGroup: 'native'
|
|
};
|
|
}
|
|
}
|
|
|
|
// Fallback Q-table lookup
|
|
const qValues = this.qTable[stateKey] || {};
|
|
|
|
// A/B Testing
|
|
if (this.abTestGroup === 'control') {
|
|
const action = availableActions[Math.floor(Math.random() * availableActions.length)];
|
|
return { action, confidence: 0, reason: 'control-group', abGroup: 'control' };
|
|
}
|
|
|
|
// Epsilon-greedy exploration
|
|
if (Math.random() < this.epsilon) {
|
|
const action = availableActions[Math.floor(Math.random() * availableActions.length)];
|
|
return { action, confidence: 0, reason: 'exploration', abGroup: 'treatment' };
|
|
}
|
|
|
|
// Exploitation
|
|
let bestAction = availableActions[0];
|
|
let bestQ = -Infinity;
|
|
|
|
for (const action of availableActions) {
|
|
const q = qValues[action] || 0;
|
|
if (q > bestQ) {
|
|
bestQ = q;
|
|
bestAction = action;
|
|
}
|
|
}
|
|
|
|
const confidence = 1 / (1 + Math.exp(-bestQ * 2));
|
|
|
|
return {
|
|
action: bestAction,
|
|
confidence: bestQ > 0 ? confidence : 0,
|
|
reason: bestQ > 0 ? 'learned-preference' : 'default',
|
|
qValues,
|
|
abGroup: 'treatment'
|
|
};
|
|
}
|
|
|
|
stateToEmbedding(state) {
|
|
// Simple hash-based embedding for state
|
|
const embedding = new Array(256).fill(0);
|
|
const chars = state.split('');
|
|
for (let i = 0; i < chars.length; i++) {
|
|
const idx = (chars[i].charCodeAt(0) * (i + 1)) % 256;
|
|
embedding[idx] += 1.0 / chars.length;
|
|
}
|
|
// Normalize
|
|
const norm = Math.sqrt(embedding.reduce((s, x) => s + x * x, 0));
|
|
return embedding.map(x => x / (norm + 1e-8));
|
|
}
|
|
|
|
async forceLearning() {
|
|
if (this.useNative && this.engine) {
|
|
return this.engine.forceLearn();
|
|
}
|
|
return 'fallback-mode';
|
|
}
|
|
|
|
getStats() {
|
|
if (this.useNative && this.engine) {
|
|
return JSON.parse(this.engine.getStats());
|
|
}
|
|
|
|
return {
|
|
patterns: Object.keys(this.qTable).length,
|
|
trajectories: this.trajectories.length,
|
|
mode: 'fallback'
|
|
};
|
|
}
|
|
|
|
async save() {
|
|
if (!this.useNative) {
|
|
// Keep trajectories bounded
|
|
if (this.trajectories.length > 1000) {
|
|
this.trajectories = this.trajectories.slice(-1000);
|
|
}
|
|
writeFileSync(LEGACY_TRAJECTORIES, JSON.stringify(this.trajectories, null, 2));
|
|
writeFileSync(LEGACY_PATTERNS, JSON.stringify(this.qTable, null, 2));
|
|
}
|
|
// Native storage is already persistent
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Native Metadata Storage using simple key-value store
|
|
*/
|
|
export class NativeMetadataStorage {
|
|
constructor(options = {}) {
|
|
this.dbPath = options.dbPath || join(DATA_DIR, 'metadata.json');
|
|
this.data = {};
|
|
}
|
|
|
|
async init() {
|
|
if (existsSync(this.dbPath)) {
|
|
try {
|
|
this.data = JSON.parse(readFileSync(this.dbPath, 'utf-8'));
|
|
} catch (e) {
|
|
this.data = {};
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
get(namespace, key) {
|
|
return this.data[`${namespace}:${key}`];
|
|
}
|
|
|
|
set(namespace, key, value) {
|
|
this.data[`${namespace}:${key}`] = value;
|
|
}
|
|
|
|
delete(namespace, key) {
|
|
delete this.data[`${namespace}:${key}`];
|
|
}
|
|
|
|
list(namespace) {
|
|
const prefix = `${namespace}:`;
|
|
return Object.entries(this.data)
|
|
.filter(([k]) => k.startsWith(prefix))
|
|
.map(([k, v]) => ({ key: k.slice(prefix.length), value: v }));
|
|
}
|
|
|
|
async save() {
|
|
writeFileSync(this.dbPath, JSON.stringify(this.data, null, 2));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Migration utility to move from JSON to native storage
|
|
*/
|
|
export async function migrateToNative(options = {}) {
|
|
const dryRun = options.dryRun || false;
|
|
const results = {
|
|
vectors: 0,
|
|
patterns: 0,
|
|
trajectories: 0,
|
|
errors: []
|
|
};
|
|
|
|
console.log('🚀 Starting migration to RuVector native storage...');
|
|
console.log(` Dry run: ${dryRun}`);
|
|
|
|
// 1. Migrate vector memory
|
|
if (existsSync(LEGACY_MEMORY)) {
|
|
try {
|
|
const memory = JSON.parse(readFileSync(LEGACY_MEMORY, 'utf-8'));
|
|
console.log(`📊 Found ${memory.length} vectors in memory.json`);
|
|
|
|
if (!dryRun && ruvectorCore) {
|
|
const vectorStore = new NativeVectorStorage({ dimensions: 128 });
|
|
await vectorStore.init();
|
|
|
|
for (const item of memory) {
|
|
if (item.embedding && item.embedding.length > 0) {
|
|
await vectorStore.insert(item.id, item.embedding, item.metadata || {});
|
|
results.vectors++;
|
|
}
|
|
}
|
|
console.log(`✅ Migrated ${results.vectors} vectors to native HNSW`);
|
|
} else {
|
|
results.vectors = memory.filter(m => m.embedding).length;
|
|
console.log(` Would migrate ${results.vectors} vectors`);
|
|
}
|
|
} catch (e) {
|
|
results.errors.push(`Vector migration: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
// 2. Migrate patterns/Q-table
|
|
if (existsSync(LEGACY_PATTERNS)) {
|
|
try {
|
|
const patterns = JSON.parse(readFileSync(LEGACY_PATTERNS, 'utf-8'));
|
|
const patternCount = Object.keys(patterns).length;
|
|
console.log(`📊 Found ${patternCount} patterns in patterns.json`);
|
|
|
|
if (!dryRun && sona) {
|
|
const reasoningBank = new NativeReasoningBank();
|
|
await reasoningBank.init();
|
|
|
|
// Convert Q-table entries to trajectories for learning
|
|
for (const [state, actions] of Object.entries(patterns)) {
|
|
if (state.startsWith('_')) continue;
|
|
|
|
for (const [action, qValue] of Object.entries(actions)) {
|
|
if (action === '_meta') continue;
|
|
|
|
reasoningBank.recordTrajectory(
|
|
state,
|
|
action,
|
|
qValue > 0 ? 'success' : 'failure',
|
|
qValue
|
|
);
|
|
results.patterns++;
|
|
}
|
|
}
|
|
|
|
// Force learning to consolidate patterns
|
|
await reasoningBank.forceLearning();
|
|
console.log(`✅ Migrated ${results.patterns} pattern entries to native ReasoningBank`);
|
|
} else {
|
|
results.patterns = Object.keys(patterns).length;
|
|
console.log(` Would migrate ${results.patterns} patterns`);
|
|
}
|
|
} catch (e) {
|
|
results.errors.push(`Pattern migration: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
// 3. Migrate trajectories
|
|
if (existsSync(LEGACY_TRAJECTORIES)) {
|
|
try {
|
|
const trajectories = JSON.parse(readFileSync(LEGACY_TRAJECTORIES, 'utf-8'));
|
|
console.log(`📊 Found ${trajectories.length} trajectories in trajectories.json`);
|
|
|
|
if (!dryRun && sona) {
|
|
const reasoningBank = new NativeReasoningBank();
|
|
await reasoningBank.init();
|
|
|
|
for (const traj of trajectories) {
|
|
reasoningBank.recordTrajectory(
|
|
traj.state,
|
|
traj.action,
|
|
traj.outcome,
|
|
traj.reward
|
|
);
|
|
results.trajectories++;
|
|
}
|
|
console.log(`✅ Migrated ${results.trajectories} trajectories to native storage`);
|
|
} else {
|
|
results.trajectories = trajectories.length;
|
|
console.log(` Would migrate ${results.trajectories} trajectories`);
|
|
}
|
|
} catch (e) {
|
|
results.errors.push(`Trajectory migration: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
console.log('\n📋 Migration Summary:');
|
|
console.log(` Vectors: ${results.vectors}`);
|
|
console.log(` Patterns: ${results.patterns}`);
|
|
console.log(` Trajectories: ${results.trajectories}`);
|
|
|
|
if (results.errors.length > 0) {
|
|
console.log('\n⚠️ Errors:');
|
|
results.errors.forEach(e => console.log(` - ${e}`));
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
// Export all storage classes
|
|
export default {
|
|
NativeVectorStorage,
|
|
NativeReasoningBank,
|
|
NativeMetadataStorage,
|
|
migrateToNative
|
|
};
|