feat: vendor midstream and sublinear-time-solver libraries (#109)

Add ruvnet/midstream (AIMDS real-time inference) and
ruvnet/sublinear-time-solver (sublinear optimization algorithms)
as vendored dependencies under vendor/.
This commit is contained in:
rUv
2026-03-02 23:34:05 -05:00
committed by GitHub
parent 14902e6b4e
commit 407b46b206
1600 changed files with 1852646 additions and 0 deletions
+189
View File
@@ -0,0 +1,189 @@
/**
* Tests for MidStream Agent
*/
import { MidStreamAgent } from '../agent';
describe('MidStreamAgent', () => {
let agent: MidStreamAgent;
beforeEach(() => {
agent = new MidStreamAgent();
});
afterEach(() => {
agent.reset();
});
describe('processMessage', () => {
it('should process a single message', () => {
const result = agent.processMessage('Hello, world!');
expect(result).toBeDefined();
expect(result.processed).toBe(true);
});
it('should maintain conversation history', () => {
agent.processMessage('First message');
agent.processMessage('Second message');
const status = agent.getStatus();
expect(status.conversationHistorySize).toBe(2);
});
it('should respect max history limit', () => {
const smallAgent = new MidStreamAgent({ maxHistory: 5 });
for (let i = 0; i < 10; i++) {
smallAgent.processMessage(`Message ${i}`);
}
const status = smallAgent.getStatus();
expect(status.conversationHistorySize).toBeLessThanOrEqual(5);
});
});
describe('analyzeConversation', () => {
it('should analyze a conversation', () => {
const messages = [
'Hello',
'How are you?',
'What is the weather?',
];
const result = agent.analyzeConversation(messages);
expect(result).toBeDefined();
expect(result.messageCount).toBe(3);
expect(result.patterns).toBeDefined();
expect(result.metaLearning).toBeDefined();
});
it('should handle empty conversation', () => {
const result = agent.analyzeConversation([]);
expect(result.messageCount).toBe(0);
});
});
describe('compareSequences', () => {
it('should compare identical sequences', () => {
const seq = ['a', 'b', 'c'];
const similarity = agent.compareSequences(seq, seq, 'dtw');
expect(similarity).toBeGreaterThan(0.9);
});
it('should compare different sequences', () => {
const seq1 = ['a', 'b', 'c'];
const seq2 = ['x', 'y', 'z'];
const similarity = agent.compareSequences(seq1, seq2, 'dtw');
expect(similarity).toBeLessThan(0.5);
});
it('should detect similarity in overlapping sequences', () => {
const seq1 = ['a', 'b', 'c', 'd'];
const seq2 = ['b', 'c', 'd', 'e'];
const similarity = agent.compareSequences(seq1, seq2, 'dtw');
expect(similarity).toBeGreaterThan(0.5);
});
});
describe('detectPattern', () => {
it('should detect pattern occurrences', () => {
const sequence = ['a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', 'c'];
const pattern = ['a', 'b', 'c'];
const positions = agent.detectPattern(sequence, pattern);
expect(positions).toEqual([0, 3, 7]);
});
it('should return empty array for non-existent pattern', () => {
const sequence = ['a', 'b', 'c'];
const pattern = ['x', 'y'];
const positions = agent.detectPattern(sequence, pattern);
expect(positions).toEqual([]);
});
it('should handle empty pattern', () => {
const sequence = ['a', 'b', 'c'];
const pattern: string[] = [];
const positions = agent.detectPattern(sequence, pattern);
expect(positions).toEqual([]);
});
});
describe('analyzeBehavior', () => {
it('should detect stable behavior', () => {
const stableRewards = [0.8, 0.81, 0.79, 0.80, 0.81];
const analysis = agent.analyzeBehavior(stableRewards);
expect(analysis.isStable).toBe(true);
expect(analysis.isChaotic).toBe(false);
});
it('should detect chaotic behavior', () => {
const chaoticRewards = [0.1, 0.9, 0.2, 0.8, 0.3, 0.7];
const analysis = agent.analyzeBehavior(chaoticRewards);
expect(analysis.isChaotic).toBe(true);
expect(analysis.isStable).toBe(false);
});
});
describe('learn', () => {
it('should perform meta-learning', () => {
agent.learn('Pattern A works well', 0.9);
agent.learn('Pattern B is suboptimal', 0.3);
const summary = agent.getMetaLearningSummary();
expect(summary).toBeDefined();
});
});
describe('getStatus', () => {
it('should return agent status', () => {
agent.processMessage('Test message');
agent.learn('Test learning', 0.8);
const status = agent.getStatus();
expect(status.conversationHistorySize).toBeGreaterThan(0);
expect(status.rewardHistorySize).toBeGreaterThan(0);
expect(status.config).toBeDefined();
expect(status.metaLearning).toBeDefined();
});
it('should calculate average reward', () => {
agent.learn('A', 0.8);
agent.learn('B', 0.6);
agent.learn('C', 0.9);
const status = agent.getStatus();
expect(status.averageReward).toBeCloseTo((0.8 + 0.6 + 0.9) / 3, 2);
});
});
describe('reset', () => {
it('should clear all history', () => {
agent.processMessage('Test');
agent.learn('Test', 0.8);
agent.reset();
const status = agent.getStatus();
expect(status.conversationHistorySize).toBe(0);
expect(status.rewardHistorySize).toBe(0);
});
});
});
+368
View File
@@ -0,0 +1,368 @@
/**
* Integration tests for MidStream
*/
import { MidStreamAgent } from '../agent';
import { WebSocketStreamServer, SSEStreamServer } from '../streaming';
import * as fs from 'fs';
import * as path from 'path';
describe('MidStream Integration Tests', () => {
let agent: MidStreamAgent;
beforeAll(() => {
agent = new MidStreamAgent({
maxHistory: 500,
embeddingDim: 3,
});
});
describe('End-to-End Conversation Analysis', () => {
it('should process and analyze a complete conversation', () => {
const conversation = [
"Hello, I need help with the weather.",
"Of course! Which city are you interested in?",
"San Francisco please.",
"The weather in San Francisco is currently 65°F and partly cloudy.",
"Perfect, thank you!",
];
// Process each message
conversation.forEach(msg => {
agent.processMessage(msg);
});
// Analyze the complete conversation
const analysis = agent.analyzeConversation(conversation);
expect(analysis).toBeDefined();
expect(analysis.messageCount).toBe(5);
expect(analysis.patterns).toBeDefined();
expect(analysis.metaLearning).toBeDefined();
// Check status
const status = agent.getStatus();
expect(status.conversationHistorySize).toBeGreaterThan(0);
});
it('should detect patterns in conversation flow', () => {
const sequence = [
'greeting',
'weather_query',
'location_query',
'weather_response',
'thanks',
];
const pattern = ['weather_query', 'location_query'];
const positions = agent.detectPattern(sequence, pattern);
expect(positions.length).toBeGreaterThan(0);
expect(positions[0]).toBe(1); // Pattern starts at index 1
});
});
describe('Temporal Sequence Comparison', () => {
it('should compare similar conversation patterns', () => {
const pattern1 = [
'greeting',
'weather_query',
'location_query',
'response',
];
const pattern2 = [
'greeting',
'weather_query',
'location_query',
'detailed_response',
];
const similarity = agent.compareSequences(pattern1, pattern2, 'lcs');
expect(similarity).toBeGreaterThan(0.7); // High similarity
});
it('should detect different conversation patterns', () => {
const weatherPattern = [
'greeting',
'weather_query',
'location',
'response',
];
const accountPattern = [
'greeting',
'account_query',
'credentials',
'verification',
];
const similarity = agent.compareSequences(weatherPattern, accountPattern, 'dtw');
expect(similarity).toBeLessThan(0.5); // Low similarity
});
});
describe('Behavior Stability Analysis', () => {
it('should detect stable learning behavior', () => {
const stableRewards = Array(20).fill(0).map((_, i) =>
0.8 + Math.sin(i * 0.1) * 0.05 // Stable with small oscillation
);
const analysis = agent.analyzeBehavior(stableRewards);
expect(analysis.isStable).toBe(true);
expect(analysis.isChaotic).toBe(false);
});
it('should detect chaotic behavior', () => {
const chaoticRewards = Array(20).fill(0).map(() =>
Math.random() // Completely random
);
const analysis = agent.analyzeBehavior(chaoticRewards);
// Chaotic patterns should be detected
expect(analysis.isChaotic).toBe(true);
});
});
describe('Meta-Learning Progression', () => {
it('should demonstrate meta-learning over multiple interactions', () => {
agent.reset(); // Start fresh
// Simulate learning from successful patterns
for (let i = 0; i < 10; i++) {
agent.learn(`Pattern ${i} is successful`, 0.85);
}
// Simulate learning from unsuccessful patterns
for (let i = 0; i < 5; i++) {
agent.learn(`Pattern ${i} failed`, 0.2);
}
const summary = agent.getMetaLearningSummary();
expect(summary).toBeDefined();
expect(summary.currentLevel).toBeDefined();
const status = agent.getStatus();
expect(status.averageReward).toBeGreaterThan(0);
expect(status.rewardHistorySize).toBe(15);
});
});
describe('Real-World Scenario: Customer Support', () => {
it('should handle a customer support conversation', () => {
const conversation = [
'Hi, I have a problem with my order',
'I apologize for the inconvenience. Can you provide your order number?',
'Sure, it\'s ORDER-12345',
'Thank you. I see your order was shipped yesterday. It should arrive in 2-3 days.',
'Oh, I see. When can I expect tracking information?',
'Tracking information has been sent to your email. Check your inbox.',
'Found it! Thank you so much for your help.',
'You\'re welcome! Is there anything else I can help you with?',
'No, that\'s all. Have a great day!',
];
// Process conversation
const analysis = agent.analyzeConversation(conversation);
expect(analysis.messageCount).toBe(9);
// Extract intent flow
const intents = [
'problem_report',
'info_request',
'info_provided',
'status_update',
'followup_question',
'solution_provided',
'gratitude',
'offer_help',
'closure',
];
// Check for common support patterns
const supportPattern = ['problem_report', 'info_request', 'info_provided'];
const positions = agent.detectPattern(intents, supportPattern);
expect(positions.length).toBeGreaterThan(0);
});
});
describe('Performance Benchmarking', () => {
it('should process messages quickly', () => {
const start = Date.now();
for (let i = 0; i < 100; i++) {
agent.processMessage(`Test message ${i}`);
}
const duration = Date.now() - start;
// Should process 100 messages in under 1 second
expect(duration).toBeLessThan(1000);
const avgTime = duration / 100;
console.log(`Average message processing time: ${avgTime.toFixed(2)}ms`);
});
it('should handle large conversations efficiently', () => {
const largeConversation = Array(500).fill(0).map((_, i) =>
`Message number ${i} in a very large conversation`
);
const start = Date.now();
const analysis = agent.analyzeConversation(largeConversation);
const duration = Date.now() - start;
expect(analysis.messageCount).toBe(500);
// Should analyze 500 messages in under 500ms
expect(duration).toBeLessThan(500);
console.log(`Large conversation analysis time: ${duration}ms`);
});
});
describe('Streaming Server Integration', () => {
let wsServer: WebSocketStreamServer;
let sseServer: SSEStreamServer;
beforeAll(async () => {
// Use non-standard ports for testing
wsServer = new WebSocketStreamServer(9001);
sseServer = new SSEStreamServer(9002);
await wsServer.start();
await sseServer.start();
});
afterAll(async () => {
await wsServer.stop();
await sseServer.stop();
});
it('should start WebSocket server', () => {
expect(wsServer).toBeDefined();
});
it('should start SSE server', () => {
expect(sseServer).toBeDefined();
});
it('should broadcast to WebSocket clients', () => {
const testData = {
type: 'test',
message: 'Hello from test',
};
// Should not throw
expect(() => wsServer.broadcast(testData)).not.toThrow();
});
it('should broadcast to SSE clients', () => {
const testData = {
type: 'test',
message: 'Hello from SSE test',
};
// Should not throw
expect(() => sseServer.broadcast(testData)).not.toThrow();
});
});
describe('File-based Examples', () => {
const examplesDir = path.join(__dirname, '../../examples');
it('should process example conversation1.json', () => {
const filePath = path.join(examplesDir, 'conversation1.json');
if (fs.existsSync(filePath)) {
const messages = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
const analysis = agent.analyzeConversation(messages);
expect(analysis.messageCount).toBeGreaterThan(0);
expect(analysis.patterns).toBeDefined();
}
});
it('should compare example sequences', () => {
const seq1Path = path.join(examplesDir, 'sequence1.json');
const seq2Path = path.join(examplesDir, 'sequence2.json');
if (fs.existsSync(seq1Path) && fs.existsSync(seq2Path)) {
const seq1 = JSON.parse(fs.readFileSync(seq1Path, 'utf-8'));
const seq2 = JSON.parse(fs.readFileSync(seq2Path, 'utf-8'));
const similarity = agent.compareSequences(seq1, seq2, 'dtw');
expect(similarity).toBeGreaterThanOrEqual(0);
expect(similarity).toBeLessThanOrEqual(1);
}
});
});
describe('Edge Cases and Error Handling', () => {
it('should handle empty messages', () => {
expect(() => agent.processMessage('')).not.toThrow();
});
it('should handle very long messages', () => {
const longMessage = 'a'.repeat(10000);
expect(() => agent.processMessage(longMessage)).not.toThrow();
});
it('should handle empty conversation analysis', () => {
const result = agent.analyzeConversation([]);
expect(result.messageCount).toBe(0);
});
it('should handle single message conversation', () => {
const result = agent.analyzeConversation(['Hello']);
expect(result.messageCount).toBe(1);
});
it('should handle empty sequences in comparison', () => {
const similarity = agent.compareSequences([], [], 'dtw');
expect(similarity).toBeGreaterThanOrEqual(0);
});
it('should handle empty rewards in behavior analysis', () => {
const analysis = agent.analyzeBehavior([]);
expect(analysis).toBeDefined();
});
});
describe('Memory Management', () => {
it('should respect max history limit', () => {
const smallAgent = new MidStreamAgent({ maxHistory: 10 });
// Add more than max history
for (let i = 0; i < 50; i++) {
smallAgent.processMessage(`Message ${i}`);
}
const status = smallAgent.getStatus();
expect(status.conversationHistorySize).toBeLessThanOrEqual(10);
});
it('should successfully reset state', () => {
// Add some data
agent.processMessage('Test');
agent.learn('Test', 0.8);
// Reset
agent.reset();
// Verify clean state
const status = agent.getStatus();
expect(status.conversationHistorySize).toBe(0);
expect(status.rewardHistorySize).toBe(0);
});
});
});
@@ -0,0 +1,435 @@
/**
* Tests for OpenAI Realtime API Integration
*/
import { OpenAIRealtimeClient, AgenticFlowProxyClient, createDefaultSessionConfig, audioToBase64, base64ToAudio } from '../openai-realtime';
import WebSocket from 'ws';
// Mock WebSocket
jest.mock('ws');
describe('OpenAI Realtime Client', () => {
let client: OpenAIRealtimeClient;
let mockWs: any;
beforeEach(() => {
mockWs = {
on: jest.fn(),
send: jest.fn(),
close: jest.fn(),
};
(WebSocket as any).mockImplementation(() => mockWs);
client = new OpenAIRealtimeClient({
apiKey: 'test-api-key',
model: 'gpt-4o-realtime-preview-2024-10-01',
});
});
afterEach(() => {
jest.clearAllMocks();
});
describe('Connection', () => {
it('should create client with config', () => {
expect(client).toBeDefined();
expect(client.isConnectedToOpenAI()).toBe(false);
});
it('should connect to OpenAI', async () => {
const connectPromise = client.connect();
// Simulate connection
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
expect(client.isConnectedToOpenAI()).toBe(true);
});
it('should handle connection errors', async () => {
const error = new Error('Connection failed');
// Add error listener to prevent unhandled error
const errorListener = jest.fn();
client.on('error', errorListener);
const connectPromise = client.connect();
// Simulate error
const errorHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'error')[1];
const closeHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'close')[1];
errorHandler(error);
closeHandler();
await expect(connectPromise).rejects.toThrow('Connection failed');
expect(errorListener).toHaveBeenCalledWith(error);
});
it('should disconnect gracefully', async () => {
// First connect the client
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
// Then disconnect
client.disconnect();
expect(mockWs.close).toHaveBeenCalled();
});
});
describe('Message Handling', () => {
beforeEach(async () => {
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
});
it('should handle session.created message', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const sessionCreatedHandler = jest.fn();
client.on('session.created', sessionCreatedHandler);
const message = JSON.stringify({
type: 'session.created',
session: { id: 'sess_123' },
});
messageHandler(Buffer.from(message));
expect(sessionCreatedHandler).toHaveBeenCalledWith({ id: 'sess_123' });
expect(client.getSessionId()).toBe('sess_123');
});
it('should handle text delta messages', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const deltaHandler = jest.fn();
client.on('response.text.delta', deltaHandler);
const message = JSON.stringify({
type: 'response.text.delta',
delta: 'Hello ',
});
messageHandler(Buffer.from(message));
expect(deltaHandler).toHaveBeenCalledWith('Hello ');
});
it('should handle audio delta messages', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const audioHandler = jest.fn();
client.on('response.audio.delta', audioHandler);
const message = JSON.stringify({
type: 'response.audio.delta',
delta: 'base64_audio_chunk',
});
messageHandler(Buffer.from(message));
expect(audioHandler).toHaveBeenCalledWith('base64_audio_chunk');
});
it('should handle error messages', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const errorHandler = jest.fn();
client.on('error', errorHandler);
const message = JSON.stringify({
type: 'error',
error: { message: 'API Error' },
});
messageHandler(Buffer.from(message));
expect(errorHandler).toHaveBeenCalled();
});
});
describe('Sending Messages', () => {
beforeEach(async () => {
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
});
it('should send text message', () => {
client.sendText('Hello, world!');
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('conversation.item.create');
expect(sentMessage.item.content[0].text).toBe('Hello, world!');
});
it('should send audio', () => {
client.sendAudio('base64_audio_data');
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('input_audio_buffer.append');
expect(sentMessage.audio).toBe('base64_audio_data');
});
it('should commit audio buffer', () => {
client.commitAudio();
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('input_audio_buffer.commit');
});
it('should clear audio buffer', () => {
client.clearAudio();
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('input_audio_buffer.clear');
});
it('should create response', () => {
client.createResponse({ modalities: ['text'] });
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('response.create');
expect(sentMessage.response.modalities).toEqual(['text']);
});
it('should cancel response', () => {
client.cancelResponse();
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('response.cancel');
});
});
describe('Session Management', () => {
beforeEach(async () => {
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
});
it('should update session configuration', () => {
const config = {
modalities: ['text', 'audio'],
voice: 'alloy',
};
client.updateSession(config as any);
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('session.update');
expect(sentMessage.session).toEqual(config);
});
it('should delete conversation item', () => {
client.deleteItem('item_123');
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('conversation.item.delete');
expect(sentMessage.item_id).toBe('item_123');
});
it('should truncate conversation', () => {
client.truncateConversation('item_123', 0, 1000);
expect(mockWs.send).toHaveBeenCalled();
const sentMessage = JSON.parse(mockWs.send.mock.calls[0][0]);
expect(sentMessage.type).toBe('conversation.item.truncate');
expect(sentMessage.item_id).toBe('item_123');
expect(sentMessage.audio_end_ms).toBe(1000);
});
});
describe('MidStream Integration', () => {
beforeEach(async () => {
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
});
it('should integrate with MidStream agent', () => {
const agent = client.getAgent();
expect(agent).toBeDefined();
});
it('should analyze conversation with MidStream', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
// Simulate conversation items
const message1 = JSON.stringify({
type: 'conversation.item.created',
item: {
id: 'item_1',
type: 'message',
role: 'user',
content: [{ type: 'text', text: 'Hello' }],
},
});
const message2 = JSON.stringify({
type: 'conversation.item.created',
item: {
id: 'item_2',
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'Hi there!' }],
},
});
messageHandler(Buffer.from(message1));
messageHandler(Buffer.from(message2));
const analysis = client.getMidStreamAnalysis();
expect(analysis).toBeDefined();
expect(analysis.messageCount).toBeGreaterThan(0);
});
it('should emit MidStream analysis events', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const analysisHandler = jest.fn();
client.on('midstream.analysis', analysisHandler);
const message = JSON.stringify({
type: 'conversation.item.created',
item: {
id: 'item_1',
type: 'message',
content: [{ type: 'text', text: 'Test message' }],
},
});
messageHandler(Buffer.from(message));
expect(analysisHandler).toHaveBeenCalled();
});
});
describe('Conversation Management', () => {
beforeEach(async () => {
const connectPromise = client.connect();
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
openHandler();
await connectPromise;
});
it('should track conversation items', () => {
const messageHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'message')[1];
const message = JSON.stringify({
type: 'conversation.item.created',
item: {
id: 'item_1',
type: 'message',
content: [{ type: 'text', text: 'Test' }],
},
});
messageHandler(Buffer.from(message));
const conversation = client.getConversation();
expect(conversation.length).toBe(1);
expect(conversation[0].id).toBe('item_1');
});
});
});
describe('AgenticFlowProxyClient', () => {
let proxyClient: AgenticFlowProxyClient;
beforeEach(() => {
proxyClient = new AgenticFlowProxyClient({
baseUrl: 'https://test-proxy.com',
apiKey: 'test-key',
openAiApiKey: 'openai-key',
});
});
it('should create proxy client', () => {
expect(proxyClient).toBeDefined();
});
it('should create realtime session', async () => {
const mockWs = {
on: jest.fn(),
send: jest.fn(),
close: jest.fn(),
};
(WebSocket as any).mockImplementation(() => mockWs);
const sessionPromise = proxyClient.createRealtimeSession({
apiKey: 'openai-key',
});
// Wait a bit for the WebSocket to be created
await new Promise(resolve => setTimeout(resolve, 10));
// Simulate connection
const openHandler = mockWs.on.mock.calls.find((call: any) => call[0] === 'open')[1];
if (openHandler) {
openHandler();
}
const client = await sessionPromise;
expect(client).toBeDefined();
expect(proxyClient.getRealtimeClient()).toBe(client);
});
});
describe('Helper Functions', () => {
it('should convert audio to base64', () => {
const buffer = Buffer.from('test audio data');
const base64 = audioToBase64(buffer);
expect(typeof base64).toBe('string');
expect(base64.length).toBeGreaterThan(0);
});
it('should convert base64 to audio', () => {
const base64 = Buffer.from('test audio data').toString('base64');
const buffer = base64ToAudio(base64);
expect(Buffer.isBuffer(buffer)).toBe(true);
expect(buffer.toString()).toBe('test audio data');
});
it('should create default session config', () => {
const config = createDefaultSessionConfig();
expect(config.modalities).toEqual(['text', 'audio']);
expect(config.voice).toBe('alloy');
expect(config.temperature).toBe(0.8);
expect(config.turn_detection).toBeDefined();
});
});
@@ -0,0 +1,437 @@
/**
* QUIC Integration Tests
*
* Comprehensive tests for QUIC client/server functionality
*/
import {
QuicConnection,
QuicServer,
QuicClient,
QuicStream,
createQuicServer,
connectQuic,
isQuicSupported
} from '../quic-integration.js';
describe('QUIC Integration', () => {
describe('QuicStream', () => {
let stream: QuicStream;
beforeEach(() => {
stream = new QuicStream(1, 0);
});
afterEach(() => {
stream.close();
});
it('should create a stream with ID', () => {
expect(stream.getStreamId()).toBe(1);
});
it('should write data to stream', (done) => {
stream.on('data', (data) => {
expect(data.toString()).toBe('test data');
done();
});
stream.write('test data');
});
it('should close stream', (done) => {
stream.on('close', () => {
expect(stream.isClosed()).toBe(true);
done();
});
stream.close();
});
it('should throw when writing to closed stream', () => {
stream.close();
expect(() => stream.write('test')).toThrow('Stream is closed');
});
it('should set and get priority', () => {
stream.setPriority(5);
expect(stream.getPriority()).toBe(5);
});
it('should handle Buffer data', (done) => {
const buffer = Buffer.from('binary data');
stream.on('data', (data) => {
expect(Buffer.isBuffer(data)).toBe(true);
expect(data.toString()).toBe('binary data');
done();
});
stream.write(buffer);
});
});
describe('QuicConnection', () => {
let connection: QuicConnection;
beforeEach(async () => {
connection = new QuicConnection({
host: 'localhost',
port: 4433
});
});
afterEach(() => {
if (connection.isConnected()) {
connection.close();
}
});
it('should create connection with default config', () => {
expect(connection).toBeDefined();
expect(connection.isConnected()).toBe(false);
});
it('should connect to server', async () => {
const connectPromise = connection.connect();
await expect(connectPromise).resolves.toBeUndefined();
expect(connection.isConnected()).toBe(true);
});
it('should emit connected event', (done) => {
connection.on('connected', () => {
expect(connection.isConnected()).toBe(true);
done();
});
connection.connect();
});
it('should open bidirectional stream', async () => {
await connection.connect();
const stream = await connection.openBiStream();
expect(stream).toBeInstanceOf(QuicStream);
expect(connection.getStreamCount()).toBe(1);
});
it('should open unidirectional stream', async () => {
await connection.connect();
const stream = await connection.openUniStream();
expect(stream).toBeInstanceOf(QuicStream);
});
it('should throw when opening stream before connect', async () => {
await expect(connection.openBiStream()).rejects.toThrow('Not connected');
});
it('should respect max streams limit', async () => {
const smallConnection = new QuicConnection({
maxStreams: 2
});
await smallConnection.connect();
await smallConnection.openBiStream();
await smallConnection.openBiStream();
await expect(smallConnection.openBiStream()).rejects.toThrow('Max streams reached');
smallConnection.close();
});
it('should emit stream event when opening stream', (done) => {
connection.connect().then(() => {
connection.on('stream', (stream) => {
expect(stream).toBeInstanceOf(QuicStream);
done();
});
connection.openBiStream();
});
});
it('should track statistics', async () => {
await connection.connect();
const stream = await connection.openBiStream();
stream.write('test data');
const stats = connection.getStats();
expect(stats.streamsOpened).toBe(1);
expect(stats.bytesSent).toBeGreaterThan(0);
});
it('should close all streams on connection close', async () => {
await connection.connect();
const stream1 = await connection.openBiStream();
const stream2 = await connection.openBiStream();
expect(connection.getStreamCount()).toBe(2);
connection.close();
expect(connection.getStreamCount()).toBe(0);
expect(stream1.isClosed()).toBe(true);
expect(stream2.isClosed()).toBe(true);
});
it('should have MidStream agent', () => {
const agent = connection.getAgent();
expect(agent).toBeDefined();
});
it('should process messages with agent', async () => {
await connection.connect();
const stream = await connection.openBiStream();
stream.write('Hello from QUIC');
const agent = connection.getAgent();
const status = agent.getStatus();
expect(status.conversationHistorySize).toBeGreaterThanOrEqual(0);
});
});
describe('QuicServer', () => {
let server: QuicServer;
beforeEach(() => {
server = new QuicServer({
port: 4434 // Different port to avoid conflicts
});
});
afterEach(() => {
if (server.isListening()) {
server.close();
}
});
it('should create server with config', () => {
expect(server).toBeDefined();
expect(server.isListening()).toBe(false);
});
it('should start listening', async () => {
await server.listen();
expect(server.isListening()).toBe(true);
});
it('should emit listening event', (done) => {
server.on('listening', (port) => {
expect(port).toBe(4434);
expect(server.isListening()).toBe(true);
done();
});
server.listen();
});
it('should track connections', async () => {
await server.listen();
// Server starts with no connections
expect(server.getConnectionCount()).toBe(0);
// Connection count is tested in integration tests
// where actual client connections are made
});
it('should close all connections on server close', async () => {
await server.listen();
// Server should close cleanly even with no connections
server.close();
expect(server.getConnectionCount()).toBe(0);
expect(server.isListening()).toBe(false);
});
it('should handle errors', (done) => {
server.on('error', (error) => {
expect(error).toBeDefined();
done();
});
// Simulate error after listening
server.listen().then(() => {
server.emit('error', new Error('Test error'));
});
});
});
describe('QuicClient', () => {
let client: QuicClient;
beforeEach(() => {
client = new QuicClient();
});
afterEach(() => {
client.disconnect();
});
it('should create client', () => {
expect(client).toBeDefined();
expect(client.getConnection()).toBeNull();
});
it('should connect to server', async () => {
const connection = await client.connect('localhost', 4433);
expect(connection).toBeInstanceOf(QuicConnection);
expect(connection.isConnected()).toBe(true);
expect(client.getConnection()).toBe(connection);
});
it('should disconnect', async () => {
await client.connect('localhost', 4433);
const connection = client.getConnection();
expect(connection?.isConnected()).toBe(true);
client.disconnect();
expect(client.getConnection()).toBeNull();
});
});
describe('Utility Functions', () => {
it('should create QUIC server with defaults', () => {
const server = createQuicServer();
expect(server).toBeInstanceOf(QuicServer);
server.close();
});
it('should create QUIC server with custom config', () => {
const server = createQuicServer({ port: 5000 });
expect(server).toBeInstanceOf(QuicServer);
server.close();
});
it('should connect to QUIC server', async () => {
const connection = await connectQuic('localhost', 4433);
expect(connection).toBeInstanceOf(QuicConnection);
expect(connection.isConnected()).toBe(true);
connection.close();
});
it('should check QUIC support', () => {
const supported = isQuicSupported();
expect(typeof supported).toBe('boolean');
expect(supported).toBe(true);
});
});
describe('Integration Tests', () => {
let server: QuicServer;
let client: QuicConnection;
beforeEach(async () => {
server = createQuicServer({ port: 4435 });
await server.listen();
});
afterEach(() => {
if (client && client.isConnected()) {
client.close();
}
if (server && server.isListening()) {
server.close();
}
});
it('should establish connection and open streams', async () => {
client = await connectQuic('localhost', 4435);
const stream = await client.openBiStream();
expect(stream).toBeInstanceOf(QuicStream);
});
it('should send and process data', async () => {
client = await connectQuic('localhost', 4435);
const stream = await client.openBiStream();
const testData = 'Hello QUIC!';
stream.write(testData);
const agent = client.getAgent();
const status = agent.getStatus();
expect(status.conversationHistorySize).toBeGreaterThanOrEqual(0);
});
it('should handle multiple streams', async () => {
client = await connectQuic('localhost', 4435);
const stream1 = await client.openBiStream();
const stream2 = await client.openBiStream();
const stream3 = await client.openBiStream();
expect(client.getStreamCount()).toBe(3);
stream1.write('Stream 1');
stream2.write('Stream 2');
stream3.write('Stream 3');
const stats = client.getStats();
expect(stats.streamsOpened).toBe(3);
});
it('should handle stream priorities', async () => {
client = await connectQuic('localhost', 4435);
const highPriority = await client.openBiStream({ priority: 10 });
const lowPriority = await client.openBiStream({ priority: 1 });
expect(highPriority.getPriority()).toBe(10);
expect(lowPriority.getPriority()).toBe(1);
});
});
describe('Performance Tests', () => {
it('should handle rapid stream creation', async () => {
const connection = new QuicConnection({ maxStreams: 100 });
await connection.connect();
const streams = [];
const startTime = Date.now();
for (let i = 0; i < 50; i++) {
streams.push(await connection.openBiStream());
}
const duration = Date.now() - startTime;
expect(streams.length).toBe(50);
expect(duration).toBeLessThan(1000); // Should be fast
connection.close();
});
it('should handle large data transfers', async () => {
const connection = new QuicConnection();
await connection.connect();
const stream = await connection.openBiStream();
const largeData = Buffer.alloc(1024 * 1024); // 1 MB
const startTime = Date.now();
stream.write(largeData);
const duration = Date.now() - startTime;
expect(duration).toBeLessThan(100); // Should be fast
const stats = connection.getStats();
expect(stats.bytesSent).toBeGreaterThanOrEqual(largeData.length);
connection.close();
});
});
});
+205
View File
@@ -0,0 +1,205 @@
/**
* MidStream Agent - High-level wrapper for Lean Agentic Learning System
*/
export interface AgentConfig {
maxHistory?: number;
embeddingDim?: number;
schedulingPolicy?: string;
}
export interface AnalysisResult {
messageCount: number;
patterns: any[];
metaLearning: any;
temporalAnalysis?: any;
}
export interface BehaviorAnalysis {
attractorType?: string;
lyapunovExponent?: number;
isStable?: boolean;
isChaotic?: boolean;
}
export class MidStreamAgent {
private wasmAgent: any;
private config: AgentConfig;
private conversationHistory: string[] = [];
private rewardHistory: number[] = [];
constructor(config: AgentConfig = {}) {
this.config = {
maxHistory: config.maxHistory || 1000,
embeddingDim: config.embeddingDim || 3,
schedulingPolicy: config.schedulingPolicy || 'EDF',
};
// Load WASM module
try {
const wasm = require('../wasm/midstream_wasm');
this.wasmAgent = new wasm.MidStreamAgent(this.config);
} catch (error) {
console.warn('WASM module not available, using fallback implementation');
this.wasmAgent = null;
}
}
/**
* Process a single message
*/
processMessage(message: string): any {
this.conversationHistory.push(message);
if (this.conversationHistory.length > this.config.maxHistory!) {
this.conversationHistory.shift();
}
if (this.wasmAgent) {
return this.wasmAgent.process_message(message);
}
// Fallback implementation
return {
processed: true,
message,
timestamp: Date.now(),
};
}
/**
* Analyze a complete conversation
*/
analyzeConversation(messages: string[]): AnalysisResult {
if (this.wasmAgent) {
return this.wasmAgent.analyze_conversation(messages);
}
// Fallback implementation
return {
messageCount: messages.length,
patterns: [],
metaLearning: {
currentLevel: 'Object',
knowledgeCounts: [messages.length, 0, 0, 0],
},
};
}
/**
* Compare two sequences using temporal analysis
*/
compareSequences(seq1: string[], seq2: string[], algorithm: string = 'dtw'): number {
if (this.wasmAgent) {
const comparator = this.wasmAgent.temporal;
return comparator?.compare(seq1, seq2, algorithm) || 0;
}
// Simple fallback: Jaccard similarity
const set1 = new Set(seq1);
const set2 = new Set(seq2);
const intersection = new Set([...set1].filter(x => set2.has(x)));
const union = new Set([...set1, ...set2]);
return intersection.size / union.size;
}
/**
* Detect pattern in sequence
*/
detectPattern(sequence: string[], pattern: string[]): number[] {
const positions: number[] = [];
if (pattern.length === 0 || sequence.length < pattern.length) {
return positions;
}
for (let i = 0; i <= sequence.length - pattern.length; i++) {
let match = true;
for (let j = 0; j < pattern.length; j++) {
if (sequence[i + j] !== pattern[j]) {
match = false;
break;
}
}
if (match) {
positions.push(i);
}
}
return positions;
}
/**
* Analyze behavior using attractor analysis
*/
analyzeBehavior(rewards: number[]): BehaviorAnalysis {
this.rewardHistory.push(...rewards);
if (this.rewardHistory.length > this.config.maxHistory!) {
this.rewardHistory = this.rewardHistory.slice(-this.config.maxHistory!);
}
// Simple stability check (fallback)
const mean = rewards.reduce((a, b) => a + b, 0) / rewards.length;
const variance = rewards.reduce((sum, r) => sum + Math.pow(r - mean, 2), 0) / rewards.length;
const stdDev = Math.sqrt(variance);
return {
isStable: stdDev < 0.1,
isChaotic: stdDev > 0.5,
lyapunovExponent: stdDev > 0.5 ? 0.5 : -0.5,
};
}
/**
* Perform meta-learning
*/
learn(content: string, reward: number): void {
this.rewardHistory.push(reward);
if (this.wasmAgent) {
this.wasmAgent.process_message(content);
}
}
/**
* Get meta-learning summary
*/
getMetaLearningSummary(): any {
if (this.wasmAgent) {
return this.wasmAgent.get_status();
}
return {
currentLevel: 'Object',
knowledgeCounts: [this.conversationHistory.length, 0, 0, 0],
numStrangeLoops: 0,
numModificationRules: 0,
safetyViolations: 0,
};
}
/**
* Get agent status
*/
getStatus(): any {
return {
conversationHistorySize: this.conversationHistory.length,
rewardHistorySize: this.rewardHistory.length,
config: this.config,
metaLearning: this.getMetaLearningSummary(),
averageReward: this.rewardHistory.length > 0
? this.rewardHistory.reduce((a, b) => a + b, 0) / this.rewardHistory.length
: 0,
};
}
/**
* Clear all history
*/
reset(): void {
this.conversationHistory = [];
this.rewardHistory = [];
}
}
+350
View File
@@ -0,0 +1,350 @@
#!/usr/bin/env node
/**
* MidStream CLI - Command-line interface for Lean Agentic Learning System
*/
import { Command } from 'commander';
import chalk from 'chalk';
import ora from 'ora';
import inquirer from 'inquirer';
import { MidStreamAgent } from './agent.js';
import { WebSocketStreamServer, SSEStreamServer } from './streaming.js';
import { MidStreamMCPServer } from './mcp-server.js';
import * as fs from 'fs';
import * as path from 'path';
const program = new Command();
program
.name('midstream')
.description('MidStream - Real-time LLM streaming with Lean Agentic Learning')
.version('0.1.0');
// ============================================================================
// Process command
// ============================================================================
program
.command('process <message>')
.description('Process a message through the agent')
.option('-o, --output <file>', 'Output file for results')
.action(async (message: string, options: any) => {
const spinner = ora('Processing message...').start();
try {
const agent = new MidStreamAgent();
const result = agent.processMessage(message);
spinner.succeed('Message processed');
console.log(chalk.bold('\nResult:'));
console.log(JSON.stringify(result, null, 2));
if (options.output) {
fs.writeFileSync(options.output, JSON.stringify(result, null, 2));
console.log(chalk.green(`\nSaved to ${options.output}`));
}
} catch (error) {
spinner.fail('Processing failed');
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// ============================================================================
// Analyze command
// ============================================================================
program
.command('analyze <file>')
.description('Analyze a conversation from a JSON file')
.option('-o, --output <file>', 'Output file for analysis results')
.action(async (file: string, options: any) => {
const spinner = ora('Analyzing conversation...').start();
try {
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
const messages = Array.isArray(data) ? data : data.messages;
const agent = new MidStreamAgent();
const result = agent.analyzeConversation(messages);
spinner.succeed('Analysis complete');
console.log(chalk.bold('\nAnalysis Results:'));
console.log(JSON.stringify(result, null, 2));
if (options.output) {
fs.writeFileSync(options.output, JSON.stringify(result, null, 2));
console.log(chalk.green(`\nSaved to ${options.output}`));
}
} catch (error) {
spinner.fail('Analysis failed');
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// ============================================================================
// Compare command
// ============================================================================
program
.command('compare <file1> <file2>')
.description('Compare two sequences using temporal analysis')
.option('-a, --algorithm <algo>', 'Algorithm: dtw, lcs, edit, correlation', 'dtw')
.action(async (file1: string, file2: string, options: any) => {
const spinner = ora('Comparing sequences...').start();
try {
const seq1 = JSON.parse(fs.readFileSync(file1, 'utf-8'));
const seq2 = JSON.parse(fs.readFileSync(file2, 'utf-8'));
const agent = new MidStreamAgent();
const similarity = agent.compareSequences(seq1, seq2, options.algorithm);
spinner.succeed('Comparison complete');
console.log(chalk.bold('\nComparison Results:'));
console.log(`Algorithm: ${options.algorithm}`);
console.log(`Similarity: ${similarity.toFixed(4)}`);
if (similarity > 0.8) {
console.log(chalk.green('Very similar sequences'));
} else if (similarity > 0.6) {
console.log(chalk.yellow('Moderately similar sequences'));
} else {
console.log(chalk.red('Different sequences'));
}
} catch (error) {
spinner.fail('Comparison failed');
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// ============================================================================
// Serve command - Start streaming servers
// ============================================================================
program
.command('serve')
.description('Start WebSocket and SSE streaming servers')
.option('-w, --ws-port <port>', 'WebSocket port', '3001')
.option('-s, --sse-port <port>', 'SSE port', '3002')
.action(async (options: any) => {
console.log(chalk.bold('Starting MidStream servers...\n'));
const wsPort = parseInt(options.wsPort);
const ssePort = parseInt(options.ssePort);
const wsServer = new WebSocketStreamServer(wsPort);
const sseServer = new SSEStreamServer(ssePort);
try {
await wsServer.start();
console.log(chalk.green(`✓ WebSocket server: ws://localhost:${wsPort}`));
await sseServer.start();
console.log(chalk.green(`✓ SSE server: http://localhost:${ssePort}`));
console.log(chalk.bold('\nServers running. Press Ctrl+C to stop.\n'));
// Handle graceful shutdown
process.on('SIGINT', async () => {
console.log(chalk.yellow('\nShutting down servers...'));
await wsServer.stop();
await sseServer.stop();
process.exit(0);
});
// Keep process alive
await new Promise(() => {});
} catch (error) {
console.error(chalk.red('Failed to start servers:'));
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// ============================================================================
// MCP command - Start MCP server
// ============================================================================
program
.command('mcp')
.description('Start MCP (Model Context Protocol) server')
.action(async () => {
console.log(chalk.bold('Starting MidStream MCP Server...\n'));
const server = new MidStreamMCPServer();
try {
await server.start();
// Handle graceful shutdown
process.on('SIGINT', async () => {
console.error(chalk.yellow('\nShutting down MCP server...'));
await server.stop();
process.exit(0);
});
} catch (error) {
console.error(chalk.red('Failed to start MCP server:'));
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// ============================================================================
// Interactive mode
// ============================================================================
program
.command('interactive')
.alias('i')
.description('Start interactive mode')
.action(async () => {
console.log(chalk.bold.cyan('MidStream Interactive Mode\n'));
const agent = new MidStreamAgent();
let running = true;
while (running) {
const { action } = await inquirer.prompt([
{
type: 'list',
name: 'action',
message: 'What would you like to do?',
choices: [
'Process message',
'Analyze conversation',
'Compare sequences',
'Check status',
'Exit',
],
},
]);
switch (action) {
case 'Process message':
const { message } = await inquirer.prompt([
{
type: 'input',
name: 'message',
message: 'Enter message:',
},
]);
const result = agent.processMessage(message);
console.log(chalk.bold('\nResult:'));
console.log(JSON.stringify(result, null, 2));
console.log();
break;
case 'Analyze conversation':
const { file } = await inquirer.prompt([
{
type: 'input',
name: 'file',
message: 'Enter conversation file path:',
},
]);
try {
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
const messages = Array.isArray(data) ? data : data.messages;
const analysis = agent.analyzeConversation(messages);
console.log(chalk.bold('\nAnalysis:'));
console.log(JSON.stringify(analysis, null, 2));
console.log();
} catch (error) {
console.error(chalk.red('Error:', error));
}
break;
case 'Compare sequences':
const { file1, file2, algorithm } = await inquirer.prompt([
{
type: 'input',
name: 'file1',
message: 'First sequence file:',
},
{
type: 'input',
name: 'file2',
message: 'Second sequence file:',
},
{
type: 'list',
name: 'algorithm',
message: 'Algorithm:',
choices: ['dtw', 'lcs', 'edit', 'correlation'],
},
]);
try {
const seq1 = JSON.parse(fs.readFileSync(file1, 'utf-8'));
const seq2 = JSON.parse(fs.readFileSync(file2, 'utf-8'));
const similarity = agent.compareSequences(seq1, seq2, algorithm);
console.log(chalk.bold(`\nSimilarity: ${similarity.toFixed(4)}\n`));
} catch (error) {
console.error(chalk.red('Error:', error));
}
break;
case 'Check status':
const status = agent.getStatus();
console.log(chalk.bold('\nAgent Status:'));
console.log(JSON.stringify(status, null, 2));
console.log();
break;
case 'Exit':
running = false;
console.log(chalk.cyan('\nGoodbye!\n'));
break;
}
}
});
// ============================================================================
// Benchmark command
// ============================================================================
program
.command('benchmark')
.description('Run performance benchmarks')
.option('-s, --size <size>', 'Sequence size for benchmarks', '100')
.option('-i, --iterations <iterations>', 'Number of iterations', '1000')
.action(async (options: any) => {
const size = parseInt(options.size);
const iterations = parseInt(options.iterations);
console.log(chalk.bold('Running benchmarks...\n'));
try {
const wasm = require('../wasm/midstream_wasm');
const dtwTime = wasm.benchmark_dtw(size, iterations);
const lcsTime = wasm.benchmark_lcs(size, iterations);
console.log(chalk.bold('Benchmark Results:'));
console.log(`Sequence size: ${size}`);
console.log(`Iterations: ${iterations}\n`);
console.log(`DTW: ${dtwTime.toFixed(3)}ms per iteration`);
console.log(`LCS: ${lcsTime.toFixed(3)}ms per iteration`);
if (dtwTime < 10) {
console.log(chalk.green('\n✓ Excellent performance'));
} else if (dtwTime < 50) {
console.log(chalk.yellow('\n⚠ Good performance'));
} else {
console.log(chalk.red('\n✗ Consider optimization'));
}
} catch (error) {
console.error(chalk.red('Benchmark failed:'));
console.error(chalk.red(error instanceof Error ? error.message : String(error)));
process.exit(1);
}
});
// Parse command line
program.parse();
+442
View File
@@ -0,0 +1,442 @@
/**
* MidStream Real-Time Dashboard
*
* Minimal console-based dashboard with WASM and streaming support
* for real-time display of MidStream introspection capabilities.
*
* Created by rUv
*/
import { MidStreamAgent, AnalysisResult } from './agent.js';
import { OpenAIRealtimeClient } from './openai-realtime.js';
import chalk from 'chalk';
import * as readline from 'readline';
// ============================================================================
// Dashboard State Management
// ============================================================================
interface DashboardState {
messageCount: number;
totalTokens: number;
patternsDetected: string[];
attractorType: string;
lyapunovExponent: number;
isStable: boolean;
isChaotic: boolean;
avgReward: number;
recentMessages: string[];
audioStreaming: boolean;
videoStreaming: boolean;
lastUpdate: Date;
fps: number;
latency: number;
}
interface StreamMetrics {
type: 'audio' | 'video' | 'text';
bytesProcessed: number;
chunksReceived: number;
avgChunkSize: number;
startTime: number;
lastChunkTime: number;
}
// ============================================================================
// Dashboard Class
// ============================================================================
export class MidStreamDashboard {
private agent: MidStreamAgent;
private state: DashboardState;
private streamMetrics: Map<string, StreamMetrics>;
private updateInterval: NodeJS.Timeout | null = null;
private startTime: number;
private frameCount: number = 0;
constructor() {
this.agent = new MidStreamAgent({
maxHistory: 1000,
embeddingDim: 3,
});
this.state = {
messageCount: 0,
totalTokens: 0,
patternsDetected: [],
attractorType: 'unknown',
lyapunovExponent: 0,
isStable: true,
isChaotic: false,
avgReward: 0,
recentMessages: [],
audioStreaming: false,
videoStreaming: false,
lastUpdate: new Date(),
fps: 0,
latency: 0,
};
this.streamMetrics = new Map();
this.startTime = Date.now();
}
// ==========================================================================
// Core Dashboard Methods
// ==========================================================================
/**
* Start the dashboard with real-time updates
*/
start(refreshRate: number = 100): void {
this.clearScreen();
this.render();
this.updateInterval = setInterval(() => {
this.frameCount++;
this.state.fps = Math.round(this.frameCount / ((Date.now() - this.startTime) / 1000));
this.clearScreen();
this.render();
}, refreshRate);
}
/**
* Stop the dashboard
*/
stop(): void {
if (this.updateInterval) {
clearInterval(this.updateInterval);
this.updateInterval = null;
}
}
/**
* Process a message and update dashboard state
*/
processMessage(message: string, tokens: number = 0): void {
const startTime = Date.now();
// Process with agent
this.agent.processMessage(message);
// Update state
this.state.messageCount++;
this.state.totalTokens += tokens;
this.state.recentMessages.unshift(message.substring(0, 50) + '...');
if (this.state.recentMessages.length > 5) {
this.state.recentMessages.pop();
}
// Get analysis
const analysis = this.agent.getStatus();
this.updateFromAnalysis(analysis);
// Calculate latency
this.state.latency = Date.now() - startTime;
this.state.lastUpdate = new Date();
}
/**
* Process streaming data (audio/video/text)
*/
processStream(streamId: string, data: Buffer, type: 'audio' | 'video' | 'text'): void {
let metrics = this.streamMetrics.get(streamId);
if (!metrics) {
metrics = {
type,
bytesProcessed: 0,
chunksReceived: 0,
avgChunkSize: 0,
startTime: Date.now(),
lastChunkTime: Date.now(),
};
this.streamMetrics.set(streamId, metrics);
}
// Update metrics
metrics.bytesProcessed += data.length;
metrics.chunksReceived++;
metrics.avgChunkSize = Math.round(metrics.bytesProcessed / metrics.chunksReceived);
metrics.lastChunkTime = Date.now();
// Update state
if (type === 'audio') {
this.state.audioStreaming = true;
} else if (type === 'video') {
this.state.videoStreaming = true;
}
}
/**
* Update dashboard from analysis results
*/
private updateFromAnalysis(analysis: AnalysisResult): void {
this.state.patternsDetected = analysis.patterns
.slice(0, 5)
.map((p: any) => `${p.type || 'pattern'} (${p.confidence || 0}%)`);
if (analysis.temporalAnalysis) {
this.state.attractorType = analysis.temporalAnalysis.attractorType || 'unknown';
this.state.lyapunovExponent = analysis.temporalAnalysis.lyapunovExponent || 0;
this.state.isStable = analysis.temporalAnalysis.isStable || false;
this.state.isChaotic = analysis.temporalAnalysis.isChaotic || false;
}
if (analysis.metaLearning) {
this.state.avgReward = analysis.metaLearning.avgReward || 0;
}
}
// ==========================================================================
// Rendering Methods
// ==========================================================================
/**
* Clear the console screen
*/
private clearScreen(): void {
process.stdout.write('\x1b[2J\x1b[0f');
}
/**
* Render the dashboard
*/
private render(): void {
const width = process.stdout.columns || 80;
const separator = '─'.repeat(width);
console.log(chalk.bold.cyan('╔' + '═'.repeat(width - 2) + '╗'));
console.log(
chalk.bold.cyan('║') +
this.centerText('MidStream Real-Time Dashboard', width - 2) +
chalk.bold.cyan('║')
);
console.log(
chalk.bold.cyan('║') +
this.centerText('Created by rUv', width - 2) +
chalk.bold.cyan('║')
);
console.log(chalk.bold.cyan('╚' + '═'.repeat(width - 2) + '╝'));
// System Metrics
this.renderSection('System Metrics', [
`Messages Processed: ${chalk.green(this.state.messageCount)}`,
`Total Tokens: ${chalk.green(this.state.totalTokens)}`,
`FPS: ${chalk.yellow(this.state.fps)}`,
`Latency: ${chalk.yellow(this.state.latency + 'ms')}`,
`Uptime: ${chalk.cyan(this.formatUptime())}`,
]);
// Temporal Analysis
this.renderSection('Temporal Analysis', [
`Attractor Type: ${this.colorizeAttractor(this.state.attractorType)}`,
`Lyapunov Exp: ${this.colorizeLyapunov(this.state.lyapunovExponent)}`,
`Stability: ${this.state.isStable ? chalk.green('STABLE') : chalk.red('UNSTABLE')}`,
`Chaos: ${this.state.isChaotic ? chalk.red('CHAOTIC') : chalk.green('ORDERED')}`,
`Avg Reward: ${chalk.cyan(this.state.avgReward.toFixed(3))}`,
]);
// Pattern Detection
this.renderSection(
'Pattern Detection',
this.state.patternsDetected.length > 0
? this.state.patternsDetected.map((p) => chalk.magenta('• ' + p))
: [chalk.gray('No patterns detected yet')]
);
// Streaming Status
this.renderStreamingStatus();
// Recent Messages
this.renderSection(
'Recent Messages',
this.state.recentMessages.length > 0
? this.state.recentMessages.map((m) => chalk.gray('• ' + m))
: [chalk.gray('No messages yet')]
);
// Stream Metrics
this.renderStreamMetrics();
// Footer
console.log(chalk.gray(separator));
console.log(
chalk.gray(
`Last Update: ${this.state.lastUpdate.toLocaleTimeString()} | Press Ctrl+C to exit`
)
);
}
/**
* Render a section with title and content
*/
private renderSection(title: string, lines: string[]): void {
const width = process.stdout.columns || 80;
console.log('\n' + chalk.bold.white(title));
console.log(chalk.gray('─'.repeat(width)));
lines.forEach((line) => console.log(line));
}
/**
* Render streaming status
*/
private renderStreamingStatus(): void {
const audioStatus = this.state.audioStreaming
? chalk.green('● ACTIVE')
: chalk.gray('○ INACTIVE');
const videoStatus = this.state.videoStreaming
? chalk.green('● ACTIVE')
: chalk.gray('○ INACTIVE');
this.renderSection('Streaming Status', [
`Audio: ${audioStatus}`,
`Video: ${videoStatus}`,
`Streams: ${chalk.cyan(this.streamMetrics.size)} active`,
]);
}
/**
* Render stream metrics
*/
private renderStreamMetrics(): void {
if (this.streamMetrics.size === 0) {
return;
}
const metrics: string[] = [];
this.streamMetrics.forEach((metric, streamId) => {
const duration = (Date.now() - metric.startTime) / 1000;
const rate = (metric.bytesProcessed / duration / 1024).toFixed(2);
metrics.push(
chalk.cyan(`${streamId} (${metric.type}):`) +
` ${chalk.yellow(metric.chunksReceived)} chunks, ` +
`${chalk.yellow(this.formatBytes(metric.bytesProcessed))}, ` +
`${chalk.yellow(rate)} KB/s`
);
});
this.renderSection('Stream Metrics', metrics);
}
// ==========================================================================
// Helper Methods
// ==========================================================================
/**
* Center text within a given width
*/
private centerText(text: string, width: number): string {
const padding = Math.max(0, width - text.length);
const leftPad = Math.floor(padding / 2);
const rightPad = padding - leftPad;
return ' '.repeat(leftPad) + text + ' '.repeat(rightPad);
}
/**
* Format uptime
*/
private formatUptime(): string {
const seconds = Math.floor((Date.now() - this.startTime) / 1000);
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = seconds % 60;
return `${hours}h ${minutes}m ${secs}s`;
}
/**
* Format bytes
*/
private formatBytes(bytes: number): string {
if (bytes < 1024) return bytes + ' B';
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(2) + ' KB';
return (bytes / (1024 * 1024)).toFixed(2) + ' MB';
}
/**
* Colorize attractor type
*/
private colorizeAttractor(type: string): string {
const colors: Record<string, any> = {
fixed: chalk.green,
periodic: chalk.blue,
chaotic: chalk.red,
unknown: chalk.gray,
};
const color = colors[type] || chalk.white;
return color(type.toUpperCase());
}
/**
* Colorize Lyapunov exponent
*/
private colorizeLyapunov(value: number): string {
const str = value.toFixed(4);
if (value < 0) return chalk.green(str);
if (value > 0) return chalk.red(str);
return chalk.yellow(str);
}
/**
* Get current agent
*/
getAgent(): MidStreamAgent {
return this.agent;
}
/**
* Get current state
*/
getState(): DashboardState {
return { ...this.state };
}
}
// ============================================================================
// Interactive Dashboard
// ============================================================================
export class InteractiveDashboard extends MidStreamDashboard {
private rl: readline.Interface | null = null;
/**
* Start interactive mode with user input
*/
startInteractive(): void {
this.start(100);
this.rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
// Setup readline to capture input without blocking display
process.stdin.on('keypress', (str, key) => {
if (key.ctrl && key.name === 'c') {
this.stop();
if (this.rl) {
this.rl.close();
}
process.exit(0);
}
});
}
/**
* Stop interactive mode
*/
stopInteractive(): void {
this.stop();
if (this.rl) {
this.rl.close();
this.rl = null;
}
}
}
// ============================================================================
// Exports
// ============================================================================
export { DashboardState, StreamMetrics };
+64
View File
@@ -0,0 +1,64 @@
/**
* MidStream - Real-time LLM Streaming with Lean Agentic Learning
*
* Main exports for npm package
*/
export { MidStreamAgent } from './agent.js';
export { WebSocketStreamServer, SSEStreamServer, HTTPStreamingClient } from './streaming.js';
export { MidStreamMCPServer } from './mcp-server.js';
export {
OpenAIRealtimeClient,
AgenticFlowProxyClient,
createDefaultSessionConfig,
audioToBase64,
base64ToAudio
} from './openai-realtime.js';
export { MidStreamDashboard, InteractiveDashboard } from './dashboard.js';
export {
RestreamClient,
WebRTCSignalingServer,
StreamSimulator
} from './restream-integration.js';
export {
QuicConnection,
QuicServer,
QuicClient,
QuicStream,
createQuicServer,
connectQuic,
isQuicSupported
} from './quic-integration.js';
// Re-export types
export type {
AgentConfig,
AnalysisResult,
BehaviorAnalysis,
} from './agent.js';
export type {
RealtimeConfig,
SessionConfig,
ConversationItem,
RealtimeMessage,
} from './openai-realtime.js';
export type {
DashboardState,
StreamMetrics
} from './dashboard.js';
export type {
RestreamConfig,
StreamFrame,
AudioChunk,
StreamAnalysis,
DetectedObject
} from './restream-integration.js';
export type {
QuicConfig,
QuicStreamConfig,
QuicConnectionStats
} from './quic-integration.js';
+410
View File
@@ -0,0 +1,410 @@
#!/usr/bin/env node
/**
* MidStream MCP (Model Context Protocol) Server
*
* Provides MCP interface for the Lean Agentic Learning System
*/
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
CallToolRequestSchema,
ListToolsRequestSchema,
Tool,
} from '@modelcontextprotocol/sdk/types.js';
import { MidStreamAgent } from './agent.js';
import { WebSocketStreamServer, SSEStreamServer } from './streaming.js';
interface MCPConfig {
port?: number;
wsPort?: number;
ssePort?: number;
maxHistory?: number;
}
class MidStreamMCPServer {
private server: Server;
private agent: MidStreamAgent;
private wsServer?: WebSocketStreamServer;
private sseServer?: SSEStreamServer;
private config: MCPConfig;
constructor(config: MCPConfig = {}) {
this.config = {
port: config.port || 3000,
wsPort: config.wsPort || 3001,
ssePort: config.ssePort || 3002,
maxHistory: config.maxHistory || 1000,
};
this.server = new Server(
{
name: 'midstream-server',
version: '0.1.0',
},
{
capabilities: {
tools: {},
},
}
);
this.agent = new MidStreamAgent({
maxHistory: this.config.maxHistory,
});
this.setupHandlers();
}
private setupHandlers(): void {
// List available tools
this.server.setRequestHandler(ListToolsRequestSchema, async () => {
return {
tools: this.getTools(),
};
});
// Handle tool calls
this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
try {
switch (name) {
case 'analyze_conversation':
return await this.analyzeConversation(args);
case 'compare_sequences':
return await this.compareSequences(args);
case 'detect_patterns':
return await this.detectPatterns(args);
case 'analyze_behavior':
return await this.analyzeBehavior(args);
case 'meta_learn':
return await this.metaLearn(args);
case 'get_status':
return await this.getStatus();
case 'stream_websocket':
return await this.setupWebSocket(args);
case 'stream_sse':
return await this.setupSSE(args);
default:
throw new Error(`Unknown tool: ${name}`);
}
} catch (error) {
return {
content: [
{
type: 'text' as const,
text: `Error: ${error instanceof Error ? error.message : String(error)}`,
},
],
};
}
});
}
private getTools(): Tool[] {
return [
{
name: 'analyze_conversation',
description: 'Analyze a conversation thread using temporal analysis and meta-learning',
inputSchema: {
type: 'object' as const,
properties: {
messages: {
type: 'array',
items: { type: 'string' },
description: 'Array of conversation messages',
},
},
required: ['messages'],
},
},
{
name: 'compare_sequences',
description: 'Compare two sequences using DTW, LCS, or edit distance',
inputSchema: {
type: 'object' as const,
properties: {
sequence1: {
type: 'array',
items: { type: 'string' },
description: 'First sequence',
},
sequence2: {
type: 'array',
items: { type: 'string' },
description: 'Second sequence',
},
algorithm: {
type: 'string',
enum: ['dtw', 'lcs', 'edit', 'correlation'],
description: 'Comparison algorithm',
},
},
required: ['sequence1', 'sequence2', 'algorithm'],
},
},
{
name: 'detect_patterns',
description: 'Detect pattern occurrences in a sequence',
inputSchema: {
type: 'object' as const,
properties: {
sequence: {
type: 'array',
items: { type: 'string' },
description: 'Sequence to search',
},
pattern: {
type: 'array',
items: { type: 'string' },
description: 'Pattern to find',
},
},
required: ['sequence', 'pattern'],
},
},
{
name: 'analyze_behavior',
description: 'Analyze agent behavior for chaos/stability using attractor analysis',
inputSchema: {
type: 'object' as const,
properties: {
rewards: {
type: 'array',
items: { type: 'number' },
description: 'Reward history',
},
},
required: ['rewards'],
},
},
{
name: 'meta_learn',
description: 'Perform meta-learning on a learning event',
inputSchema: {
type: 'object' as const,
properties: {
content: {
type: 'string',
description: 'Learning content',
},
reward: {
type: 'number',
description: 'Reward value',
},
},
required: ['content', 'reward'],
},
},
{
name: 'get_status',
description: 'Get current agent status and configuration',
inputSchema: {
type: 'object' as const,
properties: {},
},
},
{
name: 'stream_websocket',
description: 'Start WebSocket streaming server',
inputSchema: {
type: 'object' as const,
properties: {
port: {
type: 'number',
description: 'WebSocket port',
},
},
},
},
{
name: 'stream_sse',
description: 'Start SSE streaming server',
inputSchema: {
type: 'object' as const,
properties: {
port: {
type: 'number',
description: 'SSE port',
},
},
},
},
];
}
private async analyzeConversation(args: any) {
const { messages } = args;
const result = this.agent.analyzeConversation(messages);
return {
content: [
{
type: 'text' as const,
text: JSON.stringify(result, null, 2),
},
],
};
}
private async compareSequences(args: any) {
const { sequence1, sequence2, algorithm } = args;
const similarity = this.agent.compareSequences(sequence1, sequence2, algorithm);
return {
content: [
{
type: 'text' as const,
text: JSON.stringify({
algorithm,
similarity,
interpretation: similarity > 0.8 ? 'Very similar' :
similarity > 0.6 ? 'Moderately similar' :
similarity > 0.4 ? 'Somewhat similar' : 'Different',
}, null, 2),
},
],
};
}
private async detectPatterns(args: any) {
const { sequence, pattern } = args;
const positions = this.agent.detectPattern(sequence, pattern);
return {
content: [
{
type: 'text' as const,
text: JSON.stringify({
pattern_found: positions.length > 0,
occurrences: positions.length,
positions,
}, null, 2),
},
],
};
}
private async analyzeBehavior(args: any) {
const { rewards } = args;
const analysis = this.agent.analyzeBehavior(rewards);
return {
content: [
{
type: 'text' as const,
text: JSON.stringify(analysis, null, 2),
},
],
};
}
private async metaLearn(args: any) {
const { content, reward } = args;
this.agent.learn(content, reward);
const summary = this.agent.getMetaLearningSummary();
return {
content: [
{
type: 'text' as const,
text: JSON.stringify(summary, null, 2),
},
],
};
}
private async getStatus() {
const status = this.agent.getStatus();
return {
content: [
{
type: 'text' as const,
text: JSON.stringify(status, null, 2),
},
],
};
}
private async setupWebSocket(args: any) {
const port = args?.port || this.config.wsPort;
if (!this.wsServer) {
this.wsServer = new WebSocketStreamServer(port);
await this.wsServer.start();
}
return {
content: [
{
type: 'text' as const,
text: `WebSocket server started on port ${port}`,
},
],
};
}
private async setupSSE(args: any) {
const port = args?.port || this.config.ssePort;
if (!this.sseServer) {
this.sseServer = new SSEStreamServer(port);
await this.sseServer.start();
}
return {
content: [
{
type: 'text' as const,
text: `SSE server started on port ${port}`,
},
],
};
}
async start(): Promise<void> {
const transport = new StdioServerTransport();
await this.server.connect(transport);
console.error('MidStream MCP Server started');
console.error('Available tools:');
this.getTools().forEach(tool => {
console.error(` - ${tool.name}: ${tool.description}`);
});
}
async stop(): Promise<void> {
if (this.wsServer) {
await this.wsServer.stop();
}
if (this.sseServer) {
await this.sseServer.stop();
}
await this.server.close();
}
}
// Start server if run directly
if (require.main === module) {
const server = new MidStreamMCPServer();
process.on('SIGINT', async () => {
await server.stop();
process.exit(0);
});
server.start().catch(console.error);
}
export { MidStreamMCPServer };
+576
View File
@@ -0,0 +1,576 @@
/**
* OpenAI Realtime API Integration with MidStream
*
* Provides real-time audio and text streaming with OpenAI's Realtime API
* Integrates with MidStream's temporal analysis and agentic-flow proxy
*/
import WebSocket from 'ws';
import { EventEmitter } from 'events';
import { MidStreamAgent } from './agent.js';
// ============================================================================
// Types and Interfaces
// ============================================================================
export interface RealtimeConfig {
apiKey: string;
model?: string;
voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
temperature?: number;
maxTokens?: number;
agenticFlowProxy?: string;
agenticFlowApiKey?: string;
}
export interface RealtimeMessage {
type: string;
[key: string]: any;
}
export interface ConversationItem {
id: string;
type: 'message' | 'function_call' | 'function_call_output';
role?: 'user' | 'assistant' | 'system';
content?: Array<{
type: 'text' | 'audio';
text?: string;
audio?: string;
transcript?: string;
}>;
status?: 'completed' | 'in_progress' | 'incomplete';
}
export interface SessionConfig {
modalities: Array<'text' | 'audio'>;
instructions?: string;
voice?: string;
input_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
output_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
input_audio_transcription?: {
model: string;
};
turn_detection?: {
type: 'server_vad';
threshold?: number;
prefix_padding_ms?: number;
silence_duration_ms?: number;
};
tools?: Array<{
type: 'function';
name: string;
description: string;
parameters: object;
}>;
temperature?: number;
max_response_output_tokens?: number;
}
// ============================================================================
// OpenAI Realtime Client
// ============================================================================
export class OpenAIRealtimeClient extends EventEmitter {
private ws: WebSocket | null = null;
private config: RealtimeConfig;
private sessionId: string | null = null;
private conversationItems: ConversationItem[] = [];
private agent: MidStreamAgent;
private isConnected: boolean = false;
private reconnectAttempts: number = 0;
private maxReconnectAttempts: number = 5;
private messageQueue: RealtimeMessage[] = [];
constructor(config: RealtimeConfig) {
super();
this.config = {
model: config.model || 'gpt-4o-realtime-preview-2024-10-01',
voice: config.voice || 'alloy',
temperature: config.temperature || 0.8,
maxTokens: config.maxTokens || 4096,
...config,
};
this.agent = new MidStreamAgent({
maxHistory: 1000,
embeddingDim: 3,
});
}
/**
* Connect to OpenAI Realtime API
*/
async connect(): Promise<void> {
return new Promise((resolve, reject) => {
try {
// OpenAI Realtime API WebSocket URL
const url = this.config.agenticFlowProxy
? `${this.config.agenticFlowProxy}/realtime`
: 'wss://api.openai.com/v1/realtime';
const headers: any = {
'Authorization': `Bearer ${this.config.apiKey}`,
'OpenAI-Beta': 'realtime=v1',
};
if (this.config.agenticFlowApiKey) {
headers['X-Agentic-Flow-Key'] = this.config.agenticFlowApiKey;
}
this.ws = new WebSocket(`${url}?model=${this.config.model}`, {
headers,
});
this.ws.on('open', () => {
this.isConnected = true;
this.reconnectAttempts = 0;
this.emit('connected');
// Send queued messages
this.flushMessageQueue();
resolve();
});
this.ws.on('message', (data: Buffer) => {
this.handleMessage(JSON.parse(data.toString()));
});
this.ws.on('error', (error: Error) => {
this.emit('error', error);
reject(error);
});
this.ws.on('close', () => {
this.isConnected = false;
this.emit('disconnected');
this.handleReconnect();
});
} catch (error) {
reject(error);
}
});
}
/**
* Disconnect from OpenAI Realtime API
*/
disconnect(): void {
if (this.ws) {
this.ws.close();
this.ws = null;
this.isConnected = false;
}
}
/**
* Handle incoming messages from OpenAI
*/
private handleMessage(message: RealtimeMessage): void {
this.emit('message', message);
switch (message.type) {
case 'session.created':
this.sessionId = message.session.id;
this.emit('session.created', message.session);
break;
case 'session.updated':
this.emit('session.updated', message.session);
break;
case 'conversation.item.created':
this.conversationItems.push(message.item);
this.emit('conversation.item.created', message.item);
// Analyze with MidStream
if (message.item.content) {
this.analyzeWithMidStream(message.item);
}
break;
case 'conversation.item.input_audio_transcription.completed':
this.emit('transcription.completed', message);
break;
case 'response.created':
this.emit('response.created', message.response);
break;
case 'response.done':
this.emit('response.done', message.response);
break;
case 'response.output_item.added':
this.emit('response.output_item.added', message.item);
break;
case 'response.output_item.done':
this.emit('response.output_item.done', message.item);
break;
case 'response.content_part.added':
this.emit('response.content_part.added', message.part);
break;
case 'response.content_part.done':
this.emit('response.content_part.done', message.part);
break;
case 'response.text.delta':
this.emit('response.text.delta', message.delta);
break;
case 'response.text.done':
this.emit('response.text.done', message.text);
break;
case 'response.audio.delta':
this.emit('response.audio.delta', message.delta);
break;
case 'response.audio.done':
this.emit('response.audio.done', message);
break;
case 'response.audio_transcript.delta':
this.emit('response.audio_transcript.delta', message.delta);
break;
case 'response.audio_transcript.done':
this.emit('response.audio_transcript.done', message.transcript);
break;
case 'response.function_call_arguments.delta':
this.emit('response.function_call_arguments.delta', message.delta);
break;
case 'response.function_call_arguments.done':
this.emit('response.function_call_arguments.done', message.arguments);
break;
case 'rate_limits.updated':
this.emit('rate_limits.updated', message.rate_limits);
break;
case 'error':
this.emit('error', new Error(message.error?.message || 'Unknown error'));
break;
default:
this.emit('unknown_message', message);
}
}
/**
* Analyze conversation with MidStream
*/
private analyzeWithMidStream(item: ConversationItem): void {
if (!item.content) return;
for (const content of item.content) {
if (content.type === 'text' && content.text) {
this.agent.processMessage(content.text);
} else if (content.type === 'audio' && content.transcript) {
this.agent.processMessage(content.transcript);
}
}
// Emit analysis results
const status = this.agent.getStatus();
this.emit('midstream.analysis', status);
}
/**
* Send a message to OpenAI
*/
private send(message: RealtimeMessage): void {
if (!this.isConnected || !this.ws) {
this.messageQueue.push(message);
return;
}
this.ws.send(JSON.stringify(message));
}
/**
* Flush queued messages
*/
private flushMessageQueue(): void {
while (this.messageQueue.length > 0) {
const message = this.messageQueue.shift();
if (message && this.ws) {
this.ws.send(JSON.stringify(message));
}
}
}
/**
* Handle reconnection
*/
private handleReconnect(): void {
if (this.reconnectAttempts >= this.maxReconnectAttempts) {
this.emit('reconnect_failed');
return;
}
this.reconnectAttempts++;
const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts), 30000);
setTimeout(() => {
this.emit('reconnecting', this.reconnectAttempts);
this.connect().catch(() => {
// Will retry in handleReconnect
});
}, delay);
}
/**
* Update session configuration
*/
updateSession(config: Partial<SessionConfig>): void {
this.send({
type: 'session.update',
session: config,
});
}
/**
* Send text message
*/
sendText(text: string): void {
this.send({
type: 'conversation.item.create',
item: {
type: 'message',
role: 'user',
content: [
{
type: 'text',
text,
},
],
},
});
this.createResponse();
}
/**
* Send audio message
*/
sendAudio(audio: string): void {
this.send({
type: 'input_audio_buffer.append',
audio,
});
}
/**
* Commit audio buffer
*/
commitAudio(): void {
this.send({
type: 'input_audio_buffer.commit',
});
}
/**
* Clear audio buffer
*/
clearAudio(): void {
this.send({
type: 'input_audio_buffer.clear',
});
}
/**
* Create a response
*/
createResponse(config?: {
modalities?: Array<'text' | 'audio'>;
instructions?: string;
voice?: string;
temperature?: number;
max_output_tokens?: number;
}): void {
this.send({
type: 'response.create',
response: config || {},
});
}
/**
* Cancel current response
*/
cancelResponse(): void {
this.send({
type: 'response.cancel',
});
}
/**
* Truncate conversation
*/
truncateConversation(itemId: string, contentIndex: number, audioEnd?: number): void {
this.send({
type: 'conversation.item.truncate',
item_id: itemId,
content_index: contentIndex,
audio_end_ms: audioEnd,
});
}
/**
* Delete conversation item
*/
deleteItem(itemId: string): void {
this.send({
type: 'conversation.item.delete',
item_id: itemId,
});
}
/**
* Get conversation history
*/
getConversation(): ConversationItem[] {
return this.conversationItems;
}
/**
* Get MidStream agent
*/
getAgent(): MidStreamAgent {
return this.agent;
}
/**
* Get session ID
*/
getSessionId(): string | null {
return this.sessionId;
}
/**
* Check if connected
*/
isConnectedToOpenAI(): boolean {
return this.isConnected;
}
/**
* Get MidStream analysis
*/
getMidStreamAnalysis(): any {
const conversation = this.conversationItems
.filter(item => item.content)
.flatMap(item =>
item.content!
.filter(c => c.type === 'text' && c.text)
.map(c => c.text!)
);
return this.agent.analyzeConversation(conversation);
}
}
// ============================================================================
// Agentic Flow Proxy Client
// ============================================================================
export class AgenticFlowProxyClient {
private baseUrl: string;
private apiKey: string;
private realtimeClient: OpenAIRealtimeClient | null = null;
constructor(config: { baseUrl: string; apiKey: string; openAiApiKey: string }) {
this.baseUrl = config.baseUrl;
this.apiKey = config.apiKey;
}
/**
* Create a realtime session through agentic-flow proxy
*/
async createRealtimeSession(config: RealtimeConfig): Promise<OpenAIRealtimeClient> {
const client = new OpenAIRealtimeClient({
...config,
agenticFlowProxy: this.baseUrl,
agenticFlowApiKey: this.apiKey,
});
await client.connect();
this.realtimeClient = client;
return client;
}
/**
* Execute agentic workflow
*/
async executeWorkflow(workflowId: string, inputs: any): Promise<any> {
const response = await fetch(`${this.baseUrl}/workflows/${workflowId}/execute`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`,
},
body: JSON.stringify({ inputs }),
});
if (!response.ok) {
throw new Error(`Workflow execution failed: ${response.statusText}`);
}
return response.json();
}
/**
* Get current realtime client
*/
getRealtimeClient(): OpenAIRealtimeClient | null {
return this.realtimeClient;
}
}
// ============================================================================
// Helper Functions
// ============================================================================
/**
* Convert audio buffer to base64
*/
export function audioToBase64(buffer: Buffer): string {
return buffer.toString('base64');
}
/**
* Convert base64 to audio buffer
*/
export function base64ToAudio(base64: string): Buffer {
return Buffer.from(base64, 'base64');
}
/**
* Create default session config
*/
export function createDefaultSessionConfig(): SessionConfig {
return {
modalities: ['text', 'audio'],
instructions: 'You are a helpful AI assistant integrated with MidStream for real-time conversation analysis.',
voice: 'alloy',
input_audio_format: 'pcm16',
output_audio_format: 'pcm16',
input_audio_transcription: {
model: 'whisper-1',
},
turn_detection: {
type: 'server_vad',
threshold: 0.5,
prefix_padding_ms: 300,
silence_duration_ms: 200,
},
temperature: 0.8,
max_response_output_tokens: 4096,
};
}
+436
View File
@@ -0,0 +1,436 @@
/**
* MidStream QUIC Integration
*
* Node.js wrapper for QUIC transport using native bindings
* Provides low-latency, multiplexed streaming with HTTP/3
*
* Created by rUv
*/
import { EventEmitter } from 'events';
import * as dgram from 'dgram';
import { MidStreamAgent } from './agent.js';
// ============================================================================
// Types and Interfaces
// ============================================================================
export interface QuicConfig {
host?: string;
port?: number;
maxStreams?: number;
maxIdleTimeout?: number;
keepAliveInterval?: number;
cert?: string;
key?: string;
alpn?: string[];
}
export interface QuicStreamConfig {
priority?: number;
unidirectional?: boolean;
}
export interface QuicConnectionStats {
bytesReceived: number;
bytesSent: number;
packetsReceived: number;
packetsSent: number;
streamsOpened: number;
rtt: number;
congestionWindow: number;
}
// ============================================================================
// QuicStream Class
// ============================================================================
export class QuicStream extends EventEmitter {
private streamId: number;
private buffer: Buffer[] = [];
private closed: boolean = false;
private priority: number;
constructor(streamId: number, priority: number = 0) {
super();
this.streamId = streamId;
this.priority = priority;
}
/**
* Write data to the stream
*/
write(data: Buffer | string): boolean {
if (this.closed) {
throw new Error('Stream is closed');
}
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
this.buffer.push(buffer);
this.emit('data', buffer);
return true;
}
/**
* Close the stream
*/
close(): void {
if (!this.closed) {
this.closed = true;
this.emit('close');
}
}
/**
* Get stream ID
*/
getStreamId(): number {
return this.streamId;
}
/**
* Check if stream is closed
*/
isClosed(): boolean {
return this.closed;
}
/**
* Set stream priority
*/
setPriority(priority: number): void {
this.priority = priority;
}
/**
* Get stream priority
*/
getPriority(): number {
return this.priority;
}
}
// ============================================================================
// QuicConnection Class
// ============================================================================
export class QuicConnection extends EventEmitter {
private streams: Map<number, QuicStream> = new Map();
private nextStreamId: number = 0;
private connected: boolean = false;
private stats: QuicConnectionStats;
private config: QuicConfig;
private agent: MidStreamAgent;
constructor(config: QuicConfig = {}) {
super();
this.config = {
host: config.host || 'localhost',
port: config.port || 4433,
maxStreams: config.maxStreams || 1000,
maxIdleTimeout: config.maxIdleTimeout || 30000,
keepAliveInterval: config.keepAliveInterval || 5000,
alpn: config.alpn || ['h3', 'h3-29'],
...config
};
this.stats = {
bytesReceived: 0,
bytesSent: 0,
packetsReceived: 0,
packetsSent: 0,
streamsOpened: 0,
rtt: 0,
congestionWindow: 0
};
this.agent = new MidStreamAgent();
}
/**
* Connect to QUIC server
*/
async connect(): Promise<void> {
return new Promise((resolve, reject) => {
try {
// Simulate QUIC connection
// In production, this would use native QUIC bindings
setTimeout(() => {
this.connected = true;
this.emit('connected');
resolve();
}, 10);
} catch (error) {
reject(error);
}
});
}
/**
* Open a bidirectional stream
*/
async openBiStream(config?: QuicStreamConfig): Promise<QuicStream> {
if (!this.connected) {
throw new Error('Not connected');
}
if (this.streams.size >= this.config.maxStreams!) {
throw new Error('Max streams reached');
}
const streamId = this.nextStreamId++;
const priority = config?.priority || 0;
const stream = new QuicStream(streamId, priority);
this.streams.set(streamId, stream);
this.stats.streamsOpened++;
stream.on('data', (data) => {
this.stats.bytesSent += data.length;
// Process with MidStream agent
if (data.toString) {
this.agent.processMessage(data.toString());
}
});
stream.on('close', () => {
this.streams.delete(streamId);
});
this.emit('stream', stream);
return stream;
}
/**
* Open a unidirectional stream
*/
async openUniStream(config?: QuicStreamConfig): Promise<QuicStream> {
const stream = await this.openBiStream({ ...config, unidirectional: true });
return stream;
}
/**
* Close the connection
*/
close(): void {
this.streams.forEach(stream => stream.close());
this.streams.clear();
this.connected = false;
this.emit('close');
}
/**
* Get connection statistics
*/
getStats(): QuicConnectionStats {
return { ...this.stats };
}
/**
* Check if connected
*/
isConnected(): boolean {
return this.connected;
}
/**
* Get active streams count
*/
getStreamCount(): number {
return this.streams.size;
}
/**
* Get MidStream agent
*/
getAgent(): MidStreamAgent {
return this.agent;
}
}
// ============================================================================
// QuicServer Class
// ============================================================================
export class QuicServer extends EventEmitter {
private connections: Map<string, QuicConnection> = new Map();
private listening: boolean = false;
private config: QuicConfig;
private socket: dgram.Socket | null = null;
constructor(config: QuicConfig = {}) {
super();
this.config = {
host: config.host || '0.0.0.0',
port: config.port || 4433,
maxStreams: config.maxStreams || 1000,
alpn: config.alpn || ['h3', 'h3-29'],
...config
};
}
/**
* Start listening for connections
*/
async listen(): Promise<void> {
return new Promise((resolve, reject) => {
try {
// Simulate QUIC server
// In production, this would use native QUIC bindings
this.socket = dgram.createSocket('udp4');
this.socket.on('listening', () => {
this.listening = true;
this.emit('listening', this.config.port);
resolve();
});
this.socket.on('message', (msg, rinfo) => {
this.handleMessage(msg, rinfo);
});
this.socket.on('error', (error) => {
this.emit('error', error);
reject(error);
});
this.socket.bind(this.config.port, this.config.host);
} catch (error) {
reject(error);
}
});
}
/**
* Handle incoming message
*/
private handleMessage(msg: Buffer, rinfo: dgram.RemoteInfo): void {
const connectionId = `${rinfo.address}:${rinfo.port}`;
let connection = this.connections.get(connectionId);
if (!connection) {
connection = new QuicConnection(this.config);
this.connections.set(connectionId, connection);
this.emit('connection', connection);
}
// Emit data event
this.emit('data', msg, rinfo);
}
/**
* Close the server
*/
close(): void {
this.connections.forEach(conn => conn.close());
this.connections.clear();
if (this.socket) {
this.socket.close();
this.socket = null;
}
this.listening = false;
this.emit('close');
}
/**
* Check if listening
*/
isListening(): boolean {
return this.listening;
}
/**
* Get active connections count
*/
getConnectionCount(): number {
return this.connections.size;
}
}
// ============================================================================
// QuicClient Class (Helper)
// ============================================================================
export class QuicClient {
private connection: QuicConnection | null = null;
/**
* Connect to QUIC server
*/
async connect(host: string, port: number, config?: QuicConfig): Promise<QuicConnection> {
this.connection = new QuicConnection({
host,
port,
...config
});
await this.connection.connect();
return this.connection;
}
/**
* Disconnect
*/
disconnect(): void {
if (this.connection) {
this.connection.close();
this.connection = null;
}
}
/**
* Get connection
*/
getConnection(): QuicConnection | null {
return this.connection;
}
}
// ============================================================================
// Utility Functions
// ============================================================================
/**
* Create QUIC server with defaults
*/
export function createQuicServer(config?: QuicConfig): QuicServer {
return new QuicServer(config);
}
/**
* Connect to QUIC server
*/
export async function connectQuic(
host: string,
port: number,
config?: QuicConfig
): Promise<QuicConnection> {
const client = new QuicClient();
return client.connect(host, port, config);
}
/**
* Check if QUIC is supported
*/
export function isQuicSupported(): boolean {
// In production, check for native QUIC support
// For now, always return true (using simulation)
return true;
}
// ============================================================================
// Exports
// ============================================================================
export default {
QuicConnection,
QuicServer,
QuicClient,
QuicStream,
createQuicServer,
connectQuic,
isQuicSupported
};
+487
View File
@@ -0,0 +1,487 @@
/**
* MidStream Restream/WebRTC Integration
*
* Real-time video stream introspection and analysis
* Supports RTMP, WebRTC, and HLS streams
*
* Created by rUv
*/
import { EventEmitter } from 'events';
import { MidStreamAgent } from './agent.js';
import * as http from 'http';
import * as https from 'https';
// ============================================================================
// Types and Interfaces
// ============================================================================
export interface RestreamConfig {
rtmpUrl?: string;
streamKey?: string;
webrtcSignaling?: string;
apiKey?: string;
enableTranscription?: boolean;
enableObjectDetection?: boolean;
frameRate?: number;
resolution?: string;
}
export interface StreamFrame {
timestamp: number;
frameNumber: number;
data: Buffer;
width: number;
height: number;
format: string;
}
export interface AudioChunk {
timestamp: number;
data: Buffer;
sampleRate: number;
channels: number;
format: string;
}
export interface StreamAnalysis {
frameCount: number;
audioChunks: number;
avgFrameSize: number;
avgAudioSize: number;
bitrate: number;
fps: number;
detectedObjects: DetectedObject[];
transcription: string;
patterns: any[];
}
export interface DetectedObject {
label: string;
confidence: number;
boundingBox: {
x: number;
y: number;
width: number;
height: number;
};
}
// ============================================================================
// RestreamClient Class
// ============================================================================
export class RestreamClient extends EventEmitter {
private config: RestreamConfig;
private agent: MidStreamAgent;
private isStreaming: boolean = false;
private frameCount: number = 0;
private audioChunkCount: number = 0;
private startTime: number = 0;
private frameBuffer: StreamFrame[] = [];
private audioBuffer: AudioChunk[] = [];
private transcriptionBuffer: string[] = [];
constructor(config: RestreamConfig) {
super();
this.config = {
frameRate: 30,
resolution: '1920x1080',
enableTranscription: true,
enableObjectDetection: false,
...config,
};
this.agent = new MidStreamAgent();
}
// ==========================================================================
// Connection Management
// ==========================================================================
/**
* Connect to RTMP stream
*/
async connectRTMP(): Promise<void> {
if (!this.config.rtmpUrl || !this.config.streamKey) {
throw new Error('RTMP URL and stream key are required');
}
this.isStreaming = true;
this.startTime = Date.now();
this.emit('connected', {
type: 'rtmp',
url: this.config.rtmpUrl,
});
// In a real implementation, this would use a library like node-media-server
// or fluent-ffmpeg to connect to the RTMP stream
this.emit('info', 'RTMP connection established (mock)');
}
/**
* Connect to WebRTC stream
*/
async connectWebRTC(): Promise<void> {
if (!this.config.webrtcSignaling) {
throw new Error('WebRTC signaling server URL is required');
}
this.isStreaming = true;
this.startTime = Date.now();
this.emit('connected', {
type: 'webrtc',
signaling: this.config.webrtcSignaling,
});
// In a real implementation, this would use wrtc (node-webrtc)
// to establish WebRTC peer connection
this.emit('info', 'WebRTC connection established (mock)');
}
/**
* Connect to HLS stream
*/
async connectHLS(url: string): Promise<void> {
this.isStreaming = true;
this.startTime = Date.now();
this.emit('connected', {
type: 'hls',
url,
});
// Start polling HLS stream
await this.pollHLSStream(url);
}
/**
* Disconnect from stream
*/
disconnect(): void {
this.isStreaming = false;
this.emit('disconnected');
}
// ==========================================================================
// Stream Processing
// ==========================================================================
/**
* Process incoming video frame
*/
processFrame(frame: StreamFrame): void {
if (!this.isStreaming) return;
this.frameCount++;
this.frameBuffer.push(frame);
// Keep only recent frames
if (this.frameBuffer.length > 100) {
this.frameBuffer.shift();
}
// Emit frame event
this.emit('frame', frame);
// Analyze frame for patterns (simplified)
if (this.config.enableObjectDetection) {
this.analyzeFrame(frame);
}
}
/**
* Process incoming audio chunk
*/
processAudio(audio: AudioChunk): void {
if (!this.isStreaming) return;
this.audioChunkCount++;
this.audioBuffer.push(audio);
// Keep only recent audio
if (this.audioBuffer.length > 100) {
this.audioBuffer.shift();
}
// Emit audio event
this.emit('audio', audio);
// Transcribe audio if enabled
if (this.config.enableTranscription) {
this.transcribeAudio(audio);
}
}
/**
* Analyze video frame for objects/patterns
*/
private async analyzeFrame(frame: StreamFrame): Promise<void> {
// In a real implementation, this would use TensorFlow.js or similar
// to detect objects in the frame
// Mock detection
const mockObjects: DetectedObject[] = [
{
label: 'person',
confidence: 0.95,
boundingBox: { x: 100, y: 100, width: 200, height: 400 },
},
];
this.emit('objects_detected', {
frame: frame.frameNumber,
objects: mockObjects,
});
// Process with MidStream agent
this.agent.processMessage(
`Frame ${frame.frameNumber}: detected ${mockObjects.length} objects`
);
}
/**
* Transcribe audio chunk
*/
private async transcribeAudio(audio: AudioChunk): Promise<void> {
// In a real implementation, this would use OpenAI Whisper or similar
// to transcribe the audio
// Mock transcription
const mockTranscription = `Audio chunk ${this.audioChunkCount}`;
this.transcriptionBuffer.push(mockTranscription);
// Keep only recent transcriptions
if (this.transcriptionBuffer.length > 50) {
this.transcriptionBuffer.shift();
}
this.emit('transcription', mockTranscription);
// Process with MidStream agent
this.agent.processMessage(mockTranscription);
}
/**
* Poll HLS stream for segments
*/
private async pollHLSStream(url: string): Promise<void> {
const protocol = url.startsWith('https') ? https : http;
const fetchManifest = async () => {
if (!this.isStreaming) return;
try {
const response = await new Promise<string>((resolve, reject) => {
protocol
.get(url, (res) => {
let data = '';
res.on('data', (chunk) => (data += chunk));
res.on('end', () => resolve(data));
res.on('error', reject);
})
.on('error', reject);
});
this.emit('hls_manifest', response);
// Parse manifest and fetch segments
// In a real implementation, this would parse the M3U8 manifest
// and fetch video segments
// Schedule next poll
setTimeout(fetchManifest, 1000);
} catch (error) {
this.emit('error', error);
}
};
await fetchManifest();
}
// ==========================================================================
// Analysis and Metrics
// ==========================================================================
/**
* Get current stream analysis
*/
getAnalysis(): StreamAnalysis {
const duration = (Date.now() - this.startTime) / 1000;
const fps = duration > 0 ? this.frameCount / duration : 0;
const totalFrameSize = this.frameBuffer.reduce((sum, f) => sum + f.data.length, 0);
const avgFrameSize = this.frameBuffer.length > 0 ? totalFrameSize / this.frameBuffer.length : 0;
const totalAudioSize = this.audioBuffer.reduce((sum, a) => sum + a.data.length, 0);
const avgAudioSize = this.audioBuffer.length > 0 ? totalAudioSize / this.audioBuffer.length : 0;
const bitrate = duration > 0 ? ((totalFrameSize + totalAudioSize) * 8) / duration / 1000 : 0;
const agentStatus = this.agent.getStatus();
return {
frameCount: this.frameCount,
audioChunks: this.audioChunkCount,
avgFrameSize: Math.round(avgFrameSize),
avgAudioSize: Math.round(avgAudioSize),
bitrate: Math.round(bitrate),
fps: Math.round(fps * 10) / 10,
detectedObjects: [],
transcription: this.transcriptionBuffer.join(' '),
patterns: agentStatus.patterns,
};
}
/**
* Get stream statistics
*/
getStats() {
return {
isStreaming: this.isStreaming,
frameCount: this.frameCount,
audioChunks: this.audioChunkCount,
uptime: Date.now() - this.startTime,
bufferSize: {
frames: this.frameBuffer.length,
audio: this.audioBuffer.length,
},
};
}
/**
* Get MidStream agent
*/
getAgent(): MidStreamAgent {
return this.agent;
}
}
// ============================================================================
// WebRTC Signaling Server (for testing)
// ============================================================================
export class WebRTCSignalingServer extends EventEmitter {
private server: http.Server | null = null;
private peers: Map<string, any> = new Map();
/**
* Start signaling server
*/
start(port: number = 8080): Promise<void> {
return new Promise((resolve) => {
this.server = http.createServer((req, res) => {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ status: 'ok', peers: this.peers.size }));
});
this.server.listen(port, () => {
this.emit('listening', port);
resolve();
});
});
}
/**
* Stop signaling server
*/
stop(): void {
if (this.server) {
this.server.close();
this.server = null;
}
}
/**
* Register peer
*/
registerPeer(peerId: string, peerInfo: any): void {
this.peers.set(peerId, peerInfo);
this.emit('peer_registered', peerId);
}
/**
* Unregister peer
*/
unregisterPeer(peerId: string): void {
this.peers.delete(peerId);
this.emit('peer_unregistered', peerId);
}
}
// ============================================================================
// Stream Simulator (for testing)
// ============================================================================
export class StreamSimulator {
private frameRate: number;
private interval: NodeJS.Timeout | null = null;
private frameNumber: number = 0;
constructor(frameRate: number = 30) {
this.frameRate = frameRate;
}
/**
* Start simulating stream
*/
start(
onFrame: (frame: StreamFrame) => void,
onAudio?: (audio: AudioChunk) => void
): void {
const frameInterval = 1000 / this.frameRate;
this.interval = setInterval(() => {
this.frameNumber++;
// Generate mock frame
const frame: StreamFrame = {
timestamp: Date.now(),
frameNumber: this.frameNumber,
data: Buffer.alloc(1024 * 100), // 100KB frame
width: 1920,
height: 1080,
format: 'yuv420p',
};
onFrame(frame);
// Generate mock audio every 10 frames
if (onAudio && this.frameNumber % 10 === 0) {
const audio: AudioChunk = {
timestamp: Date.now(),
data: Buffer.alloc(1024 * 10), // 10KB audio
sampleRate: 48000,
channels: 2,
format: 'pcm_s16le',
};
onAudio(audio);
}
}, frameInterval);
}
/**
* Stop simulating stream
*/
stop(): void {
if (this.interval) {
clearInterval(this.interval);
this.interval = null;
}
}
/**
* Get current frame number
*/
getFrameNumber(): number {
return this.frameNumber;
}
}
// ============================================================================
// Exports
// ============================================================================
export default RestreamClient;
+349
View File
@@ -0,0 +1,349 @@
/**
* Streaming support for MidStream - WebSocket and SSE
*/
import { WebSocketServer, WebSocket } from 'ws';
import { createServer, IncomingMessage, ServerResponse } from 'http';
import { MidStreamAgent } from './agent.js';
// ============================================================================
// WebSocket Streaming Server
// ============================================================================
export class WebSocketStreamServer {
private wss: WebSocketServer | null = null;
private port: number;
private agent: MidStreamAgent;
private clients: Set<WebSocket> = new Set();
constructor(port: number = 3001) {
this.port = port;
this.agent = new MidStreamAgent();
}
async start(): Promise<void> {
this.wss = new WebSocketServer({ port: this.port });
this.wss.on('connection', (ws: WebSocket) => {
console.log('WebSocket client connected');
this.clients.add(ws);
ws.on('message', async (data: Buffer) => {
try {
const message = data.toString();
const parsed = JSON.parse(message);
const response = await this.handleMessage(parsed);
ws.send(JSON.stringify(response));
} catch (error) {
ws.send(JSON.stringify({
error: error instanceof Error ? error.message : String(error),
}));
}
});
ws.on('close', () => {
console.log('WebSocket client disconnected');
this.clients.delete(ws);
});
ws.on('error', (error) => {
console.error('WebSocket error:', error);
this.clients.delete(ws);
});
// Send welcome message
ws.send(JSON.stringify({
type: 'connected',
message: 'Connected to MidStream WebSocket server',
timestamp: Date.now(),
}));
});
console.log(`WebSocket server listening on port ${this.port}`);
}
async stop(): Promise<void> {
if (this.wss) {
this.clients.forEach(client => client.close());
this.wss.close();
this.wss = null;
}
}
broadcast(data: any): void {
const message = JSON.stringify(data);
this.clients.forEach(client => {
if (client.readyState === WebSocket.OPEN) {
client.send(message);
}
});
}
private async handleMessage(message: any): Promise<any> {
const { type, payload } = message;
switch (type) {
case 'process':
return {
type: 'result',
data: this.agent.processMessage(payload.message),
timestamp: Date.now(),
};
case 'analyze':
return {
type: 'analysis',
data: this.agent.analyzeConversation(payload.messages),
timestamp: Date.now(),
};
case 'compare':
return {
type: 'comparison',
data: {
similarity: this.agent.compareSequences(
payload.sequence1,
payload.sequence2,
payload.algorithm || 'dtw'
),
},
timestamp: Date.now(),
};
case 'detect_pattern':
return {
type: 'pattern',
data: {
positions: this.agent.detectPattern(payload.sequence, payload.pattern),
},
timestamp: Date.now(),
};
case 'behavior':
return {
type: 'behavior_analysis',
data: this.agent.analyzeBehavior(payload.rewards),
timestamp: Date.now(),
};
case 'status':
return {
type: 'status',
data: this.agent.getStatus(),
timestamp: Date.now(),
};
default:
return {
type: 'error',
error: `Unknown message type: ${type}`,
timestamp: Date.now(),
};
}
}
}
// ============================================================================
// SSE (Server-Sent Events) Streaming Server
// ============================================================================
export class SSEStreamServer {
private server: ReturnType<typeof createServer> | null = null;
private port: number;
private agent: MidStreamAgent;
private clients: Set<ServerResponse> = new Set();
constructor(port: number = 3002) {
this.port = port;
this.agent = new MidStreamAgent();
}
async start(): Promise<void> {
this.server = createServer((req: IncomingMessage, res: ServerResponse) => {
// CORS headers
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
if (req.method === 'OPTIONS') {
res.writeHead(200);
res.end();
return;
}
const url = new URL(req.url || '', `http://${req.headers.host}`);
if (url.pathname === '/stream' && req.method === 'GET') {
this.handleSSEConnection(req, res);
} else if (url.pathname === '/process' && req.method === 'POST') {
this.handleProcessRequest(req, res);
} else if (url.pathname === '/analyze' && req.method === 'POST') {
this.handleAnalyzeRequest(req, res);
} else if (url.pathname === '/status' && req.method === 'GET') {
this.handleStatusRequest(req, res);
} else {
res.writeHead(404);
res.end('Not Found');
}
});
this.server.listen(this.port);
console.log(`SSE server listening on port ${this.port}`);
}
async stop(): Promise<void> {
if (this.server) {
this.clients.forEach(client => client.end());
this.server.close();
this.server = null;
}
}
broadcast(data: any): void {
const message = `data: ${JSON.stringify(data)}\n\n`;
this.clients.forEach(client => {
try {
client.write(message);
} catch (error) {
console.error('Error broadcasting to SSE client:', error);
}
});
}
private handleSSEConnection(req: IncomingMessage, res: ServerResponse): void {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
});
this.clients.add(res);
console.log('SSE client connected');
// Send initial connection event
res.write(`data: ${JSON.stringify({
type: 'connected',
message: 'Connected to MidStream SSE server',
timestamp: Date.now(),
})}\n\n`);
// Send periodic heartbeat
const heartbeat = setInterval(() => {
res.write(`: heartbeat\n\n`);
}, 30000);
req.on('close', () => {
clearInterval(heartbeat);
this.clients.delete(res);
console.log('SSE client disconnected');
});
}
private handleProcessRequest(req: IncomingMessage, res: ServerResponse): void {
let body = '';
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', () => {
try {
const { message } = JSON.parse(body);
const result = this.agent.processMessage(message);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
data: result,
timestamp: Date.now(),
}));
// Broadcast to SSE clients
this.broadcast({
type: 'processed',
data: result,
timestamp: Date.now(),
});
} catch (error) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: false,
error: error instanceof Error ? error.message : String(error),
}));
}
});
}
private handleAnalyzeRequest(req: IncomingMessage, res: ServerResponse): void {
let body = '';
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', () => {
try {
const { messages } = JSON.parse(body);
const result = this.agent.analyzeConversation(messages);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
data: result,
timestamp: Date.now(),
}));
} catch (error) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: false,
error: error instanceof Error ? error.message : String(error),
}));
}
});
}
private handleStatusRequest(req: IncomingMessage, res: ServerResponse): void {
const status = this.agent.getStatus();
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
data: status,
timestamp: Date.now(),
}));
}
}
// ============================================================================
// HTTP Streaming Client (for use in Node.js)
// ============================================================================
export class HTTPStreamingClient {
private baseUrl: string;
constructor(baseUrl: string) {
this.baseUrl = baseUrl;
}
async stream(
endpoint: string,
onChunk: (chunk: Buffer) => void
): Promise<void> {
const https = await import('https');
const http = await import('http');
const url = new URL(endpoint, this.baseUrl);
const client = url.protocol === 'https:' ? https : http;
return new Promise((resolve, reject) => {
const req = client.get(url, (res) => {
res.on('data', onChunk);
res.on('end', resolve);
res.on('error', reject);
});
req.on('error', reject);
});
}
}