LLM Proxy WebSocket Streaming

// Establish WebSocket connection to LLM proxy
const ws = new WebSocket('wss://proxy.example.com/stream');

ws.onopen = () => {
    // Send streaming request
    ws.send(JSON.stringify({
        model: 'gpt-4',
        messages: [{ role: 'user', content: 'Explain quantum computing' }],
        stream: true
    }));
};

ws.onmessage = (event) => {
    const data = JSON.parse(event.data);
    // Render each token as it arrives
    appendToResponse(data.token);
};

ws.onerror = (error) => {
    handleConnectionError(error);
};

import asyncio
import websockets
from proxy import LLMStreamingProxy

async def handle_stream(websocket, path):
    proxy = LLMStreamingProxy()
    
    async for message in websocket:
        request = json.loads(message)
        
        # Stream tokens from LLM provider
        async for token in proxy.stream_completion(request):
            await websocket.send(json.dumps({
                'token': token.text,
                'latency': token.latency_ms
            }))

# Start WebSocket server
start_server = websockets.serve(handle_stream, "0.0.0.0", 8765)