How We Reduced LLM Latency from 8s to 800ms Without Losing Quality

// Before: Wait for complete response
const response = await openai.chat.completions.create({
  model: "gpt-4",
  messages: [...],
});
return response.choices[0].message.content;

// After: Stream tokens as they arrive
const stream = await openai.chat.completions.create({
  model: "gpt-4",
  messages: [...],
  stream: true,
});

for await (const chunk of stream) {
  const content = chunk.choices[0]?.delta?.content;
  if (content) {
    yield content; // Send to client immediately
  }
}

❌ BEFORE (2,400 tokens):
"You are an expert writing assistant with decades of experience 
in professional communication. You understand the nuances of 
business writing, academic prose, creative fiction, and technical 
documentation. When helping users, you should consider their 
audience, tone, purpose, and the specific conventions of their 
genre. You have expertise in grammar, style, rhetoric, and 
persuasion techniques. You can help with..."
[... 2,000 more tokens of context ...]

✓ AFTER (400 tokens):
"Writing assistant. Match user's tone. Be concise unless asked 
for detail. Format: brief suggestion, then explanation if needed."

interface ModelRouter {
  classify(request: UserRequest): ModelTier;
  route(request: UserRequest): Promise<LLMResponse>;
}

type ModelTier = 'fast' | 'balanced' | 'quality';

const modelConfig: Record<ModelTier, ModelConfig> = {
  fast: { 
    model: 'gpt-3.5-turbo', 
    maxTokens: 500,
    temperature: 0.3 
  },
  balanced: { 
    model: 'gpt-4-turbo', 
    maxTokens: 1000,
    temperature: 0.5 
  },
  quality: { 
    model: 'gpt-4', 
    maxTokens: 2000,
    temperature: 0.7 
  }
};

async function routeRequest(request: UserRequest): Promise<LLMResponse> {
  // Simple heuristic-based routing
  const tier = classifyRequest(request);
  
  // Start with fast model
  const response = await callModel(modelConfig[tier], request);
  
  // Quality gate: if confidence is low, escalate
  if (tier === 'fast' && response.confidence < 0.8) {
    return callModel(modelConfig['balanced'], request);
  }
  
  return response;
}

function classifyRequest(request: UserRequest): ModelTier {
  const text = request.text.toLowerCase();
  const wordCount = text.split(/\\s+/).length;
  
  // Grammar/spelling: fast model
  if (request.task === 'grammar' || request.task === 'spelling') {
    return 'fast';
  }
  
  // Short, simple rephrasing: fast model
  if (request.task === 'rephrase' && wordCount < 50) {
    return 'fast';
  }
  
  // Creative or complex: quality model
  if (request.task === 'creative' || request.task === 'essay') {
    return 'quality';
  }
  
  // Default: balanced
  return 'balanced';
}

interface CacheEntry {
  requestEmbedding: number[];
  taskType: string;
  response: string;
  quality_score: number;
  created_at: Date;
}

class SemanticCache {
  private vectorStore: VectorStore;
  private similarityThreshold = 0.92;
  
  async get(request: CacheableRequest): Promise<CacheEntry | null> {
    const embedding = await this.embed(request);
    
    const results = await this.vectorStore.search({
      vector: embedding,
      filter: { taskType: request.taskType },
      topK: 1,
      threshold: this.similarityThreshold
    });
    
    if (results.length === 0) return null;
    
    const entry = results[0];
    
    // Additional validation: check if cached response 
    // is still appropriate for this specific request
    if (!this.isResponseApplicable(request, entry)) {
      return null;
    }
    
    return entry;
  }
  
  private isResponseApplicable(
    request: CacheableRequest, 
    entry: CacheEntry
  ): boolean {
    // Don't use cache if request has specific constraints
    // that the cached response might not satisfy
    if (request.constraints?.maxLength) {
      const cachedLength = entry.response.split(/\\s+/).length;
      if (cachedLength > request.constraints.maxLength * 1.2) {
        return false;
      }
    }
    
    // Don't use stale cache for time-sensitive content
    const ageHours = (Date.now() - entry.created_at.getTime()) / 3600000;
    if (request.isTimeSensitive && ageHours > 24) {
      return false;
    }
    
    return true;
  }
}

const commonPrefixes = [
  { pattern: /^make.*more\s*$/i, likelyCompletions: ['professional', 'concise', 'formal'] },
  { pattern: /^rewrite.*as\s*$/i, likelyCompletions: ['bullet points', 'paragraph', 'email'] },
  { pattern: /^fix.*grammar/i, likelyCompletions: [] }, // No speculation needed
];

class SpeculativeExecutor {
  private pendingSpeculations: Map<string, AbortController> = new Map();
  
  async onPartialInput(sessionId: string, partialText: string, context: string) {
    // Cancel any existing speculation for this session
    this.pendingSpeculations.get(sessionId)?.abort();
    
    // Check if input matches a speculatable pattern
    for (const { pattern, likelyCompletions } of commonPrefixes) {
      if (pattern.test(partialText) && likelyCompletions.length > 0) {
        const controller = new AbortController();
        this.pendingSpeculations.set(sessionId, controller);
        
        // Start speculative generation for most likely completion
        const speculativePrompt = partialText + likelyCompletions[0];
        this.speculativeGenerate(sessionId, speculativePrompt, context, controller.signal);
        break;
      }
    }
  }
  
  private async speculativeGenerate(
    sessionId: string, 
    prompt: string, 
    context: string,
    signal: AbortSignal
  ) {
    const cacheKey = 'speculation:' + sessionId;
    
    try {
      const stream = await openai.chat.completions.create({
        model: 'gpt-3.5-turbo', // Use fast model for speculation
        messages: [
          { role: 'system', content: systemPrompt },
          { role: 'user', content: context },
          { role: 'user', content: prompt }
        ],
        stream: true,
      });
      
      let buffer = '';
      for await (const chunk of stream) {
        if (signal.aborted) break;
        buffer += chunk.choices[0]?.delta?.content || '';
        await this.cache.set(cacheKey, buffer, { ttl: 10 }); // 10s TTL
      }
    } catch (e) {
      if (e.name !== 'AbortError') throw e;
    }
  }
  
  async onFinalInput(sessionId: string, finalPrompt: string): Promise<AsyncIterable<string>> {
    const speculatedResult = await this.cache.get('speculation:' + sessionId);
    
    if (speculatedResult && this.isSpeculationUsable(speculatedResult, finalPrompt)) {
      // Speculation was correct! Stream the cached result
      return this.streamFromCache(speculatedResult);
    }
    
    // Speculation missed, generate normally
    return this.generateFresh(finalPrompt);
  }
}

// Edge function (Vercel/Cloudflare)
export const config = { runtime: 'edge' };

// Connection pool - reuse connections across requests
const connectionPool = new Map<string, OpenAI>();

function getClient(region: string): OpenAI {
  if (!connectionPool.has(region)) {
    connectionPool.set(region, new OpenAI({
      apiKey: process.env.OPENAI_API_KEY,
      maxRetries: 2,
      timeout: 30000,
    }));
  }
  return connectionPool.get(region)!;
}

export default async function handler(req: Request) {
  const region = req.headers.get('x-vercel-ip-country') || 'US';
  const client = getClient(region);
  
  // Stream directly from edge
  const stream = await client.chat.completions.create({
    model: 'gpt-4-turbo',
    messages: await req.json(),
    stream: true,
  });
  
  return new Response(
    new ReadableStream({
      async start(controller) {
        for await (const chunk of stream) {
          const text = chunk.choices[0]?.delta?.content || '';
          controller.enqueue(new TextEncoder().encode(text));
        }
        controller.close();
      },
    }),
    { headers: { 'Content-Type': 'text/event-stream' } }
  );
}