When GPT-4 Isn't Enough: Multi-Model Orchestration Patterns

interface RoutingDecision {
  model: string;
  confidence: number;
  reasoning: string;
}

class ModelRouter {
  private classifier: FastClassifier;
  private modelRegistry: Map<string, ModelConfig>;
  
  constructor() {
    this.classifier = new FastClassifier({
      model: 'distilbert-routing',
      labels: ['reasoning', 'speed', 'code', 'math', 'embedding']
    });
    
    this.modelRegistry = new Map([
      ['reasoning', { model: 'gpt-4-turbo', maxTokens: 4096 }],
      ['speed', { model: 'gpt-3.5-turbo', maxTokens: 2048 }],
      ['code', { model: 'codellama-34b', maxTokens: 4096 }],
      ['math', { model: 'gpt-4-turbo', maxTokens: 1024, systemPrompt: mathPrompt }],
      ['embedding', { model: 'text-embedding-3-large' }]
    ]);
  }
  
  async route(request: AIRequest): Promise<RoutingDecision> {
    // Fast classification (< 10ms)
    const classification = await this.classifier.classify(request.text);
    
    // Apply business rules
    const adjusted = this.applyRules(request, classification);
    
    return {
      model: this.modelRegistry.get(adjusted.label)!.model,
      confidence: adjusted.confidence,
      reasoning: this.explainRouting(request, adjusted)
    };
  }
  
  private applyRules(
    request: AIRequest, 
    classification: Classification
  ): Classification {
    // Rule 1: Long context always goes to reasoning models
    if (request.text.length > 10000) {
      return { label: 'reasoning', confidence: 0.95 };
    }
    
    // Rule 2: Code blocks suggest code task
    if (request.text.includes('function') || request.text.includes('class ')) {
      if (classification.label !== 'code') {
        return { label: 'code', confidence: 0.85 };
      }
    }
    
    // Rule 3: Numbers and calculations suggest math
    const numberDensity = (request.text.match(/\d+/g) || []).length / 
                          request.text.split(' ').length;
    if (numberDensity > 0.3) {
      return { label: 'math', confidence: 0.8 };
    }
    
    // Rule 4: Real-time requests need speed
    if (request.priority === 'realtime' && classification.label === 'reasoning') {
      return { label: 'speed', confidence: 0.9 };
    }
    
    return classification;
  }
}

interface PipelineStage {
  name: string;
  model: string;
  transform: (input: string, context: PipelineContext) => Promise<string>;
}

class ModelPipeline {
  private stages: PipelineStage[];
  
  constructor(stages: PipelineStage[]) {
    this.stages = stages;
  }
  
  async execute(input: string): Promise<PipelineResult> {
    const context: PipelineContext = {
      originalInput: input,
      intermediateResults: [],
      metadata: {}
    };
    
    let current = input;
    
    for (const stage of this.stages) {
      const startTime = Date.now();
      
      try {
        current = await stage.transform(current, context);
        context.intermediateResults.push({
          stage: stage.name,
          output: current,
          duration: Date.now() - startTime
        });
      } catch (error) {
        return {
          success: false,
          failedStage: stage.name,
          error,
          partialResults: context.intermediateResults
        };
      }
    }
    
    return {
      success: true,
      finalOutput: current,
      stages: context.intermediateResults
    };
  }
}

const documentAnalysisPipeline = new ModelPipeline([
  {
    name: 'extract',
    model: 'gpt-3.5-turbo',
    transform: async (doc) => {
      // Fast extraction of key sections
      return await llm.complete({
        model: 'gpt-3.5-turbo',
        prompt: 'Extract the key sections from this document: ' + doc,
        maxTokens: 2000
      });
    }
  },
  {
    name: 'analyze',
    model: 'gpt-4-turbo',
    transform: async (sections, context) => {
      // Deep analysis with reasoning model
      return await llm.complete({
        model: 'gpt-4-turbo',
        prompt: 'Analyze these document sections for key insights: ' + sections,
        maxTokens: 4000
      });
    }
  },
  {
    name: 'summarize',
    model: 'claude-3-sonnet',
    transform: async (analysis) => {
      // Clear summary with Claude
      return await llm.complete({
        model: 'claude-3-sonnet',
        prompt: 'Create an executive summary: ' + analysis,
        maxTokens: 500
      });
    }
  }
]);

interface EnsembleConfig {
  models: ModelWeight[];
  aggregation: 'majority' | 'weighted' | 'unanimous';
  minAgreement: number;
}

interface ModelWeight {
  model: string;
  weight: number;
}

class EnsembleOrchestrator {
  constructor(private config: EnsembleConfig) {}
  
  async decide(prompt: string): Promise<EnsembleResult> {
    // Run all models in parallel
    const responses = await Promise.all(
      this.config.models.map(async (m) => ({
        model: m.model,
        weight: m.weight,
        response: await this.callModel(m.model, prompt)
      }))
    );
    
    // Extract decisions
    const decisions = responses.map(r => ({
      ...r,
      decision: this.extractDecision(r.response)
    }));
    
    // Aggregate based on strategy
    const result = this.aggregate(decisions);
    
    return {
      decision: result.decision,
      confidence: result.confidence,
      agreement: result.agreement,
      individualResponses: decisions
    };
  }
  
  private aggregate(decisions: Decision[]): AggregatedResult {
    if (this.config.aggregation === 'majority') {
      return this.majorityVote(decisions);
    }
    
    if (this.config.aggregation === 'weighted') {
      return this.weightedVote(decisions);
    }
    
    if (this.config.aggregation === 'unanimous') {
      return this.unanimousVote(decisions);
    }
    
    throw new Error('Unknown aggregation: ' + this.config.aggregation);
  }
  
  private weightedVote(decisions: Decision[]): AggregatedResult {
    const scores: Map<string, number> = new Map();
    let totalWeight = 0;
    
    for (const d of decisions) {
      const current = scores.get(d.decision) || 0;
      scores.set(d.decision, current + d.weight);
      totalWeight += d.weight;
    }
    
    let maxDecision = '';
    let maxScore = 0;
    
    scores.forEach((score, decision) => {
      if (score > maxScore) {
        maxScore = score;
        maxDecision = decision;
      }
    });
    
    return {
      decision: maxDecision,
      confidence: maxScore / totalWeight,
      agreement: decisions.filter(d => d.decision === maxDecision).length / decisions.length
    };
  }
}

const moderationEnsemble = new EnsembleOrchestrator({
  models: [
    { model: 'gpt-4-turbo', weight: 0.4 },
    { model: 'claude-3-opus', weight: 0.35 },
    { model: 'moderation-fine-tuned', weight: 0.25 }
  ],
  aggregation: 'weighted',
  minAgreement: 0.6
});

// Usage
const result = await moderationEnsemble.decide(
  'Evaluate if this content violates our guidelines: ' + userContent
);

if (result.decision === 'violates' && result.confidence > 0.8) {
  await flagContent(userContent);
} else if (result.decision === 'violates' && result.confidence > 0.5) {
  await queueForHumanReview(userContent);
}

class SpeculativeExecutor {
  constructor(
    private fastModel: string,
    private verifyModel: string,
    private verifyThreshold: number
  ) {}
  
  async execute(prompt: string): Promise<SpeculativeResult> {
    // Start fast model
    const fastStart = Date.now();
    const fastResponse = await llm.complete({
      model: this.fastModel,
      prompt,
      maxTokens: 1000
    });
    const fastDuration = Date.now() - fastStart;
    
    // Quick confidence check
    const needsVerification = await this.shouldVerify(prompt, fastResponse);
    
    if (!needsVerification) {
      return {
        response: fastResponse,
        verified: false,
        model: this.fastModel,
        duration: fastDuration
      };
    }
    
    // Verify with slower model
    const verifyStart = Date.now();
    const verifyResponse = await llm.complete({
      model: this.verifyModel,
      prompt: this.buildVerifyPrompt(prompt, fastResponse),
      maxTokens: 1500
    });
    
    const agreement = await this.checkAgreement(fastResponse, verifyResponse);
    
    if (agreement > this.verifyThreshold) {
      return {
        response: fastResponse,
        verified: true,
        model: this.fastModel,
        duration: Date.now() - fastStart
      };
    }
    
    // Models disagree - use verified response
    return {
      response: verifyResponse,
      verified: true,
      model: this.verifyModel,
      duration: Date.now() - fastStart,
      disagreement: true
    };
  }
  
  private async shouldVerify(prompt: string, response: string): Promise<boolean> {
    // Verify if: high stakes, low confidence indicators, complex topic
    const indicators = [
      prompt.toLowerCase().includes('important'),
      prompt.toLowerCase().includes('critical'),
      response.includes('I think') || response.includes('probably'),
      prompt.length > 2000
    ];
    
    return indicators.filter(Boolean).length >= 2;
  }
}

interface FusionConfig {
  models: string[];
  fusionStrategy: 'merge' | 'select' | 'synthesize';
  fusionModel: string;
}

class ModelFusion {
  constructor(private config: FusionConfig) {}
  
  async fuse(prompt: string): Promise<FusionResult> {
    // Get responses from all models
    const responses = await Promise.all(
      this.config.models.map(model => 
        llm.complete({ model, prompt, maxTokens: 2000 })
      )
    );
    
    if (this.config.fusionStrategy === 'synthesize') {
      // Use a model to synthesize the best response
      const synthesisPrompt = this.buildSynthesisPrompt(prompt, responses);
      
      const synthesized = await llm.complete({
        model: this.config.fusionModel,
        prompt: synthesisPrompt,
        maxTokens: 2500
      });
      
      return {
        response: synthesized,
        sources: responses,
        strategy: 'synthesize'
      };
    }
    
    // ... other strategies
  }
  
  private buildSynthesisPrompt(prompt: string, responses: string[]): string {
    return 'Original question: ' + prompt + 
           '\n\nResponse A: ' + responses[0] +
           '\n\nResponse B: ' + responses[1] +
           '\n\nSynthesize the best possible response combining insights from both.';
  }
}

class AIOrchestrator {
  private router: ModelRouter;
  private pipelines: Map<string, ModelPipeline>;
  private ensembles: Map<string, EnsembleOrchestrator>;
  private speculative: SpeculativeExecutor;
  
  async process(request: AIRequest): Promise<AIResponse> {
    // 1. Determine the orchestration strategy
    const strategy = this.selectStrategy(request);
    
    switch (strategy) {
      case 'simple':
        // Direct routing for simple requests
        const routing = await this.router.route(request);
        return this.directCall(routing.model, request);
        
      case 'pipeline':
        // Multi-stage pipeline for complex analysis
        const pipelineId = this.selectPipeline(request);
        return this.pipelines.get(pipelineId)!.execute(request.text);
        
      case 'ensemble':
        // Voting for high-stakes decisions
        const ensembleId = this.selectEnsemble(request);
        return this.ensembles.get(ensembleId)!.decide(request.text);
        
      case 'speculative':
        // Fast with verification for medium-stakes
        return this.speculative.execute(request.text);
        
      default:
        throw new Error('Unknown strategy: ' + strategy);
    }
  }
  
  private selectStrategy(request: AIRequest): OrchestrationType {
    if (request.flags?.highStakes) return 'ensemble';
    if (request.flags?.complex) return 'pipeline';
    if (request.flags?.realtime) return 'speculative';
    return 'simple';
  }
}