AI Observability: Seeing Inside the Black Box

interface LLMRequestLog {
  request_id: string;
  timestamp: string;
  user_id: string;
  session_id: string;
  model: string;
  prompt_hash: string;
  tokens_in: number;
  tokens_out: number;
  latency_ms: number;
  ttft_ms: number;
  status: 'success' | 'error' | 'timeout' | 'rate_limited';
  cost_usd: number;
  error?: {
    code: string;
    message: string;
    retryable: boolean;
  };
  metadata: Record<string, any>;
}

class AILogger {
  private sensitivePatterns = [
    /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, // Email
    /\b\d{3}-\d{2}-\d{4}\b/g, // SSN
    /\b\d{16}\b/g, // Credit card
  ];
  
  async logRequest(log: LLMRequestLog, prompt?: string, response?: string) {
    // Redact sensitive data
    const safePrompt = prompt ? this.redact(prompt) : undefined;
    const safeResponse = response ? this.redact(response) : undefined;
    
    // Hash prompt for versioning without storing full text
    const promptHash = await this.hashContent(prompt || '');
    
    const enrichedLog: LLMRequestLog = {
      ...log,
      prompt_hash: promptHash,
    };
    
    // Log to appropriate destinations
    await Promise.all([
      this.logToOperational(enrichedLog),
      log.status === 'error' && this.logToErrors(enrichedLog, safePrompt, safeResponse),
      this.shouldAudit(log) && this.logToAudit(enrichedLog, safePrompt, safeResponse),
    ]);
    
    // Update metrics
    this.metrics.recordRequest(enrichedLog);
  }
  
  private redact(text: string): string {
    let redacted = text;
    for (const pattern of this.sensitivePatterns) {
      redacted = redacted.replace(pattern, '[REDACTED]');
    }
    return redacted;
  }
  
  private shouldAudit(log: LLMRequestLog): boolean {
    // Audit high-risk operations
    return log.metadata?.is_decision === true || 
           log.metadata?.affects_user_data === true;
  }
}

import { trace, context, SpanStatusCode } from '@opentelemetry/api';

const tracer = trace.getTracer('ai-service');

async function handleAIRequest(request: AIRequest) {
  return tracer.startActiveSpan('ai.request', async (span) => {
    try {
      span.setAttributes({
        'user.id': request.userId,
        'session.id': request.sessionId,
        'input.length': request.input.length,
      });
      
      // Preprocessing span
      const processed = await tracer.startActiveSpan('ai.preprocess', async (prepSpan) => {
        const result = await preprocess(request.input);
        prepSpan.setAttributes({ 'tokens.count': result.tokenCount });
        return result;
      });
      
      // Retrieval span
      const docs = await tracer.startActiveSpan('ai.retrieval', async (retSpan) => {
        const results = await vectorSearch(processed.embedding);
        retSpan.setAttributes({
          'retrieval.query': processed.query,
          'retrieval.results_count': results.length,
          'retrieval.top_score': results[0]?.score || 0,
        });
        return results;
      });
      
      // LLM call span - the most important one
      const response = await tracer.startActiveSpan('ai.llm_call', async (llmSpan) => {
        const startTime = Date.now();
        
        const result = await callLLM({
          model: request.model,
          prompt: buildPrompt(processed, docs),
        });
        
        const latency = Date.now() - startTime;
        
        llmSpan.setAttributes({
          'llm.model': request.model,
          'llm.tokens_in': result.usage.promptTokens,
          'llm.tokens_out': result.usage.completionTokens,
          'llm.ttft_ms': result.timeToFirstToken,
          'llm.latency_ms': latency,
          'llm.cost_usd': calculateCost(request.model, result.usage),
          'llm.finish_reason': result.finishReason,
        });
        
        return result;
      });
      
      span.setStatus({ code: SpanStatusCode.OK });
      return response;
      
    } catch (error) {
      span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}