The Hidden Cost of AI: Our $47K/Month Bill and How We Cut It by 70%

interface ModelConfig {
  model: string;
  costPer1kInput: number;
  costPer1kOutput: number;
  qualityScore: number;
}

const models: Record<string, ModelConfig> = {
  'gpt-4-turbo': { 
    model: 'gpt-4-turbo', 
    costPer1kInput: 0.01, 
    costPer1kOutput: 0.03,
    qualityScore: 1.0
  },
  'gpt-4o-mini': { 
    model: 'gpt-4o-mini', 
    costPer1kInput: 0.00015, 
    costPer1kOutput: 0.0006,
    qualityScore: 0.85
  },
  'gpt-3.5-turbo': { 
    model: 'gpt-3.5-turbo', 
    costPer1kInput: 0.0005, 
    costPer1kOutput: 0.0015,
    qualityScore: 0.7
  }
};

function selectModel(task: TaskType, complexity: number): string {
  // Classification with low complexity -> cheapest model
  if (task === 'classification' && complexity < 0.5) {
    return 'gpt-3.5-turbo';
  }
  
  // Entity extraction -> mid-tier model
  if (task === 'entity_extraction') {
    return 'gpt-4o-mini';
  }
  
  // Q&A with simple questions -> mid-tier
  if (task === 'qa' && complexity < 0.6) {
    return 'gpt-4o-mini';
  }
  
  // Complex analysis, summarization -> GPT-4
  return 'gpt-4-turbo';
}

function calculateComplexity(text: string, task: TaskType): number {
  let score = 0;
  
  // Length factor
const wordCount = text.split(/\\s+/).length;
  
  // Technical content detection
  const technicalTerms = text.match(/\\b(algorithm|implementation|architecture|infrastructure)\\b/gi);
  score += Math.min((technicalTerms?.length || 0) * 0.05, 0.2);
  
  // Question complexity (for Q&A)
  if (task === 'qa') {
    const complexIndicators = ['why', 'how', 'explain', 'compare', 'analyze'];
    const hasComplex = complexIndicators.some(ind => text.toLowerCase().includes(ind));
    score += hasComplex ? 0.2 : 0;
  }
  
  // Nested structure (for documents)
  const nestingDepth = (text.match(/^\s+/gm) || []).length;
  score += Math.min(nestingDepth * 0.01, 0.15);
  
  return Math.min(score, 1.0);
}

You are an expert document analyst with years of experience 
in analyzing business documents, technical specifications, 
legal contracts, and various other document types. Your task 
is to provide a comprehensive summary that captures the key 
points, main arguments, and critical details of the document 
provided below. Please ensure your summary is accurate, 
well-structured, and maintains the original meaning...

[... 200 more tokens of instructions ...]

Document to analyze:
[8000 tokens of document]

Summarize this document. Include: key points, main arguments, 
critical details. Be accurate and structured.

Document:
[document content]

function buildPrompt(task: TaskType, document: string): string {
  // Cached, minimal system prompts
  const systemPrompts: Record<TaskType, string> = {
    summarize: 'Summarize: key points, arguments, details.',
    extract: 'Extract entities as JSON: {people: [], orgs: [], dates: []}',
    qa: 'Answer based only on the provided context.',
    classify: 'Classify into exactly one category. Reply with category name only.'
  };
  
  // Remove redundant whitespace from documents
  const compressed = document
    .replace(/\\n\\s*\\n/g, '\\n')
    .replace(/\\s+/g, ' ')
    .trim();
  
  return systemPrompts[task] + '\\n\\n' + compressed;
}

async function summarizeLongDocument(doc: string): Promise<string> {
  const maxChunkTokens = 4000;
  const chunks = splitIntoChunks(doc, maxChunkTokens);
  
  if (chunks.length === 1) {
    return summarize(chunks[0]);
  }
  
  // Hierarchical summarization
  const chunkSummaries = await Promise.all(
    chunks.map(chunk => summarize(chunk, { maxTokens: 200 }))
  );
  
  // Final summary of summaries
  const combined = chunkSummaries.join('\\n\\n');
  return summarize(combined, { 
    prompt: 'Synthesize these section summaries into a cohesive document summary:' 
  });
}

const exactCache = new Map<string, CachedResponse>();

function getCacheKey(prompt: string, model: string): string {
  return crypto.createHash('sha256')
    .update(prompt + model)
    .digest('hex');
}

async function cachedCompletion(
  prompt: string, 
  options: CompletionOptions
): Promise<string> {
  const key = getCacheKey(prompt, options.model);
  
  if (exactCache.has(key)) {
    metrics.cacheHits.inc();
    return exactCache.get(key)!.response;
  }
  
  const response = await llm.complete(prompt, options);
  exactCache.set(key, { 
    response, 
    timestamp: Date.now(),
    cost: calculateCost(prompt, response, options.model)
  });
  
  return response;
}

class SemanticCache {
  private vectorStore: VectorStore;
  private threshold = 0.92;
  
  async get(query: string, context: string): Promise<string | null> {
    const embedding = await embed(query + context.slice(0, 500));
    
    const results = await this.vectorStore.search({
      vector: embedding,
      topK: 1,
      threshold: this.threshold
    });
    
    if (results.length > 0) {
      metrics.semanticCacheHits.inc();
      return results[0].metadata.response;
    }
    
    return null;
  }
  
  async set(query: string, context: string, response: string): Promise<void> {
    const embedding = await embed(query + context.slice(0, 500));
    
    await this.vectorStore.upsert({
      id: generateId(),
      vector: embedding,
      metadata: { query, response, timestamp: Date.now() }
    });
  }
}

// Before: Individual embedding calls
for (const chunk of chunks) {
  const embedding = await embed(chunk); // API call per chunk
  await store(embedding);
}

// After: Batched embedding calls
const batchSize = 100;
for (let i = 0; i < chunks.length; i += batchSize) {
  const batch = chunks.slice(i, i + batchSize);
  const embeddings = await embedBatch(batch); // Single API call
  await storeBatch(embeddings);
}

class EmbeddingStore {
  private cache: Map<string, number[]> = new Map();
  
  async getOrCreate(text: string): Promise<number[]> {
    const hash = this.hashText(text);
    
    if (this.cache.has(hash)) {
      return this.cache.get(hash)!;
    }
    
    // Check persistent store
    const stored = await this.db.getEmbedding(hash);
    if (stored) {
      this.cache.set(hash, stored);
      return stored;
    }
    
    // Generate new embedding
    const embedding = await embed(text);
    await this.db.storeEmbedding(hash, embedding);
    this.cache.set(hash, embedding);
    
    return embedding;
  }
  
  private hashText(text: string): string {
    return crypto.createHash('md5').update(text).digest('hex');
  }
}

// Before: New client per request
export async function handler(event) {
  const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
  return await client.chat.completions.create(...);
}

// After: Reused client
const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

export async function handler(event) {
  return await client.chat.completions.create(...);
}