Chunking Strategies That Actually Work: A 50-Document Benchmark

function fixedSizeChunk(text: string, chunkSize: number = 512): string[] {
  const tokens = tokenize(text);
  const chunks: string[] = [];
  
  for (let i = 0; i < tokens.length; i += chunkSize) {
    chunks.push(detokenize(tokens.slice(i, i + chunkSize)));
  }
  
  return chunks;
}

function fixedSizeOverlapChunk(
  text: string, 
  chunkSize: number = 512, 
  overlap: number = 50
): string[] {
  const tokens = tokenize(text);
  const chunks: string[] = [];
  const step = chunkSize - overlap;
  
  for (let i = 0; i < tokens.length; i += step) {
    chunks.push(detokenize(tokens.slice(i, i + chunkSize)));
    if (i + chunkSize >= tokens.length) break;
  }
  
  return chunks;
}

function sentenceChunk(text: string, sentencesPerChunk: number = 6): string[] {
  const sentences = splitIntoSentences(text);
  const chunks: string[] = [];
  
  for (let i = 0; i < sentences.length; i += sentencesPerChunk) {
    chunks.push(sentences.slice(i, i + sentencesPerChunk).join(' '));
  }
  
  return chunks;
}

function splitIntoSentences(text: string): string[] {
  // Handle edge cases: abbreviations, decimals, etc.
  return text
    .replace(/([.?!])\s+(?=[A-Z])/g, '$1|SPLIT|')
    .replace(/([.?!])\s*$/g, '$1|SPLIT|')
    .split('|SPLIT|')
    .map(s => s.trim())
    .filter(s => s.length > 0);
}

async function semanticChunk(
  text: string, 
  similarityThreshold: number = 0.75
): Promise<string[]> {
  const sentences = splitIntoSentences(text);
  const embeddings = await embedBatch(sentences);
  
  const chunks: string[] = [];
  let currentChunk: string[] = [sentences[0]];
  
  for (let i = 1; i < sentences.length; i++) {
    const similarity = cosineSimilarity(
      embeddings[i], 
      embeddings[i - 1]
    );
    
    if (similarity < similarityThreshold) {
      // Semantic shift detected - start new chunk
      chunks.push(currentChunk.join(' '));
      currentChunk = [sentences[i]];
    } else {
      currentChunk.push(sentences[i]);
    }
    
    // Also split if chunk is getting too long
    if (currentChunk.length > 10) {
      chunks.push(currentChunk.join(' '));
      currentChunk = [];
    }
  }
  
  if (currentChunk.length > 0) {
    chunks.push(currentChunk.join(' '));
  }
  
  return chunks;
}

function recursiveChunk(
  text: string, 
  maxChunkSize: number = 512
): string[] {
  const separators = [
    '\\n## ',      // H2 headers
    '\\n### ',     // H3 headers
    '\\n\\n',       // Paragraphs
    '\\n',         // Lines
    '. ',         // Sentences
    ' ',          // Words (last resort)
  ];
  
  return recursiveSplit(text, separators, maxChunkSize);
}

function recursiveSplit(
  text: string, 
  separators: string[], 
  maxSize: number
): string[] {
  if (tokenCount(text) <= maxSize) {
    return [text];
  }
  
  const separator = separators[0];
  const nextSeparators = separators.slice(1);
  
  if (!text.includes(separator)) {
    if (nextSeparators.length === 0) {
      // Force split at maxSize
      return fixedSizeChunk(text, maxSize);
    }
    return recursiveSplit(text, nextSeparators, maxSize);
  }
  
  const parts = text.split(separator);
  const chunks: string[] = [];
  let currentChunk = '';
  
  for (const part of parts) {
    const candidate = currentChunk 
      ? currentChunk + separator + part 
      : part;
    
    if (tokenCount(candidate) <= maxSize) {
      currentChunk = candidate;
    } else {
      if (currentChunk) {
        chunks.push(currentChunk);
      }
      
      if (tokenCount(part) > maxSize) {
        // Part itself is too big, recurse with finer separators
        chunks.push(...recursiveSplit(part, nextSeparators, maxSize));
        currentChunk = '';
      } else {
        currentChunk = part;
      }
    }
  }
  
  if (currentChunk) {
    chunks.push(currentChunk);
  }
  
  return chunks;
}

interface DocumentSection {
  type: 'header' | 'paragraph' | 'list' | 'code' | 'table';
  content: string;
  level?: number;
  parent?: string;
}

async function documentAwareChunk(text: string): Promise<string[]> {
  // Step 1: Parse document structure
  const sections = parseDocumentStructure(text);
  
  // Step 2: Build section hierarchy
  const hierarchy = buildHierarchy(sections);
  
  // Step 3: Create chunks that preserve context
  const chunks: string[] = [];
  
  for (const section of hierarchy) {
    const chunk = buildContextualChunk(section);
    
    if (tokenCount(chunk) > 1000) {
      // Section too large, split while preserving header context
      const subChunks = splitLargeSection(section);
      chunks.push(...subChunks);
    } else if (tokenCount(chunk) < 100) {
      // Section too small, merge with siblings
      // (handled in post-processing)
      chunks.push(chunk);
    } else {
      chunks.push(chunk);
    }
  }
  
  // Step 4: Merge tiny chunks
  return mergeSmallChunks(chunks, 150);
}

function buildContextualChunk(section: DocumentSection): string {
  const parts: string[] = [];
  
  // Include parent headers for context
  if (section.parent) {
    parts.push('[Context: ' + section.parent + ']');
  }
  
  // Include the section content
  parts.push(section.content);
  
  return parts.join('\\n\\n');
}

function parseDocumentStructure(text: string): DocumentSection[] {
  const sections: DocumentSection[] = [];
  const lines = text.split('\\n');
  
  let currentSection: DocumentSection | null = null;
  let currentContent: string[] = [];
  let lastHeader = '';
  
  for (const line of lines) {
    // Detect headers
    const headerMatch = line.match(/^(#{1,6})\\s+(.+)$/);
    if (headerMatch) {
      // Save previous section
      if (currentSection) {
        currentSection.content = currentContent.join('\\n').trim();
        if (currentSection.content) {
          sections.push(currentSection);
        }
      }
      
      const level = headerMatch[1].length;
      lastHeader = headerMatch[2];
      
      currentSection = {
        type: 'header',
        content: '',
        level,
        parent: level > 1 ? lastHeader : undefined
      };
      currentContent = [line];
      continue;
    }
    
    // Detect code blocks (checking for code fence markers)
    const isCodeFence = line.trim().match(/^'''|^"""/);
    if (isCodeFence) {
      if (currentSection?.type === 'code') {
        currentContent.push(line);
        currentSection.content = currentContent.join('\\n');
        sections.push(currentSection);
        currentSection = null;
        currentContent = [];
      } else {
        if (currentSection) {
          currentSection.content = currentContent.join('\\n').trim();
          if (currentSection.content) sections.push(currentSection);
        }
        currentSection = { type: 'code', content: '', parent: lastHeader };
        currentContent = [line];
      }
      continue;
    }
    
    // Detect tables
    if (line.includes('|') && line.trim().startsWith('|')) {
      if (currentSection?.type !== 'table') {
        if (currentSection) {
          currentSection.content = currentContent.join('\\n').trim();
          if (currentSection.content) sections.push(currentSection);
        }
        currentSection = { type: 'table', content: '', parent: lastHeader };
        currentContent = [];
      }
      currentContent.push(line);
      continue;
    }
    
    // Default: paragraph
    if (!currentSection || currentSection.type === 'table') {
      if (currentSection) {
        currentSection.content = currentContent.join('\\n').trim();
        if (currentSection.content) sections.push(currentSection);
      }
      currentSection = { type: 'paragraph', content: '', parent: lastHeader };
      currentContent = [];
    }
    
    currentContent.push(line);
  }
  
  // Don't forget the last section
  if (currentSection) {
    currentSection.content = currentContent.join('\\n').trim();
    if (currentSection.content) sections.push(currentSection);
  }
  
  return sections;
}

interface BenchmarkResult {
  strategy: string;
  documentType: string;
  precisionAt5: number;
  recallAt10: number;
  answerQuality: number;
  indexingLatencyMs: number;
  avgChunkSize: number;
  totalChunks: number;
}

async function evaluateStrategy(
  strategy: ChunkingStrategy,
  documents: Document[],
  queries: Query[]
): Promise<BenchmarkResult[]> {
  const results: BenchmarkResult[] = [];
  
  for (const doc of documents) {
    // Chunk the document
    const startTime = Date.now();
    const chunks = await strategy.chunk(doc.content);
    const chunkingTime = Date.now() - startTime;
    
    // Embed chunks
    const embedStart = Date.now();
    const embeddings = await embedBatch(chunks);
    const embeddingTime = Date.now() - embedStart;
    
    // Index chunks
    await vectorStore.upsert(
      chunks.map((chunk, i) => ({
        id: doc.id + '-' + i,
        embedding: embeddings[i],
        metadata: { docId: doc.id, content: chunk }
      }))
    );
    
    // Evaluate queries for this document
    const docQueries = queries.filter(q => q.documentId === doc.id);
    
    for (const query of docQueries) {
      const retrieved = await vectorStore.search(query.text, { topK: 10 });
      
      const precisionAt5 = calculatePrecision(
        retrieved.slice(0, 5),
        query.relevantPassages
      );
      
      const recallAt10 = calculateRecall(
        retrieved,
        query.relevantPassages
      );
      
      const answer = await generateAnswer(query.text, retrieved.slice(0, 5));
      const answerQuality = await judgeAnswer(answer, query.goldAnswer);
      
      results.push({
        strategy: strategy.name,
        documentType: doc.type,
        precisionAt5,
        recallAt10,
        answerQuality,
        indexingLatencyMs: chunkingTime + embeddingTime,
        avgChunkSize: chunks.reduce((a, c) => a + tokenCount(c), 0) / chunks.length,
        totalChunks: chunks.length
      });
    }
  }
  
  return results;
}

Original text:
"The system processes 10,000 requests per second. 
Memory usage is approximately 4GB.
To install, run pip install mypackage."

Semantic chunking kept these together (high embedding similarity),
but they're answering completely different questions!

Strategy: Recursive with 512 tokens, 10% overlap
Expected Precision@5: ~0.76

Strategy: Recursive with code-block awareness
Chunk size: 512-768 tokens
Keep code blocks intact, even if they exceed chunk size

Strategy: Document-aware with cross-reference preservation
Include defined terms in chunk metadata
Link related chunks (e.g., clause and its amendment)

Strategy: Sentence-based (6-8 sentences)
Simple is better for short, focused articles

interface ChunkingPipeline {
  detectDocumentType(content: string): DocumentType;
  selectStrategy(type: DocumentType): ChunkingStrategy;
  chunk(content: string): Promise<Chunk[]>;
  validate(chunks: Chunk[]): ValidationResult;
}

class ProductionChunker implements ChunkingPipeline {
  private strategies: Map<DocumentType, ChunkingStrategy>;
  
  constructor() {
    this.strategies = new Map([
      ['technical', new RecursiveChunker({ maxSize: 600, overlap: 0.1 })],
      ['legal', new DocumentAwareChunker({ preserveReferences: true })],
      ['support', new SentenceChunker({ sentencesPerChunk: 6 })],
      ['default', new RecursiveChunker({ maxSize: 512, overlap: 0.1 })],
    ]);
  }
  
  detectDocumentType(content: string): DocumentType {
    // Simple heuristics - could be ML-based for better accuracy
    if (content.includes('