Context Window Optimization: Getting More from Limited Tokens

┌─────────────────────────────────────────────────────────────┐
│                    CONTEXT WINDOW LAYOUT                    │
├─────────────────────────────────────────────────────────────┤
│  ┌─────────────────────────────────────────────────────┐   │
│  │ SYSTEM PROMPT (High Attention Zone)                 │   │
│  │ Role, rules, critical instructions                  │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                             │
│  ┌─────────────────────────────────────────────────────┐   │
│  │ CRITICAL CONTEXT                                     │   │
│  │ Most relevant documents, key facts                   │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                             │
│  ┌─────────────────────────────────────────────────────┐   │
│  │ MIDDLE ZONE (Lower Attention - "Lost in Middle")    │   │
│  │ Supporting context, additional documents             │   │
│  │ Less critical information                            │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                             │
│  ┌─────────────────────────────────────────────────────┐   │
│  │ RECENT CONVERSATION (Recency Boost)                  │   │
│  │ Last few messages, recent context                    │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                             │
│  ┌─────────────────────────────────────────────────────┐   │
│  │ USER QUERY (High Attention Zone)                     │   │
│  │ Current question - immediate focus                   │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                             │
│  ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐   │
│    OUTPUT SPACE (Reserved for generation)                   │
│  └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘   │
└─────────────────────────────────────────────────────────────┘

from typing import List, Tuple
from dataclasses import dataclass
import tiktoken

@dataclass
class ContextChunk:
    content: str
    source: str
    relevance_score: float
    timestamp: float
    importance: float

class ContextManager:
    def __init__(self, max_tokens: int = 32000, model: str = "gpt-4"):
        self.max_tokens = max_tokens
        self.encoder = tiktoken.encoding_for_model(model)
        
        # Token budget allocation
        self.budgets = {
            "system_prompt": int(max_tokens * 0.05),   # 5%
            "retrieved_docs": int(max_tokens * 0.50),  # 50%
            "conversation": int(max_tokens * 0.25),   # 25%
            "output_reserve": int(max_tokens * 0.15), # 15%
            "buffer": int(max_tokens * 0.05),         # 5%
        }
    
    def count_tokens(self, text: str) -> int:
        return len(self.encoder.encode(text))
    
    def build_context(
        self,
        system_prompt: str,
        retrieved_chunks: List[ContextChunk],
        conversation_history: List[dict],
        user_query: str
    ) -> str:
        """Build optimized context within token budget."""
        
        context_parts = []
        used_tokens = 0
        
        # 1. System prompt (always include, truncate if needed)
        system_tokens = self.count_tokens(system_prompt)
        if system_tokens > self.budgets["system_prompt"]:
            system_prompt = self._truncate_to_tokens(
                system_prompt, self.budgets["system_prompt"]
            )
        context_parts.append(("system", system_prompt))
        used_tokens += self.count_tokens(system_prompt)
        
        # 2. Retrieved documents (prioritized by hybrid score)
        retrieved_chunks = self._rank_chunks(retrieved_chunks, user_query)
        doc_tokens = 0
        for chunk in retrieved_chunks:
            chunk_tokens = self.count_tokens(chunk.content)
            if doc_tokens + chunk_tokens <= self.budgets["retrieved_docs"]:
                context_parts.append(("document", chunk.content))
                doc_tokens += chunk_tokens
        used_tokens += doc_tokens
        
        # 3. Conversation history (most recent first, summarize old)
        conv_tokens = 0
        recent_messages = []
        for msg in reversed(conversation_history):
            msg_tokens = self.count_tokens(str(msg))
            if conv_tokens + msg_tokens <= self.budgets["conversation"]:
                recent_messages.insert(0, msg)
                conv_tokens += msg_tokens
            else:
                # Summarize remaining old messages
                old_messages = conversation_history[:len(conversation_history) - len(recent_messages)]
                if old_messages:
                    summary = self._summarize_conversation(old_messages)
                    context_parts.append(("history_summary", summary))
                break
        
        for msg in recent_messages:
            context_parts.append(("message", str(msg)))
        used_tokens += conv_tokens
        
        # 4. User query (always at the end)
        context_parts.append(("query", user_query))
        used_tokens += self.count_tokens(user_query)
        
        return self._assemble_context(context_parts)
    
    def _rank_chunks(
        self, 
        chunks: List[ContextChunk], 
        query: str
    ) -> List[ContextChunk]:
        """Hybrid ranking: relevance + recency + importance."""
        
        def score(chunk: ContextChunk) -> float:
            # Recency decay (halves every 24 hours)
            age_hours = (time.time() - chunk.timestamp) / 3600
            recency = 0.5 ** (age_hours / 24)
            
            return (
                0.50 * chunk.relevance_score +
                0.25 * recency +
                0.25 * chunk.importance
            )
        
        return sorted(chunks, key=score, reverse=True)
    
    def _summarize_conversation(self, messages: List[dict]) -> str:
        """Use LLM to summarize old conversation."""
        # In production, call LLM here
        return f"[Summary of {len(messages)} earlier messages]"