Multimodal AI in Production: Beyond Text

import sharp from 'sharp';

interface ImageProcessingOptions {
  maxWidth: number;
  maxHeight: number;
  quality: number;
  format: 'jpeg' | 'webp' | 'png';
}

class ImageProcessor {
  private defaults: ImageProcessingOptions = {
    maxWidth: 2048,
    maxHeight: 2048,
    quality: 85,
    format: 'jpeg'
  };
  
  async prepareForVision(
    imageBuffer: Buffer,
    options: Partial<ImageProcessingOptions> = {}
  ): Promise<{ base64: string; metadata: any }> {
    const opts = { ...this.defaults, ...options };
    
    // Get original metadata
    const metadata = await sharp(imageBuffer).metadata();
    
    // Process image
    let pipeline = sharp(imageBuffer);
    
    // Resize if too large
    if (metadata.width! > opts.maxWidth || metadata.height! > opts.maxHeight) {
      pipeline = pipeline.resize(opts.maxWidth, opts.maxHeight, {
        fit: 'inside',
        withoutEnlargement: true
      });
    }
    
    // Compress and convert
    const processed = await pipeline
      .jpeg({ quality: opts.quality })
      .toBuffer();
    
    // Convert to base64
    const base64 = processed.toString('base64');
    
    return {
      base64: `data:image/jpeg;base64,${base64}`,
      metadata: {
        originalSize: imageBuffer.length,
        processedSize: processed.length,
        compression: 1 - (processed.length / imageBuffer.length),
        dimensions: {
          original: { width: metadata.width, height: metadata.height },
          processed: await sharp(processed).metadata()
        }
      }
    };
  }
  
  async tileForLargeImage(
    imageBuffer: Buffer,
    tileSize: number = 1024
  ): Promise<string[]> {
    const metadata = await sharp(imageBuffer).metadata();
    const tiles: string[] = [];
    
    const cols = Math.ceil(metadata.width! / tileSize);
    const rows = Math.ceil(metadata.height! / tileSize);
    
    for (let row = 0; row < rows; row++) {
      for (let col = 0; col < cols; col++) {
        const tile = await sharp(imageBuffer)
          .extract({
            left: col * tileSize,
            top: row * tileSize,
            width: Math.min(tileSize, metadata.width! - col * tileSize),
            height: Math.min(tileSize, metadata.height! - row * tileSize)
          })
          .jpeg({ quality: 85 })
          .toBuffer();
        
        tiles.push(`data:image/jpeg;base64,${tile.toString('base64')}`);
      }
    }
    
    return tiles;
  }
}

// Usage with OpenAI Vision
async function analyzeImage(imageBuffer: Buffer): Promise<string> {
  const processor = new ImageProcessor();
  const { base64, metadata } = await processor.prepareForVision(imageBuffer);
  
  console.log(`Compressed image by ${(metadata.compression * 100).toFixed(1)}%`);
  
  const response = await openai.chat.completions.create({
    model: 'gpt-4-vision-preview',
    messages: [
      {
        role: 'user',
        content: [
          { type: 'text', text: 'Describe this image in detail.' },
          { type: 'image_url', image_url: { url: base64, detail: 'high' } }
        ]
      }
    ],
    max_tokens: 1000
  });
  
  return response.choices[0].message.content!;
}