ai-integration

name	ai-integration
description	AI/ML model integration including vision, audio, embeddings, and RAG implementation patterns
category	integrations
triggers	["ai integration","ai ml","embeddings","rag","vision api","audio transcription","openai","anthropic"]

AI Integration

Enterprise AI/ML model integration patterns for vision, audio, embeddings, and RAG systems. This skill covers API integration, prompt engineering, and production deployment.

Purpose

Integrate AI capabilities into applications effectively:

• Implement vision and image understanding
• Add audio transcription and processing
• Build semantic search with embeddings
• Create RAG (Retrieval Augmented Generation) systems
• Handle rate limiting and error recovery
• Optimize costs and latency

Features

1. Vision API Integration

import Anthropic from '@anthropic-ai/sdk';
import OpenAI from 'openai';

const anthropic = new Anthropic();
const openai = new OpenAI();

// Analyze image with Claude
async function analyzeImageWithClaude(
  imageUrl: string | Buffer,
  prompt: string
): Promise<string> {
  const imageSource = typeof imageUrl === 'string'
    ? { type: 'url' as const, url: imageUrl }
    : {
        type: 'base64' as const,
        media_type: 'image/jpeg' as const,
        data: imageUrl.toString('base64'),
      };

  const response = await anthropic.messages.create({
    model: 'claude-sonnet-4-20250514',
    max_tokens: 1024,
    messages: [{
      role: 'user',
      content: [
        {
          type: 'image',
          source: imageSource,
        },
        {
          type: 'text',
          text: prompt,
        },
      ],
    }],
  });

  return response.content[0].type === 'text'
    ? response.content[0].text
    : '';
}

// Extract structured data from image
interface ProductInfo {
  name: string;
  description: string;
  price?: string;
  category?: string;
  features: string[];
}

async function extractProductFromImage(imageBuffer: Buffer): Promise<ProductInfo> {
  const prompt = `Analyze this product image and extract:
1. Product name
2. Description (2-3 sentences)
3. Price (if visible)
4. Category
5. Key features (list)

Return as JSON only, no explanation.`;

  const response = await analyzeImageWithClaude(imageBuffer, prompt);

  try {
    return JSON.parse(response);
  } catch {
    throw new Error('Failed to parse product information');
  }
}

// OCR with GPT-4 Vision
async function extractTextFromImage(imageUrl: string): Promise<string> {
  const response = await openai.chat.completions.create({
    model: 'gpt-4o',
    messages: [{
      role: 'user',
      content: [
        {
          type: 'image_url',
          image_url: { url: imageUrl, detail: 'high' },
        },
        {
          type: 'text',
          text: 'Extract all text from this image. Preserve the original formatting and structure as much as possible.',
        },
      ],
    }],
    max_tokens: 4096,
  });

  return response.choices[0].message.content || '';
}

// Batch image processing
async function batchAnalyzeImages(
  images: Array<{ id: string; url: string }>,
  prompt: string,
  concurrency: number = 3
): Promise<Map<string, string>> {
  const results = new Map<string, string>();
  const queue = new PQueue({ concurrency });

  await Promise.all(
    images.map(image =>
      queue.add(async () => {
        try {
          const result = await analyzeImageWithClaude(image.url, prompt);
          results.set(image.id, result);
        } catch (error) {
          results.set(image.id, `Error: ${error.message}`);
        }
      })
    )
  );

  return results;
}

2. Audio Processing

import { Readable } from 'stream';

// Transcribe audio with Whisper
async function transcribeAudio(
  audioFile: Buffer | string,
  options: {
    language?: string;
    prompt?: string;
    responseFormat?: 'json' | 'text' | 'srt' | 'vtt';
    timestamps?: boolean;
  } = {}
): Promise<TranscriptionResult> {
  const {
    language,
    prompt,
    responseFormat = 'json',
    timestamps = false,
  } = options;

  const file = typeof audioFile === 'string'
    ? fs.createReadStream(audioFile)
    : Readable.from(audioFile);

  const response = await openai.audio.transcriptions.create({
    file,
    model: 'whisper-1',
    language,
    prompt,
    response_format: timestamps ? 'verbose_json' : responseFormat,
  });

  if (timestamps && typeof response !== 'string') {
    return {
      text: response.text,
      segments: response.segments?.( ({
        : seg.,
        : seg.,
        : seg.,
      })),
      : response.,
    };
  }

   { :  response ===  ? response : response. };
}


 * (
  : 
): <> {
  
   deepgram =  (process..!);

   connection =  deepgram..({
    : ,
    : ,
    : ,
    : ,
  });

  connection.(,  {
     transcript = message.?.?.[]?.;
     (transcript) {
       transcript;
    }
  });

  
   reader = audioStream.();
   () {
     { done, value } =  reader.();
     (done) ;
    connection.(value);
  }

  connection.();
}


  (): <> {
   { voice = , model = , speed =  } = options;

   response =  openai...({
    model,
    voice,
    : text,
    speed,
  });

   .( response.());
}

3. Embeddings & Vector Search

import { Pinecone } from '@pinecone-database/pinecone';

const pinecone = new Pinecone();

// Generate embeddings
async function generateEmbeddings(
  texts: string[],
  model: string = 'text-embedding-3-small'
): Promise<number[][]> {
  const response = await openai.embeddings.create({
    model,
    input: texts,
  });

  return response.data.map(d => d.embedding);
}

// Index documents
interface Document {
  id: string;
  content: string;
  metadata?: Record<string, any>;
}

async function indexDocuments(
  documents: Document[],
  indexName: string,
  namespace: string = 'default'
): Promise<> {
   index = pinecone.(indexName);

  
   batchSize = ;
   ( i = ; i < documents.; i += batchSize) {
     batch = documents.(i, i + batchSize);

     embeddings =  (
      batch.( d.)
    );

     vectors = batch.( ({
      : doc.,
      : embeddings[j],
      : {
        : doc..(, ), 
        ...doc.,
      },
    }));

     index.(namespace).(vectors);
  }
}


  {
  : ;
  : ;
  : ;
  ?: <, >;
}

  (): <[]> {
   {
    namespace = ,
    topK = ,
    filter,
    minScore = ,
  } = options;

   [queryEmbedding] =  ([query]);

   index = pinecone.(indexName);
   results =  index.(namespace).({
    : queryEmbedding,
    topK,
    filter,
    : ,
  });

   results.
    ?.( m. && m. >= minScore)
    .( ({
      : match.,
      : match. || ,
      : match.?.   || ,
      : match.,
    })) || [];
}

4. RAG Implementation

interface RAGConfig {
  indexName: string;
  namespace?: string;
  topK?: number;
  model?: string;
  systemPrompt?: string;
}

class RAGSystem {
  private config: RAGConfig;

  constructor(config: RAGConfig) {
    this.config = {
      namespace: 'default',
      topK: 5,
      model: 'claude-sonnet-4-20250514',
      systemPrompt: 'You are a helpful assistant. Answer based on the provided context.',
      ...config,
    };
  }

  async query(question: string): Promise<RAGResponse> {
    // Step 1: Retrieve relevant documents
    const context = await semanticSearch(question, this.config.indexName, {
      namespace: this.config.namespace,
      topK: this.config.,
    });

     (context. === ) {
       {
        : ,
        : [],
        : ,
      };
    }

    
     contextText = context
      .( )
      .();

    
     response =  anthropic..({
      : ..!,
      : ,
      : ,
      : [{
        : ,
        : question,
      }],
    });

     answer = response.[]. === 
      ? response.[].
      : ;

     {
      answer,
      : context.( ({
        : c.,
        : c.,
        : c.,
      })),
      : .(...context.( c.)),
    };
  }

  
   (
    : ,
    ?: []
  ): <> {
    
     semanticResults =  (question, .., {
      : ..,
      : ..! * ,
    });

    
     results = semanticResults;
     (keywords && keywords. > ) {
      results = semanticResults.(
        keywords.(
          r..().(k.())
        )
      );
    }

    
     topResults = results.(, ..);

    
     .(question, topResults);
  }

    (
    : ,
    : []
  ): <> {
    
  }
}


 rag =  ({
  : ,
  : ,
  : ,
});

 response =  rag.();

5. Structured Output

import { z } from 'zod';
import { zodResponseFormat } from 'openai/helpers/zod';

// Define schema
const SentimentSchema = z.object({
  sentiment: z.enum(['positive', 'negative', 'neutral']),
  confidence: z.number().min(0).max(1),
  topics: z.array(z.string()),
  summary: z.string(),
});

type SentimentAnalysis = z.infer<typeof SentimentSchema>;

// Get structured output
async function analyzeSentiment(text: string): Promise<SentimentAnalysis> {
  const response = await openai.beta.chat.completions.parse({
    model: 'gpt-4o',
    messages: [{
      role: 'system',
      content: ,
    }, {
      : ,
      : text,
    }],
    : (, ),
  });

   response.[]..!;
}


  (): <{
  : [];
  : [];
  : [];
  : [];
}> {
   response =  anthropic..({
    : ,
    : ,
    : [{
      : ,
      : ,
      : {
        : ,
        : {
          : {
            : ,
            : { :  },
            : ,
          },
          : {
            : ,
            : { :  },
            : ,
          },
          : {
            : ,
            : { :  },
            : ,
          },
          : {
            : ,
            : { :  },
            : ,
          },
        },
        : [, , , ],
      },
    }],
    : { : , :  },
    : [{
      : ,
      : ,
    }],
  });

   toolUse = response..( c. === );
   toolUse?.  ;
}

6. Production Patterns

// Rate limiting and retry
import Bottleneck from 'bottleneck';

const limiter = new Bottleneck({
  reservoir: 100, // Initial tokens
  reservoirRefreshAmount: 100,
  reservoirRefreshInterval: 60 * 1000, // Per minute
  maxConcurrent: 10,
});

async function withRateLimit<T>(fn: () => Promise<T>): Promise<T> {
  return limiter.schedule(fn);
}

// Retry with exponential backoff
async function withRetry<T>(
  fn: () => Promise<T>,
  maxRetries: number = 3,
  baseDelay: number = 1000
): Promise<T> {
  let lastError: Error;

  for (let attempt = 0; attempt < maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error) {
      lastError = error as ;

      
       (error. ===  || error. >= ) {
         delay = baseDelay * .(, attempt);
          ( (r, delay));
        ;
      }

       error;
    }
  }

   lastError!;
}


 cache =  <, { : ; :  }>();

  (): <[]> {
   key = ;
   cached = cache.(key);

   (cached && cached. > .()) {
     cached.;
  }

   [embedding] =  ([text]);
  cache.(key, { : embedding, : .() + ttl });

   embedding;
}


  {
   : <, > =  ();

  (: , : , : ):  {
     pricing = [model] || { : , :  };
     cost = (inputTokens * pricing. + outputTokens * pricing.) / ;

     current = ..(model) || ;
    ..(model, current + cost);
  }

  (): <, > {
     .(.);
  }

  ():  {
     .(..()).( a + b, );
  }
}

Use Cases

1. Document Q&A System

// Build document Q&A
async function buildDocumentQA(documents: string[]): Promise<RAGSystem> {
  // Chunk documents
  const chunks = documents.flatMap((doc, docIndex) =>
    chunkText(doc, 500, 50).map((chunk, chunkIndex) => ({
      id: `doc-${docIndex}-chunk-${chunkIndex}`,
      content: chunk,
      metadata: { documentIndex: docIndex },
    }))
  );

  // Index chunks
  await indexDocuments(chunks, 'document-qa');

  // Return RAG system
  return new RAGSystem({
    indexName: 'document-qa',
    topK: 5,
    systemPrompt: 'Answer questions based on the provided documents.',
  });
}

2. Content Moderation

// Moderate content with AI
async function moderateContent(content: string): Promise<ModerationResult> {
  const response = await openai.moderations.create({ input: content });
  const result = response.results[0];

  return {
    flagged: result.flagged,
    categories: Object.entries(result.categories)
      .filter(([_, flagged]) => flagged)
      .map(([category]) => category),
    scores: result.category_scores,
  };
}

Best Practices

Do's

• Implement rate limiting - Respect API limits
• Cache embeddings - Avoid redundant API calls
• Handle errors gracefully - Implement retry logic
• Monitor costs - Track token usage
• Use streaming - For better UX with long responses
• Chunk appropriately - Balance context vs. relevance

Don'ts

• Don't expose API keys in frontend code
• Don't skip input validation
• Don't ignore rate limit errors
• Don't cache sensitive data inappropriately
• Don't use overly large context windows
• Don't forget fallback strategies

Related Skills

• api-architecture - API design patterns
• caching-strategies - Caching for AI responses
• backend-development - Integration patterns

Reference Resources

• OpenAI API Reference
• Anthropic API Reference
• Pinecone Documentation
• LangChain Documentation