openevidence-observability

name	openevidence-observability
description	Set up comprehensive observability for OpenEvidence integrations with metrics, traces, and alerts. Use when implementing monitoring for clinical AI operations, setting up dashboards, or configuring alerting for healthcare application health. Trigger with phrases like "openevidence monitoring", "openevidence metrics", "openevidence observability", "monitor openevidence", "openevidence alerts".
allowed-tools	Read, Write, Edit
version	1.0.0
license	MIT
author	Jeremy Longshore <[e-mail]>

OpenEvidence Observability

Overview

Set up comprehensive observability for OpenEvidence clinical AI integrations in healthcare environments.

Prerequisites

• Prometheus or compatible metrics backend
• OpenTelemetry SDK installed
• Grafana or similar dashboarding tool
• AlertManager or PagerDuty configured

Key Metrics

Metric	Type	Description	Alert Threshold
openevidence_requests_total	Counter	Total API requests	N/A
openevidence_request_duration_seconds	Histogram	Request latency	P95 > 15s
openevidence_errors_total	Counter	Error count by type	> 5% rate
openevidence_cache_hits_total	Counter	Cache hit count	< 50% rate
openevidence_rate_limit_remaining	Gauge	Rate limit headroom	< 10%
openevidence_deepconsult_active	Gauge	Active DeepConsults	> 50

Instructions

Step 1: Prometheus Metrics

// src/monitoring/metrics.ts
import { Registry, Counter, Histogram, Gauge, Summary } from 'prom-client';

const registry = new Registry();

// Request metrics
export const requestCounter = new Counter({
  name: 'openevidence_requests_total',
  help: 'Total OpenEvidence API requests',
  labelNames: ['method', 'specialty', 'status'],
  registers: [registry],
});

export const requestDuration = new Histogram({
  name: 'openevidence_request_duration_seconds',
  help: 'OpenEvidence request duration',
  labelNames: ['method', 'specialty'],
  buckets: [0.5, 1, 2, 5, 10, 15, 30, 60],
  registers: [registry],
});

// Error metrics
export const errorCounter = new Counter({
  : ,
  : ,
  : [, ],
  : [registry],
});


  cacheHits =  ({
  : ,
  : ,
  : [registry],
});

  cacheMisses =  ({
  : ,
  : ,
  : [registry],
});


  rateLimitRemaining =  ({
  : ,
  : ,
  : [registry],
});

  rateLimitPercent =  ({
  : ,
  : ,
  : [registry],
});


  deepConsultActive =  ({
  : ,
  : ,
  : [registry],
});

  deepConsultDuration =  ({
  : ,
  : ,
  : [, ],
  : [, , , , , ],
  : [registry],
});


  confidenceScore =  ({
  : ,
  : ,
  : [],
  : [, , , ],
  : [registry],
});

 { registry };

Step 2: Instrumented Client Wrapper

// src/monitoring/instrumented-client.ts
import { OpenEvidenceClient } from '@openevidence/sdk';
import {
  requestCounter,
  requestDuration,
  errorCounter,
  cacheHits,
  cacheMisses,
  rateLimitRemaining,
  rateLimitPercent,
  confidenceScore,
} from './metrics';
import { trace, SpanStatusCode, Span } from '@opentelemetry/api';

const tracer = trace.getTracer('openevidence-client');

export class InstrumentedOpenEvidenceClient {
  private client: OpenEvidenceClient;
  private cache: ClinicalQueryCache;

  constructor(config: any, cache: ClinicalQueryCache) {
    this.client = new OpenEvidenceClient(config);
    this.cache = cache;
  }

  async query(request: ClinicalQueryRequest): Promise<ClinicalQueryResponse> {
    const { specialty } = request.context;

    return tracer.startActiveSpan('openevidence.query',  (: ) => {
       timer = requestDuration.({ : , specialty });

       {
        
        span.(, specialty);
        span.(, request..);

        
         cached =  ..(request., request.);
         (cached) {
          cacheHits.();
          span.(, );
          ({ :  });
          requestCounter.({ : , specialty, :  });
           cached;
        }
        cacheMisses.();
        span.(, );

        
         response =  ..(request);

        
        .(response.);

        
        confidenceScore.({ specialty }, response.);

        
         ..(request., request., response);

        
        ({ :  });
        requestCounter.({ : , specialty, :  });
        span.({ : . });

         response;
      }  (: ) {
        
        ({ :  });
        requestCounter.({ : , specialty, :  });
        errorCounter.({ : error. || , specialty });

        span.({ : ., : error. });
        span.(error);

         error;
      }  {
        span.();
      }
    });
  }

   (: ):  {
     remaining = (headers[] || );
     limit = (headers[] || );

    rateLimitRemaining.(remaining);
    rateLimitPercent.(((limit - remaining) / limit) * );
  }
}

Step 3: Distributed Tracing Setup

// src/monitoring/tracing.ts
import { NodeSDK } from '@opentelemetry/sdk-node';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';

export function initTracing(): void {
  const sdk = new NodeSDK({
    resource: new Resource({
      [SemanticResourceAttributes.SERVICE_NAME]: 'clinical-evidence-api',
      [SemanticResourceAttributes.SERVICE_VERSION]: process.env.npm_package_version,
      [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: process.env.NODE_ENV,
    }),
    traceExporter:  (),
    : [
       ({
        :  {
          
           (request.?.()) {
            span.(, );
          }
        },
      }),
       (),
    ],
  });

  sdk.();

  process.(,  {
    sdk.().(.);
  });
}

Step 4: Structured Logging

// src/monitoring/logger.ts
import pino from 'pino';

export const logger = pino({
  name: 'clinical-evidence-api',
  level: process.env.LOG_LEVEL || 'info',
  formatters: {
    level: (label) => ({ level: label }),
  },
  // Redact PHI fields
  redact: {
    paths: ['patient.*', 'patientId', 'mrn', '*.ssn'],
    censor: '[REDACTED]',
  },
});

// OpenEvidence-specific logger
export const oeLogger = logger.child({ service: 'openevidence' });

export function logClinicalQuery(
  queryId: string,
  specialty: string,
  duration: number,
  cached: boolean,
  confidence?: number
): void {
  oeLogger.info({
    event: 'clinical_query',
    queryId,
    specialty,
    durationMs: duration,
    cached,
    confidence,
  });
}

  ():  {
  oeLogger.({
    : ,
    consultId,
    specialty,
    status,
    : duration,
  });
}

  ():  {
  oeLogger.({
    : ,
    : error.,
    : error.,
    ...context,
  });
}

Step 5: Alert Rules

# prometheus/openevidence_alerts.yaml
groups:
  - name: openevidence_alerts
    rules:
      # High error rate
      - alert: OpenEvidenceHighErrorRate
        expr: |
          rate(openevidence_errors_total[5m]) /
          rate(openevidence_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
          service: openevidence
        annotations:
          summary: "OpenEvidence error rate > 5%"
          description: "Error rate is {{ $value | humanizePercentage }}"

      # Critical error rate
      - alert: OpenEvidenceCriticalErrorRate
        expr: |
          rate(openevidence_errors_total[5m]) /
          rate(openevidence_requests_total[5m]) > 0.20
        for: 2m
        labels:
          severity: critical
          service: openevidence
        annotations:
          summary: "OpenEvidence error rate > 20%"
          description: "Critical error rate - possible outage"

      # High latency
      - alert: OpenEvidenceHighLatency
                  
        
           
           
        
           
           

      
        
                  
        
           
           
        
           
           

      
        
           
         
        
           
           
        
           
           

      
        
           
         
        
           
           
        
           
           

Step 6: Grafana Dashboard

{
  "dashboard": {
    "title": "OpenEvidence Clinical AI",
    "panels": [
      {
        "title": "Request Rate",
        "type": "stat",
        "targets": [{
          "expr": "sum(rate(openevidence_requests_total[5m]))",
          "legendFormat": "req/s"
        }]
      },
      {
        "title": "Error Rate",
        "type": "gauge",
        "targets": [{
          "expr": "sum(rate(openevidence_errors_total[5m])) / sum(rate(openevidence_requests_total[5m])) * 100"
        }],
        "fieldConfig" 
           
             
             
               
               
                   
                   
                   
              
            
          
        
      
      
         
         
         
           
        
      
      
         
         
         
           
           
        
      
      
         
         
         
           
        
         
           
             
             
               
               
                   
                   
                   
              
            
          
        
      
      
         
         
         
           
        
      
    
  

Metrics Endpoint

// src/routes/metrics.ts
import { Router } from 'express';
import { registry } from '../monitoring/metrics';

const router = Router();

router.get('/metrics', async (req, res) => {
  res.set('Content-Type', registry.contentType);
  res.send(await registry.metrics());
});

export default router;

Output

• Prometheus metrics collection
• Distributed tracing with OpenTelemetry
• Structured logging with PHI redaction
• Alert rules for critical conditions
• Grafana dashboard

Observability Checklist

• Metrics endpoint exposed
• Request counters and histograms
• Error counters with labels
• Cache hit/miss tracking
• Rate limit monitoring
• Distributed tracing enabled
• Structured logging configured
• Alert rules deployed
• Dashboard created

Resources

• Prometheus Best Practices
• OpenTelemetry Documentation
• Grafana Dashboards

Next Steps

For incident response, see openevidence-incident-runbook.