openevidence-ci-integration

name	openevidence-ci-integration
description	Integrate OpenEvidence testing into CI/CD pipelines. Use when setting up automated testing, configuring GitHub Actions, or implementing continuous integration for clinical AI applications. Trigger with phrases like "openevidence ci", "openevidence github actions", "openevidence pipeline", "test openevidence ci", "automate openevidence tests".
allowed-tools	Read, Write, Edit, Bash(npm:*), Grep
version	1.0.0
license	MIT
author	Jeremy Longshore <[e-mail]>

OpenEvidence CI Integration

Overview

Integrate OpenEvidence testing and validation into CI/CD pipelines for healthcare applications.

Prerequisites

• GitHub, GitLab, or other CI platform
• OpenEvidence sandbox API credentials
• Test suite configured
• Secret management in CI platform

Instructions

Step 1: GitHub Actions Workflow

# .github/workflows/openevidence-ci.yml
name: OpenEvidence CI

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

env:
  NODE_VERSION: '20'

jobs:
  lint-and-typecheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Lint
        run: npm run lint

      - name: Type check
        run: npm run typecheck

  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run unit tests
        run: npm run test:unit -- --coverage

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage/lcov.info
          fail_ci_if_error: false

  integration-tests:
    runs-on: ubuntu-latest
    # Only run on main branch or explicit trigger
    if: github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'run-integration')
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run integration tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
          OPENEVIDENCE_BASE_URL: https://api.sandbox.openevidence.com
        run: npm run test:integration
        timeout-minutes: 10

  clinical-validation:
    runs-on: ubuntu-latest
    needs: [unit-tests]
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run clinical validation tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
        run: npm run test:clinical-validation

      - name: Upload validation report
        uses: actions/upload-artifact@v4
        with:
          name: clinical-validation-report
          path: reports/clinical-validation.json

Step 2: Test Configuration

// vitest.config.ts
import { defineConfig } from 'vitest/config';

export default defineConfig({
  test: {
    globals: true,
    environment: 'node',
    include: ['tests/**/*.test.ts'],
    exclude: ['tests/integration/**', 'tests/clinical-validation/**'],
    coverage: {
      provider: 'v8',
      reporter: ['text', 'lcov'],
      exclude: ['tests/**', 'node_modules/**'],
    },
    testTimeout: 10000,
  },
});

// vitest.config.integration.ts
import { defineConfig, mergeConfig } from 'vitest/config';
import baseConfig from './vitest.config';

export default mergeConfig(baseConfig, defineConfig({
  test: {
    include: ['tests/integration/**/*.test.ts'],
    exclude: [],
    testTimeout: 60000, // 60s for API calls
    retry: 2, // Retry flaky integration tests
    maxConcurrency: , 
  },
}));

Step 3: Unit Tests with Mocks

// tests/unit/clinical-query.test.ts
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { queryClinicalEvidence } from '../../src/services/clinical-query';

// Mock the OpenEvidence SDK
vi.mock('@openevidence/sdk', () => ({
  OpenEvidenceClient: vi.fn().mockImplementation(() => ({
    query: vi.fn().mockResolvedValue({
      id: 'test-query-123',
      answer: 'Mock clinical answer for testing purposes.',
      citations: [
        { source: 'Test Journal', title: 'Test Article', year: 2025 }
      ],
      confidence: 0.95,
      lastUpdated: '2025-01-01',
    }),
  })),
}));

describe('Clinical Query Service', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('should return formatted clinical response', async () => {
    const result = await queryClinicalEvidence('Test question');

    expect(result.).();
    (result.).();
    (result.).();
  });

  (,  () => {
     {  } =  ();
    vi.().( ({
      : vi.().( ()),
    }  ));

     (())
      ..();
  });
});

Step 4: Integration Tests

// tests/integration/openevidence-api.test.ts
import { describe, it, expect, beforeAll } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

describe('OpenEvidence API Integration', () => {
  let client: OpenEvidenceClient;

  beforeAll(() => {
    // Ensure we're using sandbox credentials
    const baseUrl = process.env.OPENEVIDENCE_BASE_URL;
    if (!baseUrl?.includes('sandbox')) {
      throw new Error('Integration tests must use sandbox environment');
    }

    client = new OpenEvidenceClient({
      apiKey: process.env.OPENEVIDENCE_API_KEY!,
      orgId: process.env.OPENEVIDENCE_ORG_ID!,
      baseUrl,
    });
  });

  it('should successfully query clinical evidence', async () => {
    const response = await client.query({
      question: 'What is the half-life of aspirin?',
      context: {
        specialty: 'pharmacology',
        urgency: ,
      },
    });

    (response.).();
    (response.).();
    (response..).();
    (response..).();
    (response.).();
  }, );

  (,  () => {
     (client.({
      : ,
      : { : , :  },
    }))..();
  });

  (,  () => {
     response =  client.({
      : ,
      : { : , :  },
    });

    response..( {
      (citation.).();
      (citation.).();
      (citation.).();
    });
  }, );
});

Step 5: Clinical Validation Tests

// tests/clinical-validation/known-answers.test.ts
import { describe, it, expect } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

interface ClinicalValidationCase {
  question: string;
  specialty: string;
  expectedKeywords: string[];
  mustNotContain?: string[];
}

const VALIDATION_CASES: ClinicalValidationCase[] = [
  {
    question: 'What is the first-line treatment for type 2 diabetes in adults?',
    specialty: 'endocrinology',
    expectedKeywords: ['metformin', 'lifestyle', 'HbA1c'],
    mustNotContain: ['insulin-first'],
  },
  {
    question: 'What are the contraindications for aspirin?',
    specialty: 'internal-medicine',
    expectedKeywords: ['bleeding', 'allergy', 'ulcer'],
  },
  {
    question: 'What is the target LDL for secondary prevention of cardiovascular disease?',
    specialty: 'cardiology',
    expectedKeywords: ['LDL', 'mg/dL', 'statin'],
  },
];

describe(,  {
   client =  ({
    : process..!,
    : process..!,
  });

   : [] = [];

  .( {
    (,  () => {
       response =  client.({
        : testCase.,
        : {
          : testCase.,
          : ,
        },
      });

       answerLower = response..();

      
       foundKeywords = testCase..(
         answerLower.(kw.())
      );

      
       forbiddenFound = testCase.?.(
         answerLower.(kw.())
      ) || [];

      
      results.({
        : testCase.,
        : testCase.,
        foundKeywords,
        forbiddenFound,
        : response.,
        : response..,
        : foundKeywords. >= testCase.. /  &&
                forbiddenFound. === ,
      });

      
      (foundKeywords.).(
        .(testCase.. / ),
        
      );
      (forbiddenFound).();
      (response.).();
    }, );
  });

  ( () => {
    
     fs =  ();
     fs.(, { :  });
     fs.(
      ,
      .({
        :  ().(),
        : results.,
        : results.( r.).,
        : results.( !r.).,
        results,
      }, , )
    );
  });
});

Step 6: Secret Management

# Store these as GitHub Secrets:
# - OPENEVIDENCE_SANDBOX_API_KEY
# - OPENEVIDENCE_SANDBOX_ORG_ID
# - OPENEVIDENCE_PROD_API_KEY (for deployment only)
# - OPENEVIDENCE_PROD_ORG_ID (for deployment only)

# For local development, use .env.test:
# OPENEVIDENCE_API_KEY=oe_sandbox_***
# OPENEVIDENCE_ORG_ID=org_sandbox_***
# OPENEVIDENCE_BASE_URL=https://api.sandbox.openevidence.com

Output

• CI workflow running on push and PR
• Unit tests with mocked SDK
• Integration tests against sandbox
• Clinical validation test suite
• Coverage reports

Package.json Scripts

{
  "scripts": {
    "test": "vitest",
    "test:unit": "vitest run -c vitest.config.ts",
    "test:integration": "vitest run -c vitest.config.integration.ts",
    "test:clinical-validation": "vitest run tests/clinical-validation",
    "test:ci": "vitest run --coverage",
    "typecheck": "tsc --noEmit",
    "lint": "eslint src tests"
  }
}

Error Handling

CI Issue	Cause	Solution
Integration test timeout	API slow or rate limited	Increase timeout, add retry
Secret not found	Missing GitHub secret	Add secret in repo settings
Flaky tests	Network variability	Add retries, improve assertions
Coverage drop	New code untested	Add tests, adjust thresholds

Resources

• GitHub Actions Documentation
• Vitest Documentation

Next Steps

For deployment integration, see openevidence-deploy-integration.