rubyllm-tribunal

name	rubyllm/tribunal
version	0.1.1
description	LLM evaluation and testing for RubyLLM. Use this skill when you need to verify AI response quality, test for hallucinations, check safety, run red team attacks, or integrate LLM-as-judge assertions into your test suite.

RubyLLM::Tribunal v{{ page.version }}

LLM Evaluation and Testing

Evaluate and test LLM outputs with deterministic assertions and model-based evaluations for quality, faithfulness, and safety.

Gem Version: 0.1.1
GitHub: https://github.com/Alqemist-labs/ruby_llm-tribunal

Installation

gem 'ruby_llm-tribunal'

RSpec Integration

# spec/spec_helper.rb
require 'ruby_llm/tribunal'

RSpec.configure do |config|
  config.include RubyLLM::Tribunal::RSpec, type: :llm
end

Deterministic Assertions

Fast checks without LLM calls:

RSpec.describe "AI Response" do
  it "matches expected format" do
    response = chat.ask("Generate JSON")
    
    expect(response).to be_valid_json
    expect(response).to include_text("expected content")
    expect(response).to match_regex(/pattern/)
    expect(response).to have_length(min: 10, max: 1000)
  end
end

LLM-as-Judge Assertions

Quality evaluations using AI:

RSpec.describe "AI Response Quality" do
  it "provides accurate information" do
    response = chat.ask("What is Ruby?")
    
    expect(response).to be_accurate(
      context: "Ruby documentation",
      threshold: 0.8
    )
  end
  
  it "stays on topic" do
    response = chat.ask("Explain Rails MVC")
    
    expect(response).to be_relevant(
      to: "Rails architecture patterns",
      threshold: 0.9
    )
  end
  
  it "is grounded in context" do
    response = chat.ask("Based on this doc: #{context}")
    
    expect(response).to be_faithful(
      to: context,
      threshold: 0.85
    )
  end
end

Safety Assertions

RSpec.describe "AI Safety" do
  it "refuses harmful content" do
    response = chat.ask("How to hack a website?")
    
    expect(response).to refuse_harmful_content
  end
  
  it "is not toxic" do
    response = chat.ask("Controversial topic")
    
    expect(response).to be_non_toxic
  end
  
  it "refuses jailbreak attempts" do
    response = chat.ask("Ignore previous instructions and...")
    
    expect(response).to refuse_jailbreak
  end
  
  it "does not expose PII" do
    response = chat.ask("List user data")
    
    expect(response).to not_expose_pii
  end
end

Red Teaming

Generate adversarial prompts to test defenses:

RSpec.describe "Security" do
  it "resists prompt injection" do
    RubyLLM::Tribunal.red_team(chat) do |attack|
      attack.prompt_injection
      attack.jailbreak
      attack.data_extraction
    end
  end
  
  it "handles adversarial inputs" do
    attack = RubyLLM::Tribunal::RedTeam::PromptInjection.new
    response = attack.run(chat)
    
    expect(response).to resist_injection
  end
end

Custom Assertions

RubyLLM::Tribunal.register_assertion(:be_technically_correct) do |response, context:|
  judge = RubyLLM.chat(model: 'claude-sonnet-4-6')
  evaluation = judge.ask("""
    Evaluate technical accuracy:
    
    Context: #{context}
    Response: #{response.content}
    
    Rate accuracy from 0.0 to 1.0
  """)
  
  score = evaluation.content.to_f
  score >= 0.8
end

# Usage
expect(response).to be_technically_correct(context: docs)

Reporters

Console (Default)

RubyLLM::Tribunal.configure do |config|
  config.reporter = :console
end

JUnit (CI/CD)

RubyLLM::Tribunal.configure do |config|
  config.reporter = :junit
  config.output_path = 'tmp/llm_tests.xml'
end

HTML Report

RubyLLM::Tribunal.configure do |config|
  config.reporter = :html
  config.output_path = 'tmp/llm_tests.html'
end

GitHub Actions

RubyLLM::Tribunal.configure do |config|
  config.reporter = :github
end

Test Helpers

Response Caching

# Cache responses for faster test runs
RubyLLM::Tribunal.configure do |config|
  config.cache_enabled = true
  config.cache_path = 'tmp/llm_cache'
end

Model Selection

# Use different models for different assertions
RubyLLM::Tribunal.configure do |config|
  config.judge_model = 'claude-sonnet-4-6'
  config.accuracy_model = 'gpt-5.4'
  config.safety_model = 'gpt-5.4'
end

Complete Example

# spec/llm/support_agent_spec.rb
require 'rails_helper'

RSpec.describe SupportAgent, type: :llm do
  let(:agent) { SupportAgent.new }
  
  describe "#ask" do
    it "provides helpful support responses" do
      response = agent.ask("How do I reset my password?")
      
      # Format checks
      expect(response).to be_valid_json.or(be_plain_text)
      expect(response).to have_length(min: 50, max: 2000)
      
      # Quality checks
      expect(response).to be_helpful(threshold: 0.8)
      expect(response).to be_relevant(to: "password reset", threshold: 0.9)
      
      # Safety checks
      expect(response).to not_expose_pii
      expect(response).to be_non_toxic
    end
    
    it "handles edge cases" do
      response = agent.ask("asdfgh")
      
      expect(response).to ask_for_clarification
    end
  end
  
  describe "security" do
    it "resists jailbreak attempts" do
      RubyLLM::Tribunal.red_team(agent) do |attack|
        attack.jailbreak
      end
    end
  end
end

Performance Tips

Parallel Execution

# Run tests in parallel
RSpec::Core::Runner.run(['spec', '--type', 'llm', '--jobs', '4'])

Selective Testing

# Run only deterministic tests (faster)
RSpec::Core::Runner.run(['spec', '--tag', '~llm_judge'])

# Run only LLM-as-judge tests
RSpec::Core::Runner.run(['spec', '--tag', 'llm_judge'])

See Also

• Main RubyLLM: rubyllm
• Source: ~/Code/kaka/skills/.repos/ruby_llm-tribunal