agentic-eval

name	agentic-eval
category	developer-experience
subcategory	cli
location	file:///.rulesync/skills/agentic-eval/SKILL.md
description	Patterns and techniques for evaluating and improving AI agent outputs.
targets	["*"]
license	MIT
metadata	{"author":"GitHub, Inc. (derived)","version":"0.0.1"}
claudecode	{}
opencode	{}
codexcli	{"short-description":"Toolkit guidance for agentic-eval"}
copilot	{}
geminicli	{}
antigravity	{}

Portions derived from github/awesome-copilot (MIT License). Used under MIT License.

Agentic Evaluation Patterns

Patterns for self-improvement through iterative evaluation and refinement, built using the .NET and Microsoft.Extensions.AI ecosystem.

Overview

Evaluation patterns enable agents to assess and improve their own outputs, moving beyond single-shot generation to iterative refinement loops.

Generate → Evaluate → Critique → Refine → Output

When to Use

• Quality-critical generation: Code, reports, analysis requiring high accuracy
• Tasks with clear evaluation criteria: Defined success metrics exist
• Content requiring specific standards: Style guides, compliance, formatting

Pattern 1: Basic Reflection

Agent evaluates and improves its own output through self-critique.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;

namespace DotNetAgentHarness.Evals.Engine;

public class BasicReflection(IChatClient chatClient)
{
    public async Task<string> ReflectAndRefineAsync(string task, string[] criteria, int maxIterations = 3, CancellationToken cancellationToken = default)
    {
        var response = await chatClient.GetResponseAsync($"Complete this task:\n{task}", cancellationToken: cancellationToken);
         output = response.Text ?? .Empty;

         ( i = ; i < maxIterations; i++)
        {
             prompt = , PASSFAIL;

             critiqueResponse =  chatClient.GetResponseAsync(
                prompt,
                 ChatOptions { ResponseFormat = ChatResponseFormat.Json },
                cancellationToken);

             critiqueText = critiqueResponse.Text ?? ;
            Dictionary<, CritiqueResult>? critiqueData = ;

            
            {
                critiqueData = JsonSerializer.Deserialize<Dictionary<, CritiqueResult>>(critiqueText);
            }
             (JsonException)
            {
                
                critiqueData =  Dictionary<, CritiqueResult>();
            }

             (critiqueData !=  && critiqueData.Count >  && critiqueData.Values.All(c => c.Status == ))
            {
                 output;
            }

             failed = critiqueData?
                .Where(kvp => kvp.Value.Status == )
                .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Feedback) ??  Dictionary<, >();

             (failed.Count == )
            {
                
                ;
            }

             failedJson = JsonSerializer.Serialize(failed);

             refinePrompt = ;
             improvedResponse =  chatClient.GetResponseAsync(refinePrompt, cancellationToken: cancellationToken);
            output = improvedResponse.Text ?? .Empty;
        }

         output;
    }

      
    {
        []
          Status { ; ; } = .Empty;

        []
          Feedback { ; ; } = .Empty;
    }
}