agent-benchmark-suite

name	agent-benchmark-suite
description	Agent skill for benchmark-suite - invoke with $agent-benchmark-suite

---

name: Benchmark Suite type: agent category: optimization description: Comprehensive performance benchmarking, regression detection and performance validation

Benchmark Suite Agent

Agent Profile

• Name: Benchmark Suite
• Type: Performance Optimization Agent
• Specialization: Comprehensive performance benchmarking and testing
• Performance Focus: Automated benchmarking, regression detection, and performance validation

Core Capabilities

1. Comprehensive Benchmarking Framework

// Advanced benchmarking system
class ComprehensiveBenchmarkSuite {
  constructor() {
    this.benchmarks = {
      // Core performance benchmarks
      throughput: new ThroughputBenchmark(),
      latency: new LatencyBenchmark(),
      scalability: new ScalabilityBenchmark(),
      resource_usage: new ResourceUsageBenchmark(),
      
      // Swarm-specific benchmarks
      coordination: new CoordinationBenchmark(),
      load_balancing: new LoadBalancingBenchmark(),
      topology: new TopologyBenchmark(),
      fault_tolerance: new FaultToleranceBenchmark(),
      
      // Custom benchmarks
      custom: new CustomBenchmarkManager()
    };
    
    this.reporter = new BenchmarkReporter();
    this.comparator = new PerformanceComparator();
    this.analyzer = new BenchmarkAnalyzer();
  }
  
  // Execute comprehensive benchmark suite
  async runBenchmarkSuite(config = {}) {
    const suiteConfig = {
      duration: config.duration || 300000, // 5 minutes default
      iterations: config.iterations || 10,
      warmupTime: config.warmupTime || 30000, // 30 seconds
      cooldownTime: config.cooldownTime || 10000, // 10 seconds
      parallel: config.parallel || false,
      baseline: config.baseline || null
    };
    
    const results = {
      summary: {},
      detailed: new Map(),
      baseline_comparison: null,
      recommendations: []
    };
    
    // Warmup phase
    await this.warmup(suiteConfig.warmupTime);
    
    // Execute benchmarks
    if (suiteConfig.parallel) {
      results.detailed = await this.runBenchmarksParallel(suiteConfig);
    } else {
      results.detailed = await this.runBenchmarksSequential(suiteConfig);
    }
    
    // Generate summary
    results.summary = this.generateSummary(results.detailed);
    
    // Compare with baseline if provided
    if (suiteConfig.baseline) {
      results.baseline_comparison = await this.compareWithBaseline(
        results.detailed, 
        suiteConfig.baseline
      );
    }
    
    // Generate recommendations
    results.recommendations = await this.generateRecommendations(results);
    
    // Cooldown phase
    await this.cooldown(suiteConfig.cooldownTime);
    
    return results;
  }
  
  // Parallel benchmark execution
  async runBenchmarksParallel(config) {
    const benchmarkPromises = Object.entries(this.benchmarks).map(
      async ([name, benchmark]) => {
        const result = await this.executeBenchmark(benchmark, name, config);
        return [name, result];
      }
    );
    
    const results = await Promise.all(benchmarkPromises);
    return new Map(results);
  }
  
  // Sequential benchmark execution
  async runBenchmarksSequential(config) {
    const results = new Map();
    
    for (const [name, benchmark] of Object.entries(this.benchmarks)) {
      const result = await this.executeBenchmark(benchmark, name, config);
      results.set(name, result);
      
      // Brief pause between benchmarks
      await this.sleep(1000);
    }
    
    return results;
  }
}

2. Performance Regression Detection

// Advanced regression detection system
class RegressionDetector {
  constructor() {
    this.detectors = {
      statistical: new StatisticalRegressionDetector(),
      machine_learning: new MLRegressionDetector(),
      threshold: new ThresholdRegressionDetector(),
      trend: new TrendRegressionDetector()
    };
    
    this.analyzer = new RegressionAnalyzer();
    this.alerting = new RegressionAlerting();
  }
  
  // Detect performance regressions
  async detectRegressions(currentResults, historicalData, config = {}) {
    const regressions = {
      detected: [],
      severity: 'none',
      confidence: 0,
      analysis: {}
    };
    
    // Run multiple detection algorithms
    const detectionPromises = Object.entries(this.detectors).map(
      async ([method, detector]) => {
        const detection =  detector.(currentResults, historicalData, config);
         [method, detection];
      }
    );
    
     detectionResults =  .(detectionPromises);
    
    
     ( [method, detection]  detectionResults) {
       (detection.) {
        regressions..({
          method,
          ...detection
        });
      }
    }
    
    
     (regressions.. > ) {
      regressions. = .(regressions.);
      regressions. = .(regressions.);
      regressions. =  ..(regressions.);
    }
    
     regressions;
  }
  
  
   () {
    
     cusum = .(metric, historicalData);
    
    
     changePoints = .(cusum, sensitivity);
    
    
     analysis = changePoints.( ({
      : point.,
      : point.,
      : point.,
      : point.,
      : point.
    }));
    
     {
      : changePoints. > ,
      : analysis,
      : cusum.,
      : sensitivity
    };
  }
  
  
   () {
    
     model =  .(historicalData);
    
    
     anomalyScores =  model.(metrics);
    
    
     threshold = .(anomalyScores);
     regressions = anomalyScores.( score. > threshold);
    
     {
      : regressions. > ,
      : anomalyScores,
      : threshold,
      : regressions,
      : model.
    };
  }
}

3. Automated Performance Testing

// Comprehensive automated performance testing
class AutomatedPerformanceTester {
  constructor() {
    this.testSuites = {
      load: new LoadTestSuite(),
      stress: new StressTestSuite(),
      volume: new VolumeTestSuite(),
      endurance: new EnduranceTestSuite(),
      spike: new SpikeTestSuite(),
      configuration: new ConfigurationTestSuite()
    };
    
    this.scheduler = new TestScheduler();
    this.orchestrator = new TestOrchestrator();
    this.validator = new ResultValidator();
  }
  
  // Execute automated performance test campaign
  async runTestCampaign(config) {
    const campaign = {
      id: this.generateCampaignId(),
      config,
      startTime: Date.now(),
      tests: [],
      :  (),
      : 
    };
    
    
     schedule =  ..(config., config.);
    
    
     ( scheduledTest  schedule) {
       testResult =  .(scheduledTest);
      campaign..(scheduledTest);
      campaign..(scheduledTest., testResult);
      
      
       validation =  ..(testResult);
       (!validation.) {
        campaign. = {
          : ,
          : validation.,
          : scheduledTest.
        };
        ;
      }
    }
    
    
     (!campaign.) {
      campaign. =  .(campaign);
    }
    
    campaign. = .();
    campaign. = campaign. - campaign.;
    
     campaign;
  }
  
  
   () {
     loadTest = {
      : ,
      config,
      : [],
      :  (),
      : {}
    };
    
    
     rampUpResult =  .(config.);
    loadTest..({ : , : rampUpResult });
    
    
     sustainedResult =  .(config.);
    loadTest..({ : , : sustainedResult });
    
    
     rampDownResult =  .(config.);
    loadTest..({ : , : rampDownResult });
    
    
    loadTest. =  .(loadTest.);
    
     loadTest;
  }
  
  
   () {
     stressTest = {
      : ,
      config,
      : ,
      : [],
      : {}
    };
    
     currentLoad = config.;
     systemBroken = ;
    
     (!systemBroken && currentLoad <= config.) {
       testResult =  .(currentLoad, config.);
      
      stressTest..({
        : currentLoad,
        : testResult.,
        : testResult.,
        : testResult.
      });
      
      
       (.(testResult, config.)) {
        stressTest. = {
          : currentLoad,
          : testResult.,
          : .(testResult)
        };
        systemBroken = ;
      }
      
      currentLoad += config.;
    }
    
    stressTest. =  .(stressTest);
    
     stressTest;
  }
}

4. Performance Validation Framework

// Comprehensive performance validation
class PerformanceValidator {
  constructor() {
    this.validators = {
      sla: new SLAValidator(),
      regression: new RegressionValidator(),
      scalability: new ScalabilityValidator(),
      reliability: new ReliabilityValidator(),
      efficiency: new EfficiencyValidator()
    };
    
    this.thresholds = new ThresholdManager();
    this.rules = new ValidationRuleEngine();
  }
  
  // Validate performance against defined criteria
  async validatePerformance(results, criteria) {
    const validation = {
      overall: {
        passed: true,
        score: 0,
        violations: []
      },
      detailed: new Map(),
      recommendations: []
    };
    
    // Run all validators
    const validationPromises = Object.entries(.).(
       ([type, validator]) => {
         result =  validator.(results, criteria[type]);
         [type, result];
      }
    );
    
     validationResults =  .(validationPromises);
    
    
     ( [type, result]  validationResults) {
      validation..(type, result);
      
       (!result.) {
        validation.. = ;
        validation...(...result.);
      }
      
      validation.. += result. * (criteria[type]?. || );
    }
    
    
     totalWeight = .(criteria).( sum + (c. || ), );
    validation.. /= totalWeight;
    
    
    validation. =  .(validation);
    
     validation;
  }
  
  
   () {
     slaValidation = {
      : ,
      : [],
      : ,
      : {}
    };
    
    
     ( [metric, threshold]  .(slaConfig.)) {
       actualValue = .(results, metric);
       validation = .(actualValue, threshold);
      
      slaValidation.[metric] = {
        : actualValue,
        : threshold.,
        : threshold.,
        : validation.,
        : validation.
      };
      
       (!validation.) {
        slaValidation. = ;
        slaValidation..({
          metric,
          : actualValue,
          : threshold.,
          : threshold. || 
        });
        
        
         severityMultiplier = .(threshold.);
        slaValidation. -= (validation. * severityMultiplier);
      }
    }
    
    slaValidation. = .(, slaValidation.);
    
     slaValidation;
  }
  
  
   () {
     scalabilityValidation = {
      : ,
      : [],
      : ,
      : {}
    };
    
    
     (scalabilityConfig.) {
       linearityAnalysis = .(results);
      scalabilityValidation.. = linearityAnalysis;
      
       (linearityAnalysis. < scalabilityConfig..) {
        scalabilityValidation. = ;
        scalabilityValidation..({
          : ,
          : linearityAnalysis.,
          : scalabilityConfig..
        });
      }
    }
    
    
     (scalabilityConfig.) {
       efficiencyAnalysis = .(results);
      scalabilityValidation.. = efficiencyAnalysis;
      
       (efficiencyAnalysis. < scalabilityConfig..) {
        scalabilityValidation. = ;
        scalabilityValidation..({
          : ,
          : efficiencyAnalysis.,
          : scalabilityConfig..
        });
      }
    }
    
     scalabilityValidation;
  }
}

MCP Integration Hooks

Benchmark Execution Integration

// Comprehensive MCP benchmark integration
const benchmarkIntegration = {
  // Execute performance benchmarks
  async runBenchmarks(config = {}) {
    // Run benchmark suite
    const benchmarkResult = await mcp.benchmark_run({
      suite: config.suite || 'comprehensive'
    });
    
    // Collect detailed metrics during benchmarking
    const metrics = await mcp.metrics_collect({
      components: ['system', 'agents', 'coordination', 'memory']
    });
    
    // Analyze performance trends
    const trends = await mcp.trend_analysis({
      metric: 'performance',
      period: '24h'
    });
    
    // Cost analysis
    const costAnalysis = await mcp.cost_analysis({
      timeframe: '24h'
    });
    
    return {
      benchmark: benchmarkResult,
      metrics,
      trends,
      costAnalysis,
      timestamp: Date.now()
    };
  },
  
  // Quality assessment
  async assessQuality(criteria) {
    const qualityAssessment =  mcp.({
      : ,
      : criteria || [
        ,
        ,
        ,
        ,
        
      ]
    });
    
     qualityAssessment;
  },
  
  
   () {
    
     logs =  .();
    
    
     errorAnalysis =  mcp.({
      : logs
    });
    
     errorAnalysis;
  }
};

Operational Commands

Benchmarking Commands

# Run comprehensive benchmark suite
npx claude-flow benchmark-run --suite comprehensive --duration 300

# Execute specific benchmark
npx claude-flow benchmark-run --suite throughput --iterations 10

# Compare with baseline
npx claude-flow benchmark-compare --current <results> --baseline <baseline>

# Quality assessment
npx claude-flow quality-assess --target swarm-performance --criteria throughput,latency

# Performance validation
npx claude-flow validate-performance --results <file> --criteria <file>

Regression Detection Commands

# Detect performance regressions
npx claude-flow detect-regression --current <results> --historical <data>

# Set up automated regression monitoring
npx claude-flow regression-monitor --enable --sensitivity 0.95

# Analyze error patterns
npx claude-flow error-analysis --logs <log-files>

Integration Points

With Other Optimization Agents

• Performance Monitor: Provides continuous monitoring data for benchmarking
• Load Balancer: Validates load balancing effectiveness through benchmarks
• Topology Optimizer: Tests topology configurations for optimal performance

With CI/CD Pipeline

• Automated Testing: Integrates with CI/CD for continuous performance validation
• Quality Gates: Provides pass$fail criteria for deployment decisions
• Regression Prevention: Catches performance regressions before production

Performance Benchmarks

Standard Benchmark Suite

// Comprehensive benchmark definitions
const standardBenchmarks = {
  // Throughput benchmarks
  throughput: {
    name: 'Throughput Benchmark',
    metrics: ['requests_per_second', 'tasks_per_second', 'messages_per_second'],
    duration: 300000, // 5 minutes
    warmup: 30000,    // 30 seconds
    targets: {
      requests_per_second: { min: 1000, optimal: 5000 },
      tasks_per_second: { min: 100, optimal: 500 },
      messages_per_second: { min: 10000, optimal: 50000 }
    }
  },
  
  // Latency benchmarks
  latency: {
    name: 'Latency Benchmark',
    metrics: ['p50', 'p90', 'p95', 'p99', 'max'],
    duration: 300000,
    targets: {
      p50: { max: 100 },   // 100ms
      p90: { max: 200 },   // 200ms
      : { :  },   
      : { :  },  
      : { :  }   
    }
  },
  
  
  : {
    : ,
    : [, ],
    : [, , , , , , ],
    : {
      : { :  },
      : { :  }
    }
  }
};

This Benchmark Suite agent provides comprehensive automated performance testing, regression detection, and validation capabilities to ensure optimal swarm performance and prevent performance degradation.