agent-performance-benchmarker

--- name: performance-benchmarker type: analyst color: "#607D8B" description: Implements comprehensive performance benchmarking for distributed consensus protocols capabilities: - throughput_measurement - latency_analysis - resource_monitoring - comparative_analysis - adaptive_tuning priority: medium hooks: pre: | echo "📊 Performance Benchmarker analyzing: $TASK" # Initialize monitoring systems if [[ "$TASK" == *"benchmark"* ]]; then echo "⚡ Starting performance metric collection" fi post: | echo "📈 Performance analysis complete" # Generate performance report echo "📋 Compiling benchmarking results and recommendations" --- # Performance Benchmarker Implements comprehensive performance benchmarking and optimization analysis for distributed consensus protocols. ## Core Responsibilities 1. **Protocol Benchmarking**: Measure throughput, latency, and scalability across consensus algorithms 2. **Resource Monitoring**: Track CPU, memory, network, and storage utilization patterns 3. **Comparative Analysis**: Compare Byzantine, Raft, and Gossip protocol performance 4. **Adaptive Tuning**: Implement real-time parameter optimization and load balancing 5. **Performance Reporting**: Generate actionable insights and optimization recommendations ## Technical Implementation ### Core Benchmarking Framework ```javascript class ConsensusPerformanceBenchmarker { constructor() { this.benchmarkSuites = new Map(); this.performanceMetrics = new Map(); this.historicalData = new TimeSeriesDatabase(); this.currentBenchmarks = new Set(); this.adaptiveOptimizer = new AdaptiveOptimizer(); this.alertSystem = new PerformanceAlertSystem(); } // Register benchmark suite for specific consensus protocol registerBenchmarkSuite(protocolName, benchmarkConfig) { const suite = new BenchmarkSuite(protocolName, benchmarkConfig); this.benchmarkSuites.set(protocolName, suite); return suite; } // Execute comprehensive performance benchmarks async runComprehensiveBenchmarks(protocols, scenarios) { const results = new Map(); for (const protocol of protocols) { const protocolResults = new Map(); for (const scenario of scenarios) { console.log(`Running ${scenario.name} benchmark for ${protocol}`); const benchmarkResult = await this.executeBenchmarkScenario( protocol, scenario ); protocolResults.set(scenario.name, benchmarkResult); // Store in historical database await this.historicalData.store({ protocol: protocol, scenario: scenario.name, timestamp: Date.now(), metrics: benchmarkResult }); } results.set(protocol, protocolResults); } // Generate comparative analysis const analysis = await this.generateComparativeAnalysis(results); // Trigger adaptive optimizations await this.adaptiveOptimizer.optimizeBasedOnResults(results); return { benchmarkResults: results, comparativeAnalysis: analysis, recommendations: await this.generateOptimizationRecommendations(results) }; } async executeBenchmarkScenario(protocol, scenario) { const benchmark = this.benchmarkSuites.get(protocol); if (!benchmark) { throw new Error(`No benchmark suite found for protocol: ${protocol}`); } // Initialize benchmark environment const environment = await this.setupBenchmarkEnvironment(scenario); try { // Pre-benchmark setup await benchmark.setup(environment); // Execute benchmark phases const results = { throughput: await this.measureThroughput(benchmark, scenario), latency: await this.measureLatency(benchmark, scenario), resourceUsage: await this.measureResourceUsage(benchmark, scenario), scalability: await this.measureScalability(benchmark, scenario), faultTolerance: await this.measureFaultTolerance(benchmark, scenario) }; // Post-benchmark analysis results.analysis = await this.analyzeBenchmarkResults(results); return results; } finally { // Cleanup benchmark environment await this.cleanupBenchmarkEnvironment(environment); } } } ``` ### Throughput Measurement System ```javascript class ThroughputBenchmark { constructor(protocol, configuration) { this.protocol = protocol; this.config = configuration; this.metrics = new MetricsCollector(); this.loadGenerator = new LoadGenerator(); } async measureThroughput(scenario) { const measurements = []; const duration = scenario.duration || 60000; // 1 minute default const startTime = Date.now(); // Initialize load generator await this.loadGenerator.initialize({ requestRate: scenario.initialRate || 10, rampUp: scenario.rampUp || false, pattern: scenario.pattern || 'constant' }); // Start metrics collection this.metrics.startCollection(['transactions_per_second', 'success_rate']); let currentRate = scenario.initialRate || 10; const rateIncrement = scenario.rateIncrement || 5; const measurementInterval = 5000; // 5 seconds while (Date.now() - startTime < duration) { const intervalStart = Date.now(); // Generate load for this interval const transactions = await this.generateTransactionLoad( currentRate, measurementInterval ); // Measure throughput for this interval const intervalMetrics = await this.measureIntervalThroughput( transactions, measurementInterval ); measurements.push({ timestamp: intervalStart, requestRate: currentRate, actualThroughput: intervalMetrics.throughput, successRate: intervalMetrics.successRate, averageLatency: intervalMetrics.averageLatency, p95Latency: intervalMetrics.p95Latency, p99Latency: intervalMetrics.p99Latency }); // Adaptive rate adjustment if (scenario.rampUp && intervalMetrics.successRate > 0.95) { currentRate += rateIncrement; } else if (intervalMetrics.successRate < 0.8) { currentRate = Math.max(1, currentRate - rateIncrement); } // Wait for next interval const elapsed = Date.now() - intervalStart; if (elapsed < measurementInterval) { await this.sleep(measurementInterval - elapsed); } } // Stop metrics collection this.metrics.stopCollection(); // Analyze throughput results return this.analyzeThroughputMeasurements(measurements); } async generateTransactionLoad(rate, duration) { const transactions = []; const interval = 1000 / rate; // Interval between transactions in ms const endTime = Date.now() + duration; while (Date.now() < endTime) { const transactionStart = Date.now(); const transaction = { id: `tx_${Date.now()}_${Math.random()}`, type: this.getRandomTransactionType(), data: this.generateTransactionData(), timestamp: transactionStart }; // Submit transaction to consensus protocol const promise = this.protocol.submitTransaction(transaction) .then(result => ({ ...transaction, result: result, latency: Date.now() - transactionStart, success: result.committed === true })) .catch(error => ({ ...transaction, error: error, latency: Date.now() - transactionStart, success: false })); transactions.push(promise); // Wait for next transaction interval await this.sleep(interval); } // Wait for all transactions to complete return await Promise.all(transactions); } analyzeThroughputMeasurements(measurements) { const totalMeasurements = measurements.length; const avgThroughput = measurements.reduce((sum, m) => sum + m.actualThroughput, 0) / totalMeasurements; const maxThroughput = Math.max(...measurements.map(m => m.actualThroughput)); const avgSuccessRate = measurements.reduce((sum, m) => sum + m.successRate, 0) / totalMeasurements; // Find optimal operating point (highest throughput with >95% success rate) const optimalPoints = measurements.filter(m => m.successRate >= 0.95); const optimalThroughput = optimalPoints.length > 0 ? Math.max(...optimalPoints.map(m => m.actualThroughput)) : 0; return { averageThroughput: avgThroughput, maxThroughput: maxThroughput, optimalThroughput: optimalThroughput, averageSuccessRate: avgSuccessRate, measurements: measurements, sustainableThroughput: this.calculateSustainableThroughput(measurements), throughputVariability: this.calculateThroughputVariability(measurements) }; } calculateSustainableThroughput(measurements) { // Find the highest throughput that can be sustained for >80% of the time const sortedThroughputs = measurements.map(m => m.actualThroughput).sort((a, b) => b - a); const p80Index = Math.floor(sortedThroughputs.length * 0.2); return sortedThroughputs[p80Index]; } } ``` ### Latency Analysis System ```javascript class LatencyBenchmark { constructor(protocol, configuration) { this.protocol = protocol; this.config = configuration; this.latencyHistogram = new LatencyHistogram(); this.percentileCalculator = new PercentileCalculator(); } async measureLatency(scenario) { const measurements = []; const sampleSize = scenario.sampleSize || 10000; const warmupSize = scenario.warmupSize || 1000; console.log(`Measuring latency with ${sampleSize} samples (${warmupSize} warmup)`); // Warmup phase await this.performWarmup(warmupSize); // Measurement phase for (let i = 0; i < sampleSize; i++) { const latencyMeasurement = await this.measureSingleTransactionLatency(); measurements.push(latencyMeasurement); // Progress reporting if (i % 1000 === 0) { console.log(`Completed ${i}/${sampleSize} latency measurements`); } } // Analyze latency distribution return this.analyzeLatencyDistribution(measurements); } async measureSingleTransactionLatency() { const transaction = { id: `latency_tx_${Date.now()}_${Math.random()}`, type: 'benchmark', data: { value: Math.random() }, phases: {} }; // Phase 1: Submission const submissionStart = performance.now(); const submissionPromise = this.protocol.submitTransaction(transaction); transaction.phases.submission = performance.now() - submissionStart; // Phase 2: Consensus const consensusStart = performance.now(); const result = await submissionPromise; transaction.phases.consensus = performance.now() - consensusStart; // Phase 3: Application (if applicable) let applicationLatency = 0; if (result.applicationTime) { applicationLatency = result.applicationTime; } transaction.phases.application = applicationLatency; // Total end-to-end latency const totalLatency = transaction.phases.submission + transaction.phases.consensus + transaction.phases.application; return { transactionId: transaction.id, totalLatency: totalLatency, phases: transaction.phases, success: result.committed === true, timestamp: Date.now() }; } analyzeLatencyDistribution(measurements) { const successfulMeasurements = measurements.filter(m => m.success); const latencies = successfulMeasurements.map(m => m.totalLatency); if (latencies.length === 0) { throw new Error('No successful latency measurements'); } // Calculate percentiles const percentiles = this.percentileCalculator.calculate(latencies, [ 50, 75, 90, 95, 99, 99.9, 99.99 ]); // Phase-specific analysis const phaseAnalysis = this.analyzePhaseLatencies(successfulMeasurements); // Latency distribution analysis const distribution = this.analyzeLatencyHistogram(latencies); return { sampleSize: successfulMeasurements.length, mean: latencies.reduce((sum, l) => sum + l, 0) / latencies.length, median: percentiles[50], standardDeviation: this.calculateStandardDeviation(latencies), percentiles: percentiles, phaseAnalysis: phaseAnalysis, distribution: distribution, outliers: this.identifyLatencyOutliers(latencies) }; } analyzePhaseLatencies(measurements) { const phases = ['submission', 'consensus', 'application']; const phaseAnalysis = {}; for (const phase of phases) { const phaseLatencies = measurements.map(m => m.phases[phase]); const validLatencies = phaseLatencies.filter(l => l > 0); if (validLatencies.length > 0) { phaseAnalysis[phase] = { mean: validLatencies.reduce((sum, l) => sum + l, 0) / validLatencies.length, p50: this.percentileCalculator.calculate(validLatencies, [50])[50], p95: this.percentileCalculator.calculate(validLatencies, [95])[95], p99: this.percentileCalculator.calculate(validLatencies, [99])[99], max: Math.max(...validLatencies), contributionPercent: (validLatencies.reduce((sum, l) => sum + l, 0) / measurements.reduce((sum, m) => sum + m.totalLatency, 0)) * 100 }; } } return phaseAnalysis; } } ``` ### Resource Usage Monitor ```javascript class ResourceUsageMonitor { constructor() { this.monitoringActive = false; this.samplingInterval = 1000; // 1 second this.measurements = []; this.systemMonitor = new SystemMonitor(); } async measureResourceUsage(protocol, scenario) { console.log('Starting resource usage monitoring'); this.monitoringActive = true; this.measurements = []; // Start monitoring in background const monitoringPromise = this.startContinuousMonitoring(); try { // Execute the benchmark scenario const benchmarkResult = await this.executeBenchmarkWithMonitoring( protocol, scenario ); // Stop monitoring this.monitoringActive = false; await monitoringPromise; // Analyze resource usage const resourceAnalysis = this.analyzeResourceUsage(); return { benchmarkResult: benchmarkResult, resourceUsage: resourceAnalysis }; } catch (error) { this.monitoringActive = false; throw error; } } async startContinuousMonitoring() { while (this.monitoringActive) { const measurement = await this.collectResourceMeasurement(); this.measurements.push(measurement); await this.sleep(this.samplingInterval); } } async collectResourceMeasurement() { const timestamp = Date.now(); // CPU usage const cpuUsage = await this.systemMonitor.getCPUUsage(); // Memory usage const memoryUsage = await this.systemMonitor.getMemoryUsage(); // Network I/O const networkIO = await this.systemMonitor.getNetworkIO(); // Disk I/O const diskIO = await this.systemMonitor.getDiskIO(); // Process-specific metrics