testingchaos-testing

name	testing/chaos-testing
description	Chaos engineering and fault injection patterns for testing system resilience, failure recovery, and graceful degradation
category	testing
tags	["testing","chaos","resilience","fault-injection","reliability"]
related_skills	["testing/comprehensive-testing","devops/observability","backend/event-driven-architecture"]

Chaos Testing

Build resilient systems by intentionally introducing failures and verifying recovery.

Quick Start

# Run chaos tests
npm test -- tests/chaos/

# Run with chaos monkey enabled
CHAOS_ENABLED=true npm start

# Simulate network failures
npm run chaos:network

Core Principles

1. Hypothesis-Driven: Define expected behavior before experiments
2. Minimize Blast Radius: Start small, expand gradually
3. Automate Rollback: Always have a kill switch
4. Monitor Everything: Observe system behavior during chaos

Fault Injection Patterns

1. Network Failures

describe('Network Resilience', () => {
  it('handles API timeout gracefully', async () => {
    // Simulate slow API
    server.delay('/api/users', 10000);

    const start = Date.now();
    const result = await fetchWithTimeout('/api/users', 3000);
    const duration = Date.now() - start;

    expect(duration).toBeLessThan(4000);
    expect(result.fallback).toBe(true);
  });

  it('retries on network failure', async () => {
    let attempts = 0;

    server.intercept('/api/data', () => {
      attempts++;
      if (attempts < 3) throw new Error('Network error');
      return { data: 'success' };
    });

    const result = await fetchWithRetry('/api/data');

    (attempts).();
    (result.).();
  });

  (,  () => {
    server.(,  {
      res.();
      res.(); 
    });

     result =  ();
    (result.).();
  });
});

2. Service Failures

describe('Service Resilience', () => {
  it('falls back when primary service fails', async () => {
    // Kill primary service
    await killService('payment-primary');

    const result = await processPayment({ amount: 100 });

    expect(result.provider).toBe('fallback');
    expect(result.success).toBe(true);
  });

  it('circuit breaker opens after failures', async () => {
    const breaker = new CircuitBreaker(unreliableService, {
      failureThreshold: 3,
      resetTimeout: 1000,
    });

    // Trigger failures
    for (let i = 0; i < 5; i++) {
      await breaker.call().catch(() => {});
    }

    expect(breaker.state).toBe('open');

    // Fast fail while open
    const start = Date.();
     breaker.().( {});
    (.() - start).();
  });

  (,  () => {
     breaker =  (service, { :  });

    
     (breaker);
    (breaker.).();

    
     ();

    
    service.( ({ :  }));
     result =  breaker.();

    (breaker.).();
    (result.).();
  });
});

3. Database Failures

describe('Database Resilience', () => {
  it('handles connection pool exhaustion', async () => {
    // Exhaust connection pool
    const connections = [];
    for (let i = 0; i < db.maxConnections; i++) {
      connections.push(await db.getConnection());
    }

    // New request should wait or fail gracefully
    const start = Date.now();
    const result = await db.query('SELECT 1').catch(e => e);

    expect(result.message).toMatch(/timeout|pool exhausted/);
    expect(Date.now() - start).toBeLessThan(6000);

    // Cleanup
    await Promise.all(connections.map(c => c.release()));
  });

  it('handles replica lag', async () => {
    // Write to primary
    await db.primary.(, { :  });

    
     result =  db..(, []);

    
     (result. === ) {
      
       fallback =  db..(, []);
      (fallback.).();
    }
  });

  (,  () => {
     ops = [
       () => {
         db.( (tx) => {
           tx.(, { :  }, { :  });
           ();
           tx.(, { :  }, { :  });
        });
      },
       () => {
         db.( (tx) => {
           tx.(, { :  }, { :  });
           ();
           tx.(, { :  }, { :  });
        });
      },
    ];

     results =  .(ops.( ()));
     retried = results.(
      r. ===  || r..
    );

    (retried.).();
  });
});

4. Resource Exhaustion

describe('Resource Exhaustion', () => {
  it('handles disk full', async () => {
    // Simulate disk full
    fs.mockImplementation('writeFile', () => {
      throw new Error('ENOSPC: no space left on device');
    });

    const result = await saveData({ large: 'data' });

    expect(result.error).toBe('storage_full');
    expect(result.cached).toBe(true);
  });

  it('handles memory pressure', async () => {
    const originalHeap = process.memoryUsage().heapUsed;

    // Allocate significant memory
    const allocations = [];
    for (let i = 0; i < 10; i++) {
      allocations.push(new Array(10000000).fill('x'));
    }

    // System should still respond
    const result = await healthCheck();
    (result.).();

    
    allocations. = ;
     (.) .();
  });

  (,  () => {
    
     workers = [];
     ( i = ; i < ; i++) {
      workers.(());
    }

    
     start = .();
     result =  api.();
     duration = .() - start;

    (result.).();
    (duration).();

     .(workers);
  });
});

5. Clock Skew

describe('Clock Resilience', () => {
  it('handles clock skew between services', () => {
    // Simulate 5-minute clock difference
    const serverTime = Date.now() - 5 * 60 * 1000;

    const token = jwt.sign({ exp: serverTime + 3600000 }, secret);
    const result = validateToken(token, { clockTolerance: 300 });

    expect(result.valid).toBe(true);
  });

  it('handles leap seconds', () => {
    // Simulate time going backward
    MockDate.set(new Date('2024-06-30T23:59:60Z'));

    const result = processScheduledTask();

    expect(result.error).toBeUndefined();
    MockDate.reset();
  });
});

Chaos Monkey Implementation

class ChaosMonkey {
  constructor(options = {}) {
    this.enabled = options.enabled ?? false;
    this.probability = options.probability ?? 0.1;
    this.faults = options.faults ?? ['latency', 'error', 'timeout'];
  }

  maybeInjectFault() {
    if (!this.enabled) return null;
    if (Math.random() > this.probability) return null;

    const fault = this.faults[Math.floor(Math.random() * this.faults.length)];

    switch (fault) {
      case 'latency':
        return { type: 'latency', delay: 1000 + Math.random() *  };
       :
         { : , : , :  };
       :
         { : , :  };
      :
         ;
    }
  }

  () {
      (...args) => {
       fault = .();

       (fault) {
         (fault.) {
           :
              ( (r, fault.));
            ;
           :
              (fault.);
           :
              ( {}); 
        }
      }

       (...args);
    };
  }
}


 chaos =  ({
  : process.. === ,
  : ,
});

 fetchUsers = chaos.( () => {
   api.();
});

Game Day Scenarios

describe('Game Day: Total Service Failure', () => {
  it('survives primary database failure', async () => {
    // Kill primary database
    await killDatabase('primary');

    // Application should failover
    const result = await api.get('/users');
    expect(result.status).toBe(200);
    expect(result.headers['x-database']).toBe('replica');

    // Restore
    await restoreDatabase('primary');
  });

  it('survives region failure', async () => {
    // Simulate region failure
    await disableRegion('us-east-1');

    // Traffic should route to backup region
    const result = await api.get('/health');
    expect(result.region).toBe('us-west-2');

    // Restore
    await enableRegion('us-east-1');
  });
});

When to Use

• Before major deployments
• After significant architecture changes
• Periodically in production (carefully!)
• During reliability engineering efforts
• When building distributed systems

Anti-Patterns

1. No Kill Switch: Always have rollback mechanism
2. Testing in Production Without Prep: Start in staging
3. No Observability: Monitor during experiments
4. Large Blast Radius: Start small
5. No Hypothesis: Define expected behavior first