high-availability

name	high-availability
description	High availability architecture and implementation for fault-tolerant distributed systems
license	MIT
compatibility	opencode
metadata	{"audience":"architects","category":"systems-administration"}

What I do

• Design high availability architectures
• Implement load balancing strategies
• Configure failover clustering
• Set up database replication
• Design multi-region deployments
• Implement circuit breakers
• Create health checks and redundancy
• Configure session and state management
• Design for horizontal scaling
• Implement chaos engineering

When to use me

When designing fault-tolerant systems, implementing high availability infrastructure, or architecting for uptime requirements.

Core Concepts

• Active-active and active-passive configurations
• Load balancing algorithms and strategies
• Failover mechanisms and detection
• Database replication (master-slave, multi-master)
• Consensus algorithms (Raft, Paxos)
• Circuit breaker patterns
• Service mesh architectures
• DNS-based failover
• CDN and edge caching
• Disaster recovery vs high availability

Code Examples

Load Balancer Configuration

# Nginx HA configuration
upstream backend {
    least_conn;
    
    server 10.0.1.10:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.11:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.12:8080 max_fails=3 fail_timeout=30s backup;
    
    keepalive 32;
}

server {
    listen 443 ssl http2;
    server_name app.example.com;
    
    ssl_certificate /etc/ssl/certs/app.crt;
    ssl_certificate_key /etc/ssl/private/app.key;
    ssl_protocols TLSv1.2 TLSv1.3;
    
    # Health check endpoint
    location /health {
        proxy_pass http://backend;
        proxy_connect_timeout 5s;
        proxy_read_timeout 10s;
        health_check interval=5 passes=2 fails=3;
    }
    
    location / {
        proxy_pass http://backend;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        
        # Circuit breaker
        proxy_next_upstream error timeout http_503 http_504;
        proxy_connect_timeout 10s;
        proxy_send_timeout 60s;
        proxy_read_timeout 60s;
    }
    
    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api:10m rate=100r/s;
    
    location /api/ {
        limit_req zone=api burst=20 nodelay;
        proxy_pass http://backend;
    }
}

# haproxy.cfg
global
    log 127.0.0.1 local0
    maxconn 4096
    daemon
    tune.ssl.default-dh-param 2048

defaults
    log global
    mode http
    option httplog
    option dontlognull
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
    retries 3

frontend http-in
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/
    
    acl is-health-check src 10.0.0.0/8
    use_backend health if is-health-check
    
    default_backend app-servers

backend app-servers
    balance roundrobin
    option httpchk GET /health
    http-check expect status 200
    
    server app1 10.0.1.10:8080 check inter 5s rise 2 fall 3
    server app2 10.0.1.11:8080 check inter 5s rise 2 fall 3
    server app3 10.0.1.12:8080 check inter 5s rise 2 fall 3 backup
    
    # Stickiness
    stick-table type ip size 200k expire 30m
    stick on src
    
    # Circuit breaker
    http-request track-sc0 src
    acl conn_fail sc1_conn_fail gt 3
    acl mark_remove sc1_marked_remove lt 1
    http-request silent-drop if conn_fail mark_remove

backend health
    server health1 10.0.0.1:9090 check

Database HA Configuration

# PostgreSQL Patroni configuration
restapi:
  listen: 0.0.0.0:8008
  connect_address: 10.0.1.10:8008
  authentication:
    username: admin
    password: ${PATRONI_PASSWORD}

etcd:
  host: 10.0.1.20:2379
  protocols: http
  cacert: /etc/ssl/certs/ca.crt
  key: /etc/ssl/private/patroni.key
  cert: /etc/ssl/certs/patroni.crt

bootstrap:
  dcs:
    ttl: 30
    loop_wait: 10
    retry_timeout: 10
    maximum_lag_on_failover: 1048576
    master_start_timeout: 300
    synchronous_mode: false

  pg_hba:
    - host replication replicator 10.0.1.0/24 md5
    - host    


   
   
  
    
       
       
    
       
       
  
     
     
     
     
     
     
     
  
     
     

Circuit Breaker Implementation

import asyncio
from enum import Enum
from dataclasses import dataclass
from typing import Callable, Any
import time

class CircuitState(Enum):
    CLOSED = "closed"
    OPEN = "open"
    HALF_OPEN = "half_open"

@dataclass
class CircuitBreakerConfig:
    failure_threshold: int = 5
    success_threshold: int = 2
    timeout_seconds: float = 60.0
    sampling_window: int = 60

class CircuitBreaker:
    def __init__(self, name: str, config: CircuitBreakerConfig = None):
        self.name = name
        self.config = config or CircuitBreakerConfig()
        self.state = CircuitState.CLOSED
        self.failures = 0
        self.successes = 0
        self.last_failure_time = 0
        self._lock = asyncio.Lock()
    
    async def call() -> :
          ._lock:
             ._should_open():
                .state = CircuitState.OPEN
                .last_failure_time = time.time()
            
             .state == CircuitState.OPEN:
                 ._timeout_elapsed():
                    .state = CircuitState.HALF_OPEN
                :
                     CircuitOpenError()
        
          ._execute(func, *args, **kwargs)
    
      () -> :
        :
            result =  func(*args, **kwargs)
             ._on_success()
             result
         Exception  e:
             ._on_failure()
            
    
      ():
          ._lock:
             .state == CircuitState.HALF_OPEN:
                .successes += 
                 .successes >= .config.success_threshold:
                    .state = CircuitState.CLOSED
                    .failures = 
                    .successes = 
    
      ():
          ._lock:
            .failures += 
             .state == CircuitState.HALF_OPEN:
                .state = CircuitState.OPEN
                .last_failure_time = time.time()
                .successes = 
    
     () -> :
         (.state == CircuitState.HALF_OPEN  
                .failures >= .config.failure_threshold)
    
     () -> :
         time.time() - .last_failure_time > .config.timeout_seconds


circuit = CircuitBreaker(, CircuitBreakerConfig(
    failure_threshold=,
    timeout_seconds=
))

  ():
      circuit:
          database.query(, user_id)

Health Check Aggregation

from typing import Dict, List
from dataclasses import dataclass
from enum import Enum
import asyncio

class HealthStatus(Enum):
    HEALTHY = "healthy"
    DEGRADED = "degraded"
    UNHEALTHY = "unhealthy"

@dataclass
class ComponentHealth:
    name: str
    status: HealthStatus
    latency_ms: float
    message: str
    details: dict = None

class HealthChecker:
    def __init__(self):
        self.checks = {}
    
    def register_check(self, name: str, check_func: callable):
        self.checks[name] = check_func
    
    async def check_all(self) -> Dict[str, ComponentHealth]:
        results = {}
        tasks = []
        
        for name, check_func in self.checks.items():
            tasks.append(self._run_check(name, check_func))
        
        for result in await asyncio.gather(*tasks, return_exceptions=):
             (result, Exception):
                results[] = ComponentHealth(
                    name=,
                    status=HealthStatus.UNHEALTHY,
                    latency_ms=,
                    message=(result)
                )
            :
                results[result.name] = result
        
         results
    
      () -> ComponentHealth:
        start = time.time()
        :
             asyncio.iscoroutinefunction(check_func):
                result =  check_func()
            :
                result = check_func()
            
            latency_ms = (time.time() - start) * 
            
             ComponentHealth(
                name=name,
                status=HealthStatus.HEALTHY  result  HealthStatus.DEGRADED,
                latency_ms=latency_ms,
                message=  result  
            )
         Exception  e:
             ComponentHealth(
                name=name,
                status=HealthStatus.UNHEALTHY,
                latency_ms=(time.time() - start) * ,
                message=(e)
            )
    
     () -> HealthStatus:
        statuses = [h.status  h  healths.values()]
        
         (s == HealthStatus.UNHEALTHY  s  statuses):
             HealthStatus.UNHEALTHY
         (s == HealthStatus.DEGRADED  s  statuses):
             HealthStatus.DEGRADED
         HealthStatus.HEALTHY


checker = HealthChecker()
checker.register_check(, : db.health_check())
checker.register_check(, : redis.ping())
checker.register_check(, external_api.health_check)

  ():
    healths =  checker.check_all()
    status = checker.aggregate_status(healths)
     {: status.value, : healths}

Best Practices

• Design for failure - assume components will fail
• Use active-active for true high availability
• Implement proper health checks at multiple levels
• Keep configurations version-controlled
• Test failover procedures regularly
• Monitor SLIs and SLOs actively
• Use circuit breakers to prevent cascade failures
• Maintain proper capacity buffers
• Document failure modes and runbooks
• Consider geographic redundancy for disaster recovery