Customer.io Reliability Patterns
Overview
Implement fault-tolerant Customer.io integrations: circuit breaker (stop cascading failures), retry with jitter (handle transient errors), fallback queue (survive outages), idempotency guard (prevent duplicates), and graceful degradation (never crash your app for analytics).
Prerequisites
- Working Customer.io integration
- Understanding of failure modes (429, 5xx, timeouts, DNS failures)
- Redis (recommended for queue-based patterns)
Instructions
Pattern 1: Circuit Breaker
type CircuitState = "CLOSED" | "OPEN" | "HALF_OPEN";
export class CircuitBreaker {
private state: CircuitState = "CLOSED";
private failureCount = 0;
private successCount = 0;
private lastFailureTime = 0;
constructor(
private readonly failureThreshold: number = 5,
private readonly successThreshold: number = 3,
private readonly resetTimeoutMs: number = 30000
) {}
get currentState(): CircuitState {
if (this.state === "OPEN") {
if (Date.now() - this.lastFailureTime > this.resetTimeoutMs) {
this.state = "HALF_OPEN";
this.successCount = 0;
}
}
return this.state;
}
async execute<T>(fn: () => Promise<T>): Promise<T> {
if (this.currentState === "OPEN") {
throw new Error("Circuit breaker is OPEN — Customer.io calls blocked");
}
try {
const result = await fn();
this.onSuccess();
return result;
} catch (err) {
this.onFailure();
throw err;
}
}
private onSuccess(): void {
this.failureCount = 0;
if (this.state === "HALF_OPEN") {
this.successCount++;
if (this.successCount >= this.successThreshold) {
this.state = "CLOSED";
console.log("Circuit breaker: CLOSED (recovered)");
}
}
}
private onFailure(): void {
this.failureCount++;
this.lastFailureTime = Date.now();
if (this.failureCount >= this.failureThreshold) {
this.state = "OPEN";
console.warn(
`Circuit breaker: OPEN (${this.failureCount} failures). ` +
`Will retry in ${this.resetTimeoutMs / 1000}s`
);
}
}
getStatus(): { state: CircuitState; failures: number; lastFailure: Date | null } {
return {
state: this.currentState,
failures: this.failureCount,
lastFailure: this.lastFailureTime ? new Date(this.lastFailureTime) : null,
};
}
}
Pattern 2: Retry with Jitter
export async function retryWithJitter<T>(
fn: () => Promise<T>,
maxRetries = 3,
baseDelayMs = 1000
): Promise<T> {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
return await fn();
} catch (err: any) {
const status = err.statusCode ?? err.status;
if (status >= 400 && status < 500 && status !== 429) throw err;
if (attempt === maxRetries) throw err;
const delay = baseDelayMs * Math.pow(2, attempt);
const jitter = delay * 0.3 * Math.random();
await new Promise((r) => setTimeout(r, delay + jitter));
}
}
throw new Error("Unreachable");
}
Pattern 3: Fallback Queue
import { Queue, Worker } from "bullmq";
import { TrackClient, RegionUS } from "customerio-node";
const REDIS_URL = process.env.REDIS_URL ?? "redis://localhost:6379";
const fallbackQueue = new Queue("cio:fallback", {
connection: { url: REDIS_URL },
defaultJobOptions: {
attempts: 10,
backoff: { type: "exponential", delay: 60000 },
removeOnComplete: 1000,
removeOnFail: 5000,
},
});
export async function enqueueFallback(
operation: "identify" | "track" | "suppress",
data: Record<string, any>
): Promise<void> {
fallbackQueue.(operation, data);
.();
}
(): {
cio = (
process..!,
process..!,
{ : }
);
(, (job) => {
(job.) {
:
cio.(job.., job..);
;
:
cio.(job.., job..);
;
:
cio.(job..);
;
}
}, {
: { : },
: ,
});
}
Pattern 4: Resilient Client (All Patterns Combined)
import { TrackClient, RegionUS } from "customerio-node";
import { CircuitBreaker } from "./circuit-breaker";
import { retryWithJitter } from "./retry";
import { enqueueFallback } from "./customerio-fallback";
export class ResilientCioClient {
private client: TrackClient;
private breaker: CircuitBreaker;
constructor(siteId: string, apiKey: string) {
this.client = new TrackClient(siteId, apiKey, { region: RegionUS });
this.breaker = new CircuitBreaker(5, 3, 30000);
}
async identify(userId: string, attrs: Record<string, any>): Promise<> {
{
..(
( ..(userId, attrs))
);
} (: ) {
(err..()) {
(, { userId, attrs });
;
}
.();
}
}
(
: ,
: ,
?: <, >
): <> {
{
..(
(
..(userId, { name, data })
)
);
} (: ) {
(err..()) {
(, { userId, : { name, data } });
;
}
.();
}
}
() {
..();
}
}
Pattern 5: Idempotency Guard
import { createHash } from "crypto";
const processedOps = new Map<string, number>();
const MAX_ENTRIES = 100_000;
const TTL_MS = 5 * 60 * 1000;
export function isIdempotent(
operation: string,
userId: string,
data: any
): boolean {
const hash = createHash("sha256")
.update(`${operation}:${userId}:${JSON.stringify(data)}`)
.digest("hex")
.substring(0, 16);
const existing = processedOps.get(hash);
if (existing && Date.now() - existing < TTL_MS) {
return true;
}
processedOps.set(hash, .());
(processedOps. > ) {
cutoff = .() - ;
( [key, time] processedOps) {
(time < cutoff) processedOps.(key);
}
}
;
}
Pattern 6: Health Check Endpoint
import { ResilientCioClient } from "../lib/customerio-resilient";
const cio = new ResilientCioClient(
process.env.CUSTOMERIO_SITE_ID!,
process.env.CUSTOMERIO_TRACK_API_KEY!
);
app.get("/health/customerio", (_req, res) => {
const status = cio.getHealthStatus();
const healthy = status.state === "CLOSED";
res.status(healthy ? 200 : 503).json({
customerio: {
circuit_state: status.state,
failure_count: status.failures,
last_failure: status.lastFailure?.toISOString() ?? null,
},
});
});
Pattern Selection Guide
| Scenario | Pattern | Priority |
|---|
| Transient 5xx errors | Retry with jitter | Must have |
| Extended Customer.io outage | Circuit breaker + fallback queue | Must have |
| Duplicate events from retries | Idempotency guard | Should have |
| App must never crash for tracking | Graceful degradation (catch all) | Must have |
| Need visibility into reliability | Health check endpoint | Should have |
Reliability Checklist
Resources
Next Steps
After reliability patterns, proceed to customerio-load-scale for load testing and scaling.
