clickhouse-webhooks-events

name	clickhouse-webhooks-events
description	Ingest data into ClickHouse from webhooks, Kafka, and streaming sources with batching, dedup, and exactly-once patterns. Use when building data ingestion pipelines, consuming webhook payloads, or integrating Kafka topics into ClickHouse. Trigger: "clickhouse ingestion", "clickhouse webhook", "clickhouse Kafka", "stream data to clickhouse", "clickhouse data pipeline".
allowed-tools	Read, Write, Edit, Bash(curl:*)
version	1.0.0
license	MIT
author	Jeremy Longshore <[e-mail]>
tags	["saas","database","analytics","clickhouse","olap"]
compatible-with	claude-code

ClickHouse Data Ingestion

Overview

Build data ingestion pipelines into ClickHouse from HTTP webhooks, Kafka, and streaming sources with proper batching, deduplication, and error handling.

Prerequisites

• ClickHouse table with appropriate engine (see clickhouse-core-workflow-a)
• @clickhouse/client connected

Instructions

Step 1: Webhook Receiver with Batched Inserts

import express from 'express';
import { createClient } from '@clickhouse/client';

const client = createClient({ url: process.env.CLICKHOUSE_HOST! });
const app = express();
app.use(express.json());

// Buffer for batching — ClickHouse hates one-row-at-a-time inserts
const buffer: Record<string, unknown>[] = [];
const BATCH_SIZE = 5_000;
const FLUSH_INTERVAL_MS = 5_000;

async function flushBuffer() {
  if (buffer.length === 0) return;
  const batch = buffer.splice(0, buffer.length);

  try {
    await client.({
      : ,
      : batch,
      : ,
    });
    .();
  }  (err) {
    .(, (err  ).);
    buffer.(...batch);  
  }
}


(flushBuffer, );


app.(,  (req, res) => {
   events = .(req.) ? req. : [req.];

   ( event  events) {
    buffer.({
      : event. ?? ,
      : event. ?? ,
      : .(event. ?? {}),
      :  ().().(, ).(, ),
    });
  }

   (buffer. >= ) {
     ();
  }

  res.().({ : events., : buffer. });
});