From 0c088962934cdf829c9e6b282d2fa5a4e07723a3 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Wed, 26 Nov 2025 09:47:41 -0800 Subject: [PATCH 01/25] clickhouse setup --- apps/backend/.env.development | 8 + apps/backend/package.json | 1 + apps/backend/scripts/clickhouse-migrations.ts | 25 ++ apps/backend/scripts/db-migrations.ts | 3 + .../app/api/latest/analytics/query/route.tsx | 74 ++++ apps/backend/src/lib/clickhouse.tsx | 56 +++ apps/dev-launchpad/public/index.html | 10 + .../endpoints/api/v1/analytics-query.test.ts | 387 ++++++++++++++++++ apps/e2e/tests/snapshot-serializer.ts | 2 + docker/dependencies/docker.compose.yaml | 20 + packages/stack-shared/src/known-errors.tsx | 24 ++ pnpm-lock.yaml | 16 + 12 files changed, 626 insertions(+) create mode 100644 apps/backend/scripts/clickhouse-migrations.ts create mode 100644 apps/backend/src/app/api/latest/analytics/query/route.tsx create mode 100644 apps/backend/src/lib/clickhouse.tsx create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts diff --git a/apps/backend/.env.development b/apps/backend/.env.development index 0b57a9c090..312e599c26 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -70,3 +70,11 @@ STACK_QSTASH_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}25 STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0= STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs + +# Clickhouse +CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}33 +CLICKHOUSE_ADMIN_USER=stackframe +CLICKHOUSE_ADMIN_PASSWORD=PASSWORD-PLACEHOLDER--9gKyMxJeMx +CLICKHOUSE_EXTERNAL_USER=limited_user +CLICKHOUSE_EXTERNAL_PASSWORD=PASSWORD-PLACEHOLDER--EZeHscBMzE +CLICKHOUSE_DATABASE=analytics diff --git a/apps/backend/package.json b/apps/backend/package.json index dc06f209a3..abad48d881 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -43,6 +43,7 @@ "dependencies": { "@ai-sdk/openai": "^1.3.23", "@aws-sdk/client-s3": "^3.855.0", + "@clickhouse/client": "^1.14.0", "@next/bundle-analyzer": "15.2.3", "@node-oauth/oauth2-server": "^5.1.0", "@opentelemetry/api": "^1.9.0", diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts new file mode 100644 index 0000000000..99b9b1b5e3 --- /dev/null +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -0,0 +1,25 @@ +import { createClickhouseClient } from "@/lib/clickhouse"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; + +export async function runClickhouseMigrations() { + console.log("Running Clickhouse migrations..."); + const client = createClickhouseClient("admin"); + const clickhouseExternalPassword = getEnvVariable("CLICKHOUSE_EXTERNAL_PASSWORD"); + // todo: create migration files + await client.exec({ + query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH plaintext_password BY {clickhouseExternalPassword:String}", + query_params: { clickhouseExternalPassword }, + }); + const queries = [ + "GRANT SELECT ON analytics.allowed_table1 TO limited_user;", + "REVOKE ALL ON system.* FROM limited_user;", + "REVOKE CREATE, ALTER, DROP, INSERT ON *.* FROM limited_user;" + ]; + for (const query of queries) { + console.log(query); + await client.exec({ query }); + } + console.log("Clickhouse migrations complete"); + await client.close(); +} + diff --git a/apps/backend/scripts/db-migrations.ts b/apps/backend/scripts/db-migrations.ts index d833ed688c..a37e5ed796 100644 --- a/apps/backend/scripts/db-migrations.ts +++ b/apps/backend/scripts/db-migrations.ts @@ -8,6 +8,7 @@ import path from "path"; import * as readline from "readline"; import { seed } from "../prisma/seed"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { runClickhouseMigrations } from "./clickhouse-migrations"; const dropSchema = async () => { await globalPrismaClient.$executeRaw(Prisma.sql`DROP SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} CASCADE`); @@ -151,6 +152,8 @@ const migrate = async (selectedMigrationFiles?: { migrationName: string, sql: st console.log('='.repeat(60) + '\n'); + await runClickhouseMigrations(); + return result; }; diff --git a/apps/backend/src/app/api/latest/analytics/query/route.tsx b/apps/backend/src/app/api/latest/analytics/query/route.tsx new file mode 100644 index 0000000000..ab372f6bb6 --- /dev/null +++ b/apps/backend/src/app/api/latest/analytics/query/route.tsx @@ -0,0 +1,74 @@ +import { createClickhouseClient, getQueryTimingStats } from "@/lib/clickhouse"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { adaptSchema, jsonSchema, serverOrHigherAuthTypeSchema, yupBoolean, yupMixed, yupNumber, yupObject, yupRecord, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { randomUUID } from "crypto"; + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Execute analytics query", + description: "Execute a ClickHouse query against the analytics database", + tags: ["Analytics"], + }, + request: yupObject({ + auth: yupObject({ + type: serverOrHigherAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + body: yupObject({ + include_all_branches: yupBoolean().default(false), + query: yupString().defined().nonEmpty(), + params: yupRecord(yupString().defined(), yupMixed().defined()).default({}), + timeout_ms: yupNumber().integer().min(1).max(60000).default(1000), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + result: jsonSchema.defined(), + stats: yupObject({ + cpu_time: yupNumber().defined(), + wall_clock_time: yupNumber().defined(), + }).defined(), + }).defined(), + }), + async handler({ body, auth }) { + const client = createClickhouseClient("external", body.timeout_ms); + const queryId = randomUUID(); + const resultSet = await Result.fromPromise(client.query({ + query: body.query, + query_id: queryId, + query_params: body.params, + clickhouse_settings: { + SQL_tenancy_id: auth.tenancy.id, + }, + format: "JSONEachRow", + })); + + if (resultSet.status === "error") { + const message = resultSet.error instanceof Error ? resultSet.error.message : null; + if (message === "Timeout error.") { + throw new KnownErrors.AnalyticsQueryTimeout(body.timeout_ms); + } + throw new KnownErrors.AnalyticsQueryError(message ?? "Unknown error"); + } + + const rows = await resultSet.data.json[]>(); + const stats = await getQueryTimingStats(client, queryId); + + return { + statusCode: 200, + bodyType: "json", + body: { + result: rows, + stats: { + cpu_time: stats.cpu_time_ms, + wall_clock_time: stats.wall_clock_time_ms, + }, + }, + }; + }, +}); + diff --git a/apps/backend/src/lib/clickhouse.tsx b/apps/backend/src/lib/clickhouse.tsx new file mode 100644 index 0000000000..86184f439d --- /dev/null +++ b/apps/backend/src/lib/clickhouse.tsx @@ -0,0 +1,56 @@ +import { createClient, type ClickHouseClient } from "@clickhouse/client"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; + +const clickhouseUrl = getEnvVariable("CLICKHOUSE_URL"); +const clickhouseAdminUser = getEnvVariable("CLICKHOUSE_ADMIN_USER"); +const clickhouseExternalUser = getEnvVariable("CLICKHOUSE_EXTERNAL_USER"); +const clickhouseAdminPassword = getEnvVariable("CLICKHOUSE_ADMIN_PASSWORD"); +const clickhouseExternalPassword = getEnvVariable("CLICKHOUSE_EXTERNAL_PASSWORD"); +const clickhouseDatabase = getEnvVariable("CLICKHOUSE_DATABASE"); + +export function createClickhouseClient(authType: "admin" | "external", timeoutMs?: number) { + return createClient({ + url: clickhouseUrl, + username: authType === "admin" ? clickhouseAdminUser : clickhouseExternalUser, + password: authType === "admin" ? clickhouseAdminPassword : clickhouseExternalPassword, + database: clickhouseDatabase, + request_timeout: timeoutMs, + }); +} + + +export const getQueryTimingStats = async (client: ClickHouseClient, queryId: string) => { + // Flush logs to ensure system.query_log has latest query result. + // Todo: for performance we should instead poll for this row to become available asynchronously after returning result. Flushed every 7.5 seconds by default + await client.exec({ + query: "SYSTEM FLUSH LOGS", + auth: { + username: clickhouseAdminUser, + password: clickhouseAdminPassword, + }, + }); + const profile = await client.query({ + query: ` + SELECT + ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms, + ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms + FROM system.query_log + WHERE query_id = {query_id:String} AND type = 'QueryFinish' + ORDER BY event_time DESC + LIMIT 1 + `, + query_params: { query_id: queryId }, + auth: { + username: clickhouseAdminUser, + password: clickhouseAdminPassword, + }, + format: "JSON", + }); + + const stats = await profile.json<{ + cpu_time_ms: number, + wall_clock_time_ms: number, + }>(); + return stats.data[0]; +}; + diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html index 8890d533fd..3716ac5207 100644 --- a/apps/dev-launchpad/public/index.html +++ b/apps/dev-launchpad/public/index.html @@ -129,6 +129,7 @@

Background services

{ suffix: "22", label: "Freestyle mock" }, { suffix: "24", label: "LocalStack Gateway (AWS mock)" }, { suffix: "25", label: "QStash mock" }, + { suffix: "33", label: "ClickHouse" }, { range: ["50", "99"], label: "Reserved for LocalStack (external services)" }, ]; @@ -300,6 +301,15 @@

Background services

"React example", ], }, + { + name: "ClickHouse", + portSuffix: "33", + description: [ + "ClickHouse", + ], + importance: 1, + img: "https://thumbs.bfldr.com/at/qkjfv3nvsv4rbwn94zmtb4t/v/1197417003?expiry=1764357242&fit=bounds&height=800&sig=NjEwNzA0OThjZmJiZDQzZmUwNjIyY2UxYzZiNGYxNmQ3NjJiYjc0OA%3D%3D&width=1100", + }, { name: "MCPJam Inspector", portSuffix: "26", diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts new file mode 100644 index 0000000000..6f8c092d43 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -0,0 +1,387 @@ +import { it } from "../../../../helpers"; +import { Auth, Project, niceBackendFetch } from "../../../backend-helpers"; + +it("can execute a basic query with server access", async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + await Auth.Otp.signIn(); + + const response = await niceBackendFetch("/api/v1/analytics/query", { + method: "POST", + accessType: "server", + body: { + query: "SELECT 1 as value", + }, + }); + + expect(response).toMatchInlineSnapshot(` + NiceResponse { + "status": 200, + "body": { + "result": [{ "value": 1 }], + "stats": { + "cpu_time": , + "wall_clock_time": , + }, + }, + "headers": Headers {