From 4e402e24fb1d93248dada50a7149817ee0e997d1 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Thu, 9 Apr 2026 19:40:58 -0700 Subject: [PATCH 1/2] Fix cron jobs using dev env instead of test env in CI workflows The custom-base-port and db-migration-backwards-compatibility workflows were running cron jobs with `with-env:dev` instead of `with-env:test`, causing ClickHouse sync mismatches in verify-data-integrity. --- .github/workflows/db-migration-backwards-compatibility.yaml | 4 ++-- .github/workflows/e2e-custom-base-port-api-tests.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/db-migration-backwards-compatibility.yaml b/.github/workflows/db-migration-backwards-compatibility.yaml index 19b6241d16..122e5d6638 100644 --- a/.github/workflows/db-migration-backwards-compatibility.yaml +++ b/.github/workflows/db-migration-backwards-compatibility.yaml @@ -200,7 +200,7 @@ jobs: uses: JarvusInnovations/background-action@v1.0.7 if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }} with: - run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream & + run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream & wait-on: | http://localhost:8102 tail: true @@ -394,7 +394,7 @@ jobs: uses: JarvusInnovations/background-action@v1.0.7 if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }} with: - run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream & + run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream & wait-on: | http://localhost:8102 tail: true diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml index 66517efc4b..dc0a7f4c85 100644 --- a/.github/workflows/e2e-custom-base-port-api-tests.yaml +++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml @@ -145,7 +145,7 @@ jobs: - name: Start run-cron-jobs in background uses: JarvusInnovations/background-action@v1.0.7 with: - run: pnpm -C apps/backend run run-cron-jobs --log-order=stream & + run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream & wait-on: | http://localhost:6702 tail: true From 1d1dd0fdbf1fa5cfbcba49a565f62052f657488a Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Thu, 9 Apr 2026 19:54:47 -0700 Subject: [PATCH 2/2] Fix ClickHouse sync verification race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues: 1. verify-data-integrity runs before all QStash sync callbacks have been delivered, causing stale data in ClickHouse. Fix by calling syncExternalDatabases() directly before verifying each tenancy. 2. notification_preferences sort keys used ["id"] but the ClickHouse table has no id column — it uses (user_id, notification_category_id) as the composite key. Fix sort keys to match. --- .../verify-data-integrity/clickhouse-sync-verifier.ts | 2 +- apps/backend/scripts/verify-data-integrity/index.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts index 9eeb0104e3..dfa48f8707 100644 --- a/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts +++ b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts @@ -18,7 +18,7 @@ const SORT_KEYS = { team_invitations: ["id"], email_outboxes: ["id"], project_permissions: ["user_id", "permission_id"], - notification_preferences: ["id"], + notification_preferences: ["user_id", "notification_category_id"], refresh_tokens: ["id"], connected_accounts: ["user_id", "provider", "provider_account_id"], } satisfies Record; diff --git a/apps/backend/scripts/verify-data-integrity/index.ts b/apps/backend/scripts/verify-data-integrity/index.ts index 48cc992cce..f63ae32a7f 100644 --- a/apps/backend/scripts/verify-data-integrity/index.ts +++ b/apps/backend/scripts/verify-data-integrity/index.ts @@ -1,3 +1,4 @@ +import { syncExternalDatabases } from "@/lib/external-db-sync"; import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client"; import type { OrganizationRenderedConfig } from "@stackframe/stack-shared/dist/config/schema"; @@ -228,6 +229,10 @@ async function main() { if (!shouldSkipClickhouse && clickhouseAvailable && tenancy) { await recurse("[clickhouse sync]", async (recurse) => { + // Flush any pending ClickHouse syncs by running a direct sync before verifying. + // This avoids race conditions where QStash hasn't delivered all sync callbacks yet. + await syncExternalDatabases(tenancy); + await verifyClickhouseSync({ tenancy, projectId,