clickhouse new syncs and verify-data

.gitignore

@@ -52,6 +52,7 @@ dist
 .docusaurus
 .cache-loader
 **.tsbuildinfo
+implementation.generated.ts
 
 .xata*
 

apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "Team" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;
+
+-- AlterTable
+ALTER TABLE "TeamMember" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "EmailOutbox" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "SessionReplay" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "TeamMemberDirectPermission" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;
+
+-- AlterTable
+ALTER TABLE "VerificationCode" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "ProjectUserDirectPermission" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;
+
+-- AlterTable
+ALTER TABLE "UserNotificationPreference" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "ProjectUserRefreshToken" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;
+
+-- AlterTable
+ALTER TABLE "ProjectUserOAuthAccount" ADD COLUMN "sequenceId" BIGINT,
+ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true;

apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql

@@ -0,0 +1,154 @@
+-- Team indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "Team_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."Team"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("tenancyId", "sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("shouldUpdateSequenceId", "tenancyId");
+
+-- TeamMember indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("tenancyId", "sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("shouldUpdateSequenceId", "tenancyId");
+
+-- EmailOutbox indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("tenancyId", "sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("shouldUpdateSequenceId", "tenancyId");
+
+-- SessionReplay indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("tenancyId", "sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("shouldUpdateSequenceId", "tenancyId");
+
+-- TeamMemberDirectPermission indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("shouldUpdateSequenceId", "tenancyId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("tenancyId", "sequenceId");
+
+-- VerificationCode indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_shouldUpdateSequenceId_type_idx" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("shouldUpdateSequenceId", "type");
+
+-- ProjectUserDirectPermission indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("shouldUpdateSequenceId", "tenancyId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("tenancyId", "sequenceId");
+
+-- UserNotificationPreference indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("shouldUpdateSequenceId", "tenancyId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("tenancyId", "sequenceId");
+
+-- ProjectUserRefreshToken indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("shouldUpdateSequenceId", "tenancyId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("tenancyId", "sequenceId");
+
+-- ProjectUserOAuthAccount indexes
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("sequenceId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("shouldUpdateSequenceId", "tenancyId");
+
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
+-- RUN_OUTSIDE_TRANSACTION_SENTINEL
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("tenancyId", "sequenceId");

apps/backend/prisma/schema.prisma

@@ -179,11 +179,16 @@ model Team {
   serverMetadata         Json?
   profileImageUrl        String?
 
+  sequenceId             BigInt? @unique
+  shouldUpdateSequenceId Boolean @default(true)
+
   teamMembers   TeamMember[]
   projectApiKey ProjectApiKey[]
 
   @@id([tenancyId, teamId])
   @@unique([mirroredProjectId, mirroredBranchId, teamId])
+  @@index([tenancyId, sequenceId], name: "Team_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "Team_shouldUpdateSequenceId_idx")
 }
 
 // This is used for fields that are boolean but only the true value is part of a unique constraint.
@@ -205,6 +210,9 @@ model TeamMember {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
+  sequenceId             BigInt? @unique
+  shouldUpdateSequenceId Boolean @default(true)
+
   projectUser                 ProjectUser                  @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade)
   team                        Team                         @relation(fields: [tenancyId, teamId], references: [tenancyId, teamId], onDelete: Cascade)
   isSelected                  BooleanTrue?
@@ -213,6 +221,8 @@ model TeamMember {
   @@id([tenancyId, projectUserId, teamId])
   @@unique([tenancyId, projectUserId, isSelected])
   @@index([tenancyId, projectUserId, isSelected], map: "TeamMember_projectUserId_isSelected_idx")
+  @@index([tenancyId, sequenceId], name: "TeamMember_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMember_shouldUpdateSequenceId_idx")
 }
 
 model ProjectUserDirectPermission {
@@ -226,7 +236,12 @@ model ProjectUserDirectPermission {
 
   projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@unique([tenancyId, projectUserId, permissionId])
+  @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserDirectPermission_shouldUpdateSequenceId_idx")
+  @@index([tenancyId, sequenceId], name: "ProjectUserDirectPermission_tenancyId_sequenceId_idx")
 }
 
 model TeamMemberDirectPermission {
@@ -241,7 +256,12 @@ model TeamMemberDirectPermission {
 
   teamMember TeamMember @relation(fields: [tenancyId, projectUserId, teamId], references: [tenancyId, projectUserId, teamId], onDelete: Cascade)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@unique([tenancyId, projectUserId, teamId, permissionId])
+  @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMemberDirectPermission_shouldUpdateSequenceId_idx")
+  @@index([tenancyId, sequenceId], name: "TeamMemberDirectPermission_tenancyId_sequenceId_idx")
 }
 
 model ProjectUser {
@@ -343,9 +363,14 @@ model ProjectUserOAuthAccount {
   allowConnectedAccounts Boolean          @default(true)
   allowSignIn            Boolean          @default(true)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@id([tenancyId, id])
   @@unique([tenancyId, configOAuthProviderId, projectUserId, providerAccountId])
   @@index([tenancyId, projectUserId])
+  @@index([tenancyId, sequenceId], name: "ProjectUserOAuthAccount_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx")
 }
 
 model SessionReplay {
@@ -361,6 +386,9 @@ model SessionReplay {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
+  sequenceId             BigInt? @unique
+  shouldUpdateSequenceId Boolean @default(true)
+
   projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade)
   tenancy     Tenancy     @relation(fields: [tenancyId], references: [id], onDelete: Cascade)
 
@@ -371,6 +399,8 @@ model SessionReplay {
   @@index([tenancyId, lastEventAt])
   // index by updatedAt instead of lastEventAt because event timing can be spoofed
   @@index([tenancyId, refreshTokenId, updatedAt])
+  @@index([tenancyId, sequenceId], name: "SessionReplay_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "SessionReplay_shouldUpdateSequenceId_idx")
   @@map("SessionReplay")
 }
 
@@ -612,7 +642,12 @@ model ProjectUserRefreshToken {
   expiresAt       DateTime?
   isImpersonation Boolean   @default(false)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@id([tenancyId, id])
+  @@index([tenancyId, sequenceId], name: "ProjectUserRefreshToken_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserRefreshToken_shouldUpdateSequenceId_idx")
 }
 
 model ProjectUserAuthorizationCode {
@@ -655,9 +690,13 @@ model VerificationCode {
   data         Json
   attemptCount Int                  @default(0)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@id([projectId, branchId, id])
   @@unique([projectId, branchId, code])
   @@index([data(ops: JsonbPathOps)], type: Gin)
+  @@index([shouldUpdateSequenceId, type], name: "VerificationCode_shouldUpdateSequenceId_type_idx")
 }
 
 enum VerificationCodeType {
@@ -1012,13 +1051,18 @@ model EmailOutbox {
   unsubscribedAt DateTime?
   markedAsSpamAt DateTime?
 
+  sequenceId             BigInt? @unique
+  shouldUpdateSequenceId Boolean @default(true)
+
   tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade)
 
   @@id([tenancyId, id])
   @@index([tenancyId, finishedSendingAt(sort: Desc), scheduledAtIfNotYetQueued(sort: Desc), priority, id], map: "EmailOutbox_ordering_idx")
   @@index([tenancyId, simpleStatus], map: "EmailOutbox_simple_status_tenancy_idx")
   @@index([tenancyId, status], map: "EmailOutbox_status_tenancy_idx")
   @@index([isQueued], map: "EmailOutbox_isQueued_idx")
+  @@index([tenancyId, sequenceId], name: "EmailOutbox_tenancyId_sequenceId_idx")
+  @@index([shouldUpdateSequenceId, tenancyId], name: "EmailOutbox_shouldUpdateSequenceId_idx")
 }
 
 model EmailOutboxProcessingMetadata {
@@ -1078,8 +1122,13 @@ model UserNotificationPreference {
   enabled     Boolean
   projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade)
 
+  sequenceId             BigInt?  @unique
+  shouldUpdateSequenceId Boolean  @default(true)
+
   @@id([tenancyId, id])
   @@unique([tenancyId, projectUserId, notificationCategoryId])
+  @@index([shouldUpdateSequenceId, tenancyId], name: "UserNotificationPreference_shouldUpdateSequenceId_idx")
+  @@index([tenancyId, sequenceId], name: "UserNotificationPreference_tenancyId_sequenceId_idx")
 }
 
 model ThreadMessage {