diff --git a/.gitignore b/.gitignore index a8264f9e7d..eab9ce0b48 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ dist .docusaurus .cache-loader **.tsbuildinfo +implementation.generated.ts .xata* diff --git a/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql new file mode 100644 index 0000000000..fc5562113d --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "Team" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- AlterTable +ALTER TABLE "TeamMember" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql new file mode 100644 index 0000000000..c296a66581 --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "EmailOutbox" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql new file mode 100644 index 0000000000..50a85170e2 --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "SessionReplay" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql new file mode 100644 index 0000000000..d3158401a8 --- /dev/null +++ b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "TeamMemberDirectPermission" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- AlterTable +ALTER TABLE "VerificationCode" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql new file mode 100644 index 0000000000..d39d03dd21 --- /dev/null +++ b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "ProjectUserDirectPermission" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- AlterTable +ALTER TABLE "UserNotificationPreference" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql new file mode 100644 index 0000000000..61906b1ba2 --- /dev/null +++ b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "ProjectUserRefreshToken" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- AlterTable +ALTER TABLE "ProjectUserOAuthAccount" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql b/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql new file mode 100644 index 0000000000..c96f0f5648 --- /dev/null +++ b/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql @@ -0,0 +1,154 @@ +-- Team indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "Team_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."Team"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("shouldUpdateSequenceId", "tenancyId"); + +-- TeamMember indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("shouldUpdateSequenceId", "tenancyId"); + +-- EmailOutbox indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("shouldUpdateSequenceId", "tenancyId"); + +-- SessionReplay indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("shouldUpdateSequenceId", "tenancyId"); + +-- TeamMemberDirectPermission indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("tenancyId", "sequenceId"); + +-- VerificationCode indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_shouldUpdateSequenceId_type_idx" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("shouldUpdateSequenceId", "type"); + +-- ProjectUserDirectPermission indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("tenancyId", "sequenceId"); + +-- UserNotificationPreference indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("tenancyId", "sequenceId"); + +-- ProjectUserRefreshToken indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("tenancyId", "sequenceId"); + +-- ProjectUserOAuthAccount indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("tenancyId", "sequenceId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 48403f9daf..774d4d04e9 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -179,11 +179,16 @@ model Team { serverMetadata Json? profileImageUrl String? + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + teamMembers TeamMember[] projectApiKey ProjectApiKey[] @@id([tenancyId, teamId]) @@unique([mirroredProjectId, mirroredBranchId, teamId]) + @@index([tenancyId, sequenceId], name: "Team_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "Team_shouldUpdateSequenceId_idx") } // This is used for fields that are boolean but only the true value is part of a unique constraint. @@ -205,6 +210,9 @@ model TeamMember { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) team Team @relation(fields: [tenancyId, teamId], references: [tenancyId, teamId], onDelete: Cascade) isSelected BooleanTrue? @@ -213,6 +221,8 @@ model TeamMember { @@id([tenancyId, projectUserId, teamId]) @@unique([tenancyId, projectUserId, isSelected]) @@index([tenancyId, projectUserId, isSelected], map: "TeamMember_projectUserId_isSelected_idx") + @@index([tenancyId, sequenceId], name: "TeamMember_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMember_shouldUpdateSequenceId_idx") } model ProjectUserDirectPermission { @@ -226,7 +236,12 @@ model ProjectUserDirectPermission { projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@unique([tenancyId, projectUserId, permissionId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserDirectPermission_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "ProjectUserDirectPermission_tenancyId_sequenceId_idx") } model TeamMemberDirectPermission { @@ -241,7 +256,12 @@ model TeamMemberDirectPermission { teamMember TeamMember @relation(fields: [tenancyId, projectUserId, teamId], references: [tenancyId, projectUserId, teamId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@unique([tenancyId, projectUserId, teamId, permissionId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMemberDirectPermission_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "TeamMemberDirectPermission_tenancyId_sequenceId_idx") } model ProjectUser { @@ -343,9 +363,14 @@ model ProjectUserOAuthAccount { allowConnectedAccounts Boolean @default(true) allowSignIn Boolean @default(true) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) @@unique([tenancyId, configOAuthProviderId, projectUserId, providerAccountId]) @@index([tenancyId, projectUserId]) + @@index([tenancyId, sequenceId], name: "ProjectUserOAuthAccount_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx") } model SessionReplay { @@ -361,6 +386,9 @@ model SessionReplay { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@ -371,6 +399,8 @@ model SessionReplay { @@index([tenancyId, lastEventAt]) // index by updatedAt instead of lastEventAt because event timing can be spoofed @@index([tenancyId, refreshTokenId, updatedAt]) + @@index([tenancyId, sequenceId], name: "SessionReplay_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "SessionReplay_shouldUpdateSequenceId_idx") @@map("SessionReplay") } @@ -612,7 +642,12 @@ model ProjectUserRefreshToken { expiresAt DateTime? isImpersonation Boolean @default(false) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) + @@index([tenancyId, sequenceId], name: "ProjectUserRefreshToken_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserRefreshToken_shouldUpdateSequenceId_idx") } model ProjectUserAuthorizationCode { @@ -655,9 +690,13 @@ model VerificationCode { data Json attemptCount Int @default(0) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([projectId, branchId, id]) @@unique([projectId, branchId, code]) @@index([data(ops: JsonbPathOps)], type: Gin) + @@index([shouldUpdateSequenceId, type], name: "VerificationCode_shouldUpdateSequenceId_type_idx") } enum VerificationCodeType { @@ -1012,6 +1051,9 @@ model EmailOutbox { unsubscribedAt DateTime? markedAsSpamAt DateTime? + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@id([tenancyId, id]) @@ -1019,6 +1061,8 @@ model EmailOutbox { @@index([tenancyId, simpleStatus], map: "EmailOutbox_simple_status_tenancy_idx") @@index([tenancyId, status], map: "EmailOutbox_status_tenancy_idx") @@index([isQueued], map: "EmailOutbox_isQueued_idx") + @@index([tenancyId, sequenceId], name: "EmailOutbox_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "EmailOutbox_shouldUpdateSequenceId_idx") } model EmailOutboxProcessingMetadata { @@ -1078,8 +1122,13 @@ model UserNotificationPreference { enabled Boolean projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) @@unique([tenancyId, projectUserId, notificationCategoryId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "UserNotificationPreference_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "UserNotificationPreference_tenancyId_sequenceId_idx") } model ThreadMessage { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index d94f84baa0..8db2a40777 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -2,42 +2,86 @@ import { getClickhouseAdminClient } from "@/lib/clickhouse"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export async function runClickhouseMigrations() { + const start = performance.now(); console.log("[Clickhouse] Running Clickhouse migrations..."); const client = getClickhouseAdminClient(); const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD"); - await client.exec({ - query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}", - query_params: { clickhouseExternalPassword }, - }); - // todo: create migration files - await client.exec({ query: EXTERNAL_ANALYTICS_DB_SQL }); - await client.exec({ query: SYNC_METADATA_TABLE_SQL }); - await client.exec({ query: EVENTS_TABLE_BASE_SQL }); - await client.exec({ query: EVENTS_VIEW_SQL }); - await client.exec({ query: USERS_TABLE_BASE_SQL }); - await client.exec({ query: USERS_VIEW_SQL }); - await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); - await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); - await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); - await client.exec({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL }); - // Recreate the events view so SELECT * picks up columns added by EVENTS_ADD_REPLAY_COLUMNS_SQL - await client.exec({ query: EVENTS_VIEW_SQL }); - const queries = [ - "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;", - "REVOKE ALL FROM limited_user;", - "GRANT SELECT ON default.events TO limited_user;", - "GRANT SELECT ON default.users TO limited_user;", + + // Setup — database, user, sync metadata + await client.command({ query: EXTERNAL_ANALYTICS_DB_SQL }); + await Promise.all([ + client.command({ + query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}", + query_params: { clickhouseExternalPassword }, + }), + client.command({ query: SYNC_METADATA_TABLE_SQL }), + ]); + + // Create all tables in parallel + await Promise.all([ + client.command({ query: EVENTS_TABLE_BASE_SQL }), + client.command({ query: USERS_TABLE_BASE_SQL }), + client.command({ query: CONTACT_CHANNELS_TABLE_BASE_SQL }), + client.command({ query: TEAMS_TABLE_BASE_SQL }), + client.command({ query: TEAM_MEMBER_PROFILES_TABLE_BASE_SQL }), + client.command({ query: TEAM_PERMISSIONS_TABLE_BASE_SQL }), + client.command({ query: TEAM_INVITATIONS_TABLE_BASE_SQL }), + client.command({ query: EMAIL_OUTBOXES_TABLE_BASE_SQL }), + + client.command({ query: PROJECT_PERMISSIONS_TABLE_BASE_SQL }), + client.command({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }), + client.command({ query: REFRESH_TOKENS_TABLE_BASE_SQL }), + client.command({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }), + ]); + + // Alter events table (must come before views that reference new columns) + await client.command({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); + + // Create all views in parallel + await Promise.all([ + client.command({ query: EVENTS_VIEW_SQL }), + client.command({ query: USERS_VIEW_SQL }), + client.command({ query: CONTACT_CHANNELS_VIEW_SQL }), + client.command({ query: TEAMS_VIEW_SQL }), + client.command({ query: TEAM_MEMBER_PROFILES_VIEW_SQL }), + client.command({ query: TEAM_PERMISSIONS_VIEW_SQL }), + client.command({ query: TEAM_INVITATIONS_VIEW_SQL }), + client.command({ query: EMAIL_OUTBOXES_VIEW_SQL }), + + client.command({ query: PROJECT_PERMISSIONS_VIEW_SQL }), + client.command({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }), + client.command({ query: REFRESH_TOKENS_VIEW_SQL }), + client.command({ query: CONNECTED_ACCOUNTS_VIEW_SQL }), + ]); + + // Data migrations (mutations) + await Promise.all([ + client.command({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }), + client.command({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }), + client.command({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL }), + ]); + + // Row policies in parallel + const tables = [ + "events", "users", "contact_channels", "teams", "team_member_profiles", + "team_permissions", "team_invitations", "email_outboxes", + "project_permissions", "notification_preferences", "refresh_tokens", "connected_accounts", ]; - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS users_project_isolation ON default.users FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - for (const query of queries) { - await client.exec({ query }); - } - console.log("[Clickhouse] Clickhouse migrations complete"); + await Promise.all(tables.map(table => + client.command({ + query: `CREATE ROW POLICY IF NOT EXISTS ${table}_project_isolation ON default.${table} FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user`, + }) + )); + + // Grants + await client.command({ query: "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;" }); + await client.command({ query: "REVOKE ALL FROM limited_user;" }); + await Promise.all(tables.map(table => + client.command({ query: `GRANT SELECT ON default.${table} TO limited_user;` }) + )); + + const elapsed = ((performance.now() - start) / 1000).toFixed(1); + console.log(`[Clickhouse] Clickhouse migrations complete (${elapsed}s)`); await client.close(); } @@ -197,6 +241,410 @@ WHERE event_type = '$token-refresh' AND data.refresh_token_id::Nullable(String) IS NOT NULL; `; +const CONTACT_CHANNELS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.contact_channels ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + type LowCardinality(String), + value String, + is_primary UInt8, + is_verified UInt8, + used_for_auth UInt8, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const CONTACT_CHANNELS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.contact_channels +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + type, + value, + is_primary, + is_verified, + used_for_auth, + created_at +FROM analytics_internal.contact_channels +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAMS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.teams ( + project_id String, + branch_id String, + id UUID, + display_name String, + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + client_metadata String, + client_read_only_metadata String, + server_metadata String, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const TEAMS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.teams +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + display_name, + profile_image_url, + created_at, + client_metadata, + client_read_only_metadata, + server_metadata +FROM analytics_internal.teams +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAM_MEMBER_PROFILES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_member_profiles ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + display_name Nullable(String), + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, team_id, user_id); +`; + +const TEAM_MEMBER_PROFILES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_member_profiles +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + team_id, + user_id, + display_name, + profile_image_url, + created_at +FROM analytics_internal.team_member_profiles +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAM_PERMISSIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, team_id, user_id, id); +`; + +const TEAM_PERMISSIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_permissions +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + team_id, + user_id, + id, + created_at +FROM analytics_internal.team_permissions +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAM_INVITATIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_invitations ( + project_id String, + branch_id String, + id UUID, + team_id UUID, + team_display_name String, + recipient_email String, + expires_at_millis Int64, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const TEAM_INVITATIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_invitations +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + team_id, + team_display_name, + recipient_email, + expires_at_millis, + created_at +FROM analytics_internal.team_invitations +FINAL +WHERE sync_is_deleted = 0; +`; + +const EMAIL_OUTBOXES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( + project_id String, + branch_id String, + id UUID, + status LowCardinality(String), + simple_status LowCardinality(String), + created_with LowCardinality(String), + email_draft_id Nullable(String), + email_programmatic_call_template_id Nullable(String), + theme_id Nullable(String), + is_high_priority UInt8, + is_transactional Nullable(UInt8), + subject Nullable(String), + notification_category_id Nullable(String), + started_rendering_at Nullable(DateTime64(3, 'UTC')), + rendered_at Nullable(DateTime64(3, 'UTC')), + render_error Nullable(String), + scheduled_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + updated_at DateTime64(3, 'UTC'), + started_sending_at Nullable(DateTime64(3, 'UTC')), + server_error Nullable(String), + delivered_at Nullable(DateTime64(3, 'UTC')), + opened_at Nullable(DateTime64(3, 'UTC')), + clicked_at Nullable(DateTime64(3, 'UTC')), + unsubscribed_at Nullable(DateTime64(3, 'UTC')), + marked_as_spam_at Nullable(DateTime64(3, 'UTC')), + bounced_at Nullable(DateTime64(3, 'UTC')), + delivery_delayed_at Nullable(DateTime64(3, 'UTC')), + can_have_delivery_info Nullable(UInt8), + skipped_reason LowCardinality(Nullable(String)), + skipped_details Nullable(String), + send_retries Int32, + is_paused UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const EMAIL_OUTBOXES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.email_outboxes +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + status, + simple_status, + created_with, + email_draft_id, + email_programmatic_call_template_id, + theme_id, + is_high_priority, + is_transactional, + subject, + notification_category_id, + started_rendering_at, + rendered_at, + render_error, + scheduled_at, + created_at, + updated_at, + started_sending_at, + server_error, + delivered_at, + opened_at, + clicked_at, + unsubscribed_at, + marked_as_spam_at, + bounced_at, + delivery_delayed_at, + can_have_delivery_info, + skipped_reason, + skipped_details, + send_retries, + is_paused +FROM analytics_internal.email_outboxes +FINAL +WHERE sync_is_deleted = 0; +`; + + +const PROJECT_PERMISSIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( + project_id String, + branch_id String, + user_id UUID, + id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, user_id, id); +`; + +const PROJECT_PERMISSIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.project_permissions +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + user_id, + id, + created_at +FROM analytics_internal.project_permissions +FINAL +WHERE sync_is_deleted = 0; +`; + +const NOTIFICATION_PREFERENCES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( + project_id String, + branch_id String, + user_id UUID, + notification_category_id String, + enabled UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +ORDER BY (project_id, branch_id, user_id, notification_category_id); +`; + +const NOTIFICATION_PREFERENCES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.notification_preferences +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + user_id, + notification_category_id, + enabled +FROM analytics_internal.notification_preferences +FINAL +WHERE sync_is_deleted = 0; +`; + +const REFRESH_TOKENS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.refresh_tokens ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + created_at DateTime64(3, 'UTC'), + last_used_at DateTime64(3, 'UTC'), + is_impersonation UInt8, + expires_at Nullable(DateTime64(3, 'UTC')), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const REFRESH_TOKENS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.refresh_tokens +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + created_at, + last_used_at, + is_impersonation, + expires_at +FROM analytics_internal.refresh_tokens +FINAL +WHERE sync_is_deleted = 0; +`; + +const CONNECTED_ACCOUNTS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( + project_id String, + branch_id String, + user_id UUID, + provider String, + provider_account_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, user_id, provider, provider_account_id); +`; + +const CONNECTED_ACCOUNTS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.connected_accounts +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + user_id, + provider, + provider_account_id, + created_at +FROM analytics_internal.connected_accounts +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts index 98b9680ce5..abd2214958 100644 --- a/apps/backend/scripts/run-cron-jobs.ts +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -30,8 +30,7 @@ async function main() { if (runResult.status === "error") { captureError("run-cron-jobs", runResult.error); } - // Vercel only guarantees minute-granularity for cron jobs, so we randomize the interval - await wait(Math.random() * 120_000); + await wait(1000); } }); } diff --git a/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts new file mode 100644 index 0000000000..9eeb0104e3 --- /dev/null +++ b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts @@ -0,0 +1,242 @@ +import { getClickhouseAdminClient } from "@/lib/clickhouse"; +import { CLICKHOUSE_COLUMN_NORMALIZERS } from "@/lib/external-db-sync"; +import type { Tenancy } from "@/lib/tenancies"; +import { getPrismaClientForTenancy } from "@/prisma-client"; +import { DEFAULT_DB_SYNC_MAPPINGS } from "@stackframe/stack-shared/dist/config/db-sync-mappings"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; + +import type { RecurseFunction } from "./recurse"; + +// Sort key columns for each mapping (after project_id, branch_id), matching ClickHouse ORDER BY +const SORT_KEYS = { + users: ["id"], + contact_channels: ["id"], + teams: ["id"], + team_member_profiles: ["team_id", "user_id"], + team_permissions: ["team_id", "user_id", "id"], + team_invitations: ["id"], + email_outboxes: ["id"], + project_permissions: ["user_id", "permission_id"], + notification_preferences: ["id"], + refresh_tokens: ["id"], + connected_accounts: ["user_id", "provider", "provider_account_id"], +} satisfies Record; + +const SYNC_COLUMNS_TO_STRIP = ["sync_sequence_id", "sync_is_deleted", "sync_created_at", "tenancyId"]; + +function compareRows(a: Record, b: Record, sortKeys: string[]): number { + for (const key of sortKeys) { + const aVal = String(a[key] ?? ""); + const bVal = String(b[key] ?? ""); + if (aVal < bVal) return -1; + if (aVal > bVal) return 1; + } + return 0; +} + +function normalizePostgresValue(value: unknown, columnType: string | undefined): unknown { + if (value === null || value === undefined) return null; + if (columnType === "json") { + // Postgres returns parsed JS values for jsonb columns; always stringify for consistent comparison + return JSON.stringify(value); + } + if (columnType === "boolean") { + if (typeof value === "boolean") return value ? 1 : 0; + return value; + } + if (columnType === "nullable_boolean") { + if (typeof value === "boolean") return value ? 1 : 0; + return value; + } + if (columnType === "bigint") { + return Number(value); + } + // For dates, normalize to ms epoch + if (value instanceof Date) { + return value.getTime(); + } + return value; +} + +function normalizeClickhouseValue(value: unknown, columnType: string | undefined): unknown { + if (value === null || value === undefined) return null; + if (columnType === "json") { + // ClickHouse stores null JSON as the literal string "null" + if (value === "null") return null; + return typeof value === "string" ? value : JSON.stringify(value); + } + // For dates (ClickHouse returns as string like "2024-01-01 00:00:00.000" in UTC) + if (typeof value === "string" && /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}/.test(value)) { + // Append 'Z' to force UTC interpretation if no timezone indicator present + const dateStr = value.includes("Z") || value.includes("+") ? value : value.replace(" ", "T") + "Z"; + return new Date(dateStr).getTime(); + } + return value; +} + +function normalizeRow( + row: Record, + normalizers: Record, + side: "postgres" | "clickhouse", +): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(row)) { + const columnType = normalizers[key]; + result[key] = side === "postgres" + ? normalizePostgresValue(value, columnType) + : normalizeClickhouseValue(value, columnType); + } + return result; +} + +// Strip null values and empty objects from nested structures. +// ClickHouse's native JSON type omits these, so we need to normalize before comparing. +function stripNullsAndEmpties(value: unknown): unknown { + if (value === null || value === undefined) return undefined; + if (typeof value !== "object") return value; + if (Array.isArray(value)) return value.map(stripNullsAndEmpties); + const obj = value as Record; + const result: Record = {}; + for (const [k, v] of Object.entries(obj)) { + const stripped = stripNullsAndEmpties(v); + if (stripped === undefined) continue; + if (typeof stripped === "object" && stripped !== null && !Array.isArray(stripped) && Object.keys(stripped).length === 0) continue; + result[k] = stripped; + } + return result; +} + +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (a === null || b === null) return a === b; + if (typeof a !== typeof b) return false; + if (typeof a === "object" && typeof b === "object") { + // Normalize both sides to handle ClickHouse JSON stripping nulls/empties + const aNorm = stripNullsAndEmpties(a) as Record; + const bNorm = stripNullsAndEmpties(b) as Record; + const aKeys = Object.keys(aNorm); + const bKeys = Object.keys(bNorm); + if (aKeys.length !== bKeys.length) return false; + return aKeys.every(key => deepEqual(aNorm[key], bNorm[key])); + } + return false; +} + +function findDifferences( + pgRow: Record, + chRow: Record, +): string[] { + const diffs: string[] = []; + const allKeys = new Set([...Object.keys(pgRow), ...Object.keys(chRow)]); + for (const key of allKeys) { + if (!deepEqual(pgRow[key], chRow[key])) { + diffs.push(`${key}: pg=${JSON.stringify(pgRow[key])} ch=${JSON.stringify(chRow[key])}`); + } + } + return diffs; +} + +export async function verifyClickhouseSync(options: { + tenancy: Tenancy, + projectId: string, + branchId: string, + recurse: RecurseFunction, +}) { + const { tenancy, projectId, branchId, recurse } = options; + const clickhouseClient = getClickhouseAdminClient(); + const prisma = await getPrismaClientForTenancy(tenancy); + + for (const [mappingName, mapping] of Object.entries(DEFAULT_DB_SYNC_MAPPINGS)) { + await recurse(`[${mappingName}]`, async () => { + const fetchQuery = mapping.internalDbFetchQueries.clickhouse; + if (!fetchQuery) return; + + if (!(mappingName in SORT_KEYS)) { + throw new StackAssertionError(`No sort keys defined for mapping ${mappingName}`); + } + const sortKeys = SORT_KEYS[mappingName as keyof typeof SORT_KEYS]; + + const normalizers = CLICKHOUSE_COLUMN_NORMALIZERS[mappingName] ?? {}; + + // Fetch all non-deleted rows from Postgres using the same query the sync uses + const pgRows: Record[] = []; + let lastSequenceId = -1; + const BATCH_LIMIT = 1000; + + // eslint-disable-next-line no-constant-condition + while (true) { + const batch = await prisma.$queryRawUnsafe[]>( + fetchQuery, + tenancy.id, + lastSequenceId, + ); + + if (batch.length === 0) break; + + for (const row of batch) { + const syncIsDeleted = row.sync_is_deleted; + if (syncIsDeleted === true || syncIsDeleted === "true") continue; + + const stripped: Record = {}; + for (const [key, value] of Object.entries(row)) { + if (!SYNC_COLUMNS_TO_STRIP.includes(key)) { + stripped[key] = value; + } + } + pgRows.push(stripped); + } + + // Find max sequence ID in batch for cursor + let maxSeq = lastSequenceId; + for (const row of batch) { + const seq = Number(row.sync_sequence_id); + if (Number.isFinite(seq) && seq > maxSeq) { + maxSeq = seq; + } + } + lastSequenceId = maxSeq; + + if (batch.length < BATCH_LIMIT) break; + } + + // Fetch all rows from ClickHouse view (already FINAL + sync_is_deleted = 0) + const chResult = await clickhouseClient.query({ + query: `SELECT * FROM default.${mapping.targetTable} WHERE project_id = {project_id:String} AND branch_id = {branch_id:String}`, + query_params: { project_id: projectId, branch_id: branchId }, + format: "JSONEachRow", + }); + const chRows = await chResult.json>(); + + // Compare row counts + if (pgRows.length !== chRows.length) { + throw new StackAssertionError(deindent` + ClickHouse sync row count mismatch for ${mappingName}. + Postgres: ${pgRows.length} rows, ClickHouse: ${chRows.length} rows. + `); + } + + if (pgRows.length === 0) return; + + // Sort both by primary key columns + const fullSortKeys = ["project_id", "branch_id", ...sortKeys]; + pgRows.sort((a, b) => compareRows(a, b, fullSortKeys)); + chRows.sort((a, b) => compareRows(a, b, fullSortKeys)); + + // Compare row by row + for (let i = 0; i < pgRows.length; i++) { + const normalizedPg = normalizeRow(pgRows[i], normalizers, "postgres"); + const normalizedCh = normalizeRow(chRows[i], normalizers, "clickhouse"); + + if (!deepEqual(normalizedPg, normalizedCh)) { + const diffs = findDifferences(normalizedPg, normalizedCh); + const keyValues = fullSortKeys.map(k => `${k}=${pgRows[i][k]}`).join(", "); + throw new StackAssertionError(deindent` + ClickHouse sync data mismatch for ${mappingName} at row ${keyValues}. + Differences: ${diffs.join("; ")} + `); + } + } + }); + } +} diff --git a/apps/backend/scripts/verify-data-integrity/index.ts b/apps/backend/scripts/verify-data-integrity/index.ts index ea1f01d703..48cc992cce 100644 --- a/apps/backend/scripts/verify-data-integrity/index.ts +++ b/apps/backend/scripts/verify-data-integrity/index.ts @@ -9,6 +9,7 @@ import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; import fs from "fs"; import { createApiHelpers, loadOutputData, type OutputData } from "./api"; +import { verifyClickhouseSync } from "./clickhouse-sync-verifier"; import { createPaymentsVerifier } from "./payments-verifier"; import { createRecurse } from "./recurse"; import { verifyStripePayoutIntegrity } from "./stripe-payout-integrity"; @@ -78,6 +79,7 @@ async function main() { const shouldSkipNeon = flags.includes("--skip-neon"); const recentFirst = flags.includes("--recent-first"); const noBail = flags.includes("--no-bail"); + const shouldSkipClickhouse = flags.includes("--skip-clickhouse"); const maxUsersPerProjectFlag = flags.find(f => f.startsWith("--max-users-per-project=")); const maxUsersPerProject = maxUsersPerProjectFlag ? parseInt(maxUsersPerProjectFlag.split("=")[1], 10) @@ -154,6 +156,13 @@ async function main() { console.warn("Using mock Stripe server (STACK_STRIPE_SECRET_KEY=sk_test_mockstripekey); skipping Stripe payout integrity checks."); } + const clickhouseAvailable = getEnvVariable("STACK_CLICKHOUSE_URL", "") !== ""; + if (shouldSkipClickhouse) { + console.log(`Will skip ClickHouse sync verification.`); + } else if (!clickhouseAvailable) { + console.log(`STACK_CLICKHOUSE_URL not set; skipping ClickHouse sync verification.`); + } + if (maxUsersPerProject !== Infinity) { console.log(`Will check at most ${maxUsersPerProject} users per project.`); } @@ -217,6 +226,17 @@ async function main() { }); } + if (!shouldSkipClickhouse && clickhouseAvailable && tenancy) { + await recurse("[clickhouse sync]", async (recurse) => { + await verifyClickhouseSync({ + tenancy, + projectId, + branchId: DEFAULT_BRANCH_ID, + recurse, + }); + }); + } + const verifiedTeams = new Set(); if (!skipUsers) { diff --git a/apps/backend/src/app/api/latest/auth/password/update/route.tsx b/apps/backend/src/app/api/latest/auth/password/update/route.tsx index db6d43f244..75756e4783 100644 --- a/apps/backend/src/app/api/latest/auth/password/update/route.tsx +++ b/apps/backend/src/app/api/latest/auth/password/update/route.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncRefreshTokenDeletionsForUser } from "@/lib/external-db-sync"; import { getPrismaClientForTenancy, globalPrismaClient, retryTransaction } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { KnownErrors } from "@stackframe/stack-shared"; @@ -78,6 +79,12 @@ export const POST = createSmartRouteHandler({ }); // reset all other refresh tokens + await recordExternalDbSyncRefreshTokenDeletionsForUser(globalPrismaClient, { + tenancyId: tenancy.id, + projectUserId: user.id, + excludeRefreshToken: refreshToken?.[0], + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: tenancy.id, diff --git a/apps/backend/src/app/api/latest/auth/sessions/crud.tsx b/apps/backend/src/app/api/latest/auth/sessions/crud.tsx index 5f7d9d126a..759a2f81ee 100644 --- a/apps/backend/src/app/api/latest/auth/sessions/crud.tsx +++ b/apps/backend/src/app/api/latest/auth/sessions/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { globalPrismaClient } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; import { SmartRequestAuth } from "@/route-handlers/smart-request"; @@ -71,6 +72,12 @@ export const sessionsCrudHandlers = createLazyProxy(() => createCrudHandlers(ses throw new KnownErrors.CannotDeleteCurrentSession(); } + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "ProjectUserRefreshToken", + tenancyId: auth.tenancy.id, + refreshTokenId: params.id, + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: auth.tenancy.id, diff --git a/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx b/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx index 6df9ab3a5f..9ab3716eec 100644 --- a/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx +++ b/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { Prisma } from "@/generated/prisma/client"; @@ -32,6 +33,13 @@ export const DELETE = createSmartRouteHandler({ try { const prisma = await getPrismaClientForTenancy(tenancy); + + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "ProjectUserRefreshToken", + tenancyId: tenancy.id, + refreshTokenId, + }); + const result = await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: tenancy.id, diff --git a/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx b/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx index dd9deed63a..e9b9c8583e 100644 --- a/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx +++ b/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx @@ -1,3 +1,4 @@ +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { listNotificationCategories } from "@/lib/notification-categories"; import { ensureUserExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy } from "@/prisma-client"; @@ -40,15 +41,15 @@ export const notificationPreferencesCrudHandlers = createLazyProxy(() => createC notificationCategoryId: params.notification_category_id, }, }, - update: { + update: withExternalDbSyncUpdate({ enabled: data.enabled, - }, - create: { + }), + create: withExternalDbSyncUpdate({ tenancyId: auth.tenancy.id, projectUserId: userId, notificationCategoryId: params.notification_category_id, enabled: data.enabled, - }, + }), }); return { diff --git a/apps/backend/src/app/api/latest/emails/outbox/crud.tsx b/apps/backend/src/app/api/latest/emails/outbox/crud.tsx index 2d36b499d8..73ec227026 100644 --- a/apps/backend/src/app/api/latest/emails/outbox/crud.tsx +++ b/apps/backend/src/app/api/latest/emails/outbox/crud.tsx @@ -447,6 +447,9 @@ export const emailOutboxCrudHandlers = createLazyProxy(() => createCrudHandlers( set("updatedAt", Prisma.sql`NOW()`); } + // Mark for external DB sync + set("shouldUpdateSequenceId", Prisma.sql`TRUE`); + const updateQuery: RawQuery = { supportedPrismaClients: ["global"], readOnlyQuery: false, @@ -543,6 +546,8 @@ function parseEmailOutboxFromJson(j: Record): EmailOutbox { clickedAt: dateOrNull("clickedAt"), unsubscribedAt: dateOrNull("unsubscribedAt"), markedAsSpamAt: dateOrNull("markedAsSpamAt"), + sequenceId: j.sequenceId != null ? BigInt(j.sequenceId as string | number) : null, + shouldUpdateSequenceId: j.shouldUpdateSequenceId as boolean, }; } diff --git a/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx b/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx index 9e04384779..19e5696a16 100644 --- a/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx +++ b/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx @@ -1,3 +1,4 @@ +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client"; import { VerificationCodeType } from "@/generated/prisma/client"; @@ -51,15 +52,15 @@ export async function GET(request: NextRequest) { notificationCategoryId: notification_category_id, }, }, - update: { + update: withExternalDbSyncUpdate({ enabled: false, - }, - create: { + }), + create: withExternalDbSyncUpdate({ tenancyId: tenancy.id, projectUserId: user_id, notificationCategoryId: notification_category_id, enabled: false, - }, + }), }); return new Response('

Successfully unsubscribed from notification group

', { diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index c7808fb53b..0a35024ca6 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -1,5 +1,6 @@ import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; import { enqueueExternalDbSyncBatch } from "@/lib/external-db-sync-queue"; +import { Prisma } from "@/generated/prisma/client"; import { globalPrismaClient } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { traceSpan } from "@/utils/telemetry"; @@ -109,6 +110,282 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const teamTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string, teamId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "teamId" + FROM "Team" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "Team" t + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE t."tenancyId" = r."tenancyId" + AND t."teamId" = r."teamId" + RETURNING t."tenancyId", t."teamId" + ) + SELECT DISTINCT "tenancyId", "teamId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-tenants", teamTenants.length); + + if (teamTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamTenants.map(t => t.tenancyId)); + didUpdate = true; + + // Cascade: when a team changes, mark related TEAM_INVITATION verification codes for re-sync + // so the team_display_name in team_invitations stays fresh + await globalPrismaClient.$executeRaw` + UPDATE "VerificationCode" + SET "shouldUpdateSequenceId" = TRUE + FROM ( + SELECT DISTINCT "Tenancy"."projectId", "Tenancy"."branchId", "Team"."teamId" + FROM "Team" + JOIN "Tenancy" ON "Tenancy"."id" = "Team"."tenancyId" + WHERE "Team"."tenancyId" IN (${Prisma.join(teamTenants.map(t => t.tenancyId))}) + AND "Team"."shouldUpdateSequenceId" = FALSE + AND "Team"."sequenceId" IS NOT NULL + ) AS changed_teams + WHERE "VerificationCode"."projectId" = changed_teams."projectId" + AND "VerificationCode"."branchId" = changed_teams."branchId" + AND "VerificationCode"."type" = 'TEAM_INVITATION' + AND "VerificationCode"."data"->>'team_id' = changed_teams."teamId" + AND "VerificationCode"."shouldUpdateSequenceId" = FALSE + `; + } + + const teamMemberTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId", "teamId" + FROM "TeamMember" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "TeamMember" tm + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE tm."tenancyId" = r."tenancyId" + AND tm."projectUserId" = r."projectUserId" + AND tm."teamId" = r."teamId" + RETURNING tm."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-member-tenants", teamMemberTenants.length); + + if (teamMemberTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamMemberTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const teamPermissionTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id" + FROM "TeamMemberDirectPermission" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "TeamMemberDirectPermission" tp + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE tp."id" = r."id" + RETURNING tp."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-permission-tenants", teamPermissionTenants.length); + + if (teamPermissionTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamPermissionTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const teamInvitationTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "projectId", "branchId", "id" + FROM "VerificationCode" + WHERE "shouldUpdateSequenceId" = TRUE + AND "type" = 'TEAM_INVITATION' + ORDER BY "projectId", "branchId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "VerificationCode" vc + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE vc."projectId" = r."projectId" + AND vc."branchId" = r."branchId" + AND vc."id" = r."id" + RETURNING vc."projectId", vc."branchId" + ) + SELECT DISTINCT "Tenancy"."id" AS "tenancyId" + FROM updated_rows + JOIN "Tenancy" ON "Tenancy"."projectId" = updated_rows."projectId" + AND "Tenancy"."branchId" = updated_rows."branchId" + `; + + span.setAttribute("stack.external-db-sync.team-invitation-tenants", teamInvitationTenants.length); + + if (teamInvitationTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamInvitationTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const emailOutboxTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "EmailOutbox" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "EmailOutbox" eo + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE eo."tenancyId" = r."tenancyId" + AND eo."id" = r."id" + RETURNING eo."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.email-outbox-tenants", emailOutboxTenants.length); + + if (emailOutboxTenants.length > 0) { + await enqueueExternalDbSyncBatch(emailOutboxTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const projectPermissionTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id" + FROM "ProjectUserDirectPermission" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserDirectPermission" pp + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE pp."id" = r."id" + RETURNING pp."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.project-permission-tenants", projectPermissionTenants.length); + + if (projectPermissionTenants.length > 0) { + await enqueueExternalDbSyncBatch(projectPermissionTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const notificationPreferenceTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "UserNotificationPreference" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "UserNotificationPreference" np + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE np."tenancyId" = r."tenancyId" + AND np."id" = r."id" + RETURNING np."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.notification-preference-tenants", notificationPreferenceTenants.length); + + if (notificationPreferenceTenants.length > 0) { + await enqueueExternalDbSyncBatch(notificationPreferenceTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const refreshTokenTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "ProjectUserRefreshToken" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserRefreshToken" rt + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE rt."tenancyId" = r."tenancyId" + AND rt."id" = r."id" + RETURNING rt."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.refresh-token-tenants", refreshTokenTenants.length); + + if (refreshTokenTenants.length > 0) { + await enqueueExternalDbSyncBatch(refreshTokenTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const oauthAccountTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "ProjectUserOAuthAccount" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserOAuthAccount" oa + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE oa."tenancyId" = r."tenancyId" + AND oa."id" = r."id" + RETURNING oa."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.oauth-account-tenants", oauthAccountTenants.length); + + if (oauthAccountTenants.length > 0) { + await enqueueExternalDbSyncBatch(oauthAccountTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -138,7 +415,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, EO=${emailOutboxTenants.length}, PP=${projectPermissionTenants.length}, NP=${notificationPreferenceTenants.length}, RT=${refreshTokenTenants.length}, CA=${oauthAccountTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts index 701c8818a0..88845389e6 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts @@ -87,6 +87,15 @@ const globalSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + teams: sequenceStatsSchema.defined(), + team_members: sequenceStatsSchema.defined(), + team_permissions: sequenceStatsSchema.defined(), + team_invitations: sequenceStatsSchema.defined(), + email_outboxes: sequenceStatsSchema.defined(), + project_permissions: sequenceStatsSchema.defined(), + notification_preferences: sequenceStatsSchema.defined(), + refresh_tokens: sequenceStatsSchema.defined(), + connected_accounts: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -119,6 +128,15 @@ const responseSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + teams: sequenceStatsSchema.defined(), + team_members: sequenceStatsSchema.defined(), + team_permissions: sequenceStatsSchema.defined(), + team_invitations: sequenceStatsSchema.defined(), + email_outboxes: sequenceStatsSchema.defined(), + project_permissions: sequenceStatsSchema.defined(), + notification_preferences: sequenceStatsSchema.defined(), + refresh_tokens: sequenceStatsSchema.defined(), + connected_accounts: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -232,11 +250,21 @@ function maxBigIntString(values: Array): string | nul } function buildMappingInternalStats( - projectUsersStats: SequenceStats, + stats: { + projectUsersStats: SequenceStats, + contactChannelStats: SequenceStats, + teamStats: SequenceStats, + teamMemberStats: SequenceStats, + teamPermissionStats: SequenceStats, + teamInvitationStats: SequenceStats, + emailOutboxStats: SequenceStats, + projectPermissionStats: SequenceStats, + notificationPreferenceStats: SequenceStats, + refreshTokenStats: SequenceStats, + connectedAccountStats: SequenceStats, + }, deletedRowsByTable: DeletedRowSummary[], ) { - const deletedProjectUserStats = deletedRowsByTable.find((row) => row.table_name === "ProjectUser") ?? null; - const mappingInternalStats = new Map(); - const usersMappingMin = minBigIntString([ - projectUsersStats.min_sequence_id, - deletedProjectUserStats?.min_sequence_id, - ]); - const usersMappingMax = maxBigIntString([ - projectUsersStats.max_sequence_id, - deletedProjectUserStats?.max_sequence_id, - ]); - const usersMappingPending = addBigIntStrings( - projectUsersStats.pending, - deletedProjectUserStats?.pending, - ); - - mappingInternalStats.set("users", { - mapping_id: "users", - internal_min_sequence_id: usersMappingMin, - internal_max_sequence_id: usersMappingMax, - internal_pending_count: usersMappingPending, - }); + function addMapping(mappingId: string, primaryStats: SequenceStats, deletedRowTableName: string | null) { + const deletedStats = deletedRowTableName + ? deletedRowsByTable.find((row) => row.table_name === deletedRowTableName) ?? null + : null; + mappingInternalStats.set(mappingId, { + mapping_id: mappingId, + internal_min_sequence_id: minBigIntString([primaryStats.min_sequence_id, deletedStats?.min_sequence_id]), + internal_max_sequence_id: maxBigIntString([primaryStats.max_sequence_id, deletedStats?.max_sequence_id]), + internal_pending_count: addBigIntStrings(primaryStats.pending, deletedStats?.pending), + }); + } + + addMapping("users", stats.projectUsersStats, "ProjectUser"); + addMapping("contact_channels", stats.contactChannelStats, "ContactChannel"); + addMapping("teams", stats.teamStats, "Team"); + addMapping("team_member_profiles", stats.teamMemberStats, "TeamMember"); + addMapping("team_permissions", stats.teamPermissionStats, "TeamMemberDirectPermission"); + addMapping("team_invitations", stats.teamInvitationStats, "VerificationCode_TEAM_INVITATION"); + addMapping("email_outboxes", stats.emailOutboxStats, "EmailOutbox"); + addMapping("project_permissions", stats.projectPermissionStats, "ProjectUserDirectPermission"); + addMapping("notification_preferences", stats.notificationPreferenceStats, "UserNotificationPreference"); + addMapping("refresh_tokens", stats.refreshTokenStats, "ProjectUserRefreshToken"); + addMapping("connected_accounts", stats.connectedAccountStats, "ProjectUserOAuthAccount"); const mappings = Array.from(mappingInternalStats.values()); const mappingStatuses = mappings.map((mapping) => ({ @@ -300,6 +332,107 @@ async function fetchInternalStats(tenancyId: string | null) { ${tenancyWhere} `).at(0) ?? throwErr("Contact channel stats query returned no rows."); + const teamStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "Team" + ${tenancyWhere} + `).at(0) ?? throwErr("Team stats query returned no rows."); + + const teamMemberStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "TeamMember" + ${tenancyWhere} + `).at(0) ?? throwErr("Team member stats query returned no rows."); + + const teamPermissionStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "TeamMemberDirectPermission" + ${tenancyWhere} + `).at(0) ?? throwErr("Team permission stats query returned no rows."); + + const teamInvitationStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "VerificationCode" + ${tenancyId + ? Prisma.sql`JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" AND "Tenancy"."branchId" = "VerificationCode"."branchId" WHERE "type" = 'TEAM_INVITATION' AND "Tenancy"."id" = ${tenancyId}::uuid` + : Prisma.sql`WHERE "type" = 'TEAM_INVITATION'`} + `).at(0) ?? throwErr("Team invitation stats query returned no rows."); + + const emailOutboxStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "EmailOutbox" + ${tenancyWhere} + `).at(0) ?? throwErr("Email outbox stats query returned no rows."); + + const projectPermissionStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserDirectPermission" + ${tenancyWhere} + `).at(0) ?? throwErr("Project permission stats query returned no rows."); + + const notificationPreferenceStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "UserNotificationPreference" + ${tenancyWhere} + `).at(0) ?? throwErr("Notification preference stats query returned no rows."); + + const refreshTokenStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserRefreshToken" + ${tenancyWhere} + `).at(0) ?? throwErr("Refresh token stats query returned no rows."); + + const connectedAccountStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserOAuthAccount" + ${tenancyWhere} + `).at(0) ?? throwErr("Connected account stats query returned no rows."); + const deletedRowStatsRow = (await globalPrismaClient.$queryRaw` SELECT COUNT(*)::bigint AS "total", @@ -346,6 +479,15 @@ async function fetchInternalStats(tenancyId: string | null) { const projectUsersStats = formatSequenceStats(projectUserStatsRow); const contactChannelStats = formatSequenceStats(contactChannelStatsRow); + const teamStats = formatSequenceStats(teamStatsRow); + const teamMemberStats = formatSequenceStats(teamMemberStatsRow); + const teamPermissionStats = formatSequenceStats(teamPermissionStatsRow); + const teamInvitationStats = formatSequenceStats(teamInvitationStatsRow); + const emailOutboxStats = formatSequenceStats(emailOutboxStatsRow); + const projectPermissionStats = formatSequenceStats(projectPermissionStatsRow); + const notificationPreferenceStats = formatSequenceStats(notificationPreferenceStatsRow); + const refreshTokenStats = formatSequenceStats(refreshTokenStatsRow); + const connectedAccountStats = formatSequenceStats(connectedAccountStatsRow); const deletedRowStats = formatSequenceStats(deletedRowStatsRow); const deletedRowsByTable = deletedRowsByTableRows.map((row) => ({ @@ -353,11 +495,32 @@ async function fetchInternalStats(tenancyId: string | null) { ...formatSequenceStats(row), })); - const { mappings, mappingStatuses } = buildMappingInternalStats(projectUsersStats, deletedRowsByTable); + const { mappings, mappingStatuses } = buildMappingInternalStats({ + projectUsersStats, + contactChannelStats, + teamStats, + teamMemberStats, + teamPermissionStats, + teamInvitationStats, + emailOutboxStats, + projectPermissionStats, + notificationPreferenceStats, + refreshTokenStats, + connectedAccountStats, + }, deletedRowsByTable); return { projectUsersStats, contactChannelStats, + teamStats, + teamMemberStats, + teamPermissionStats, + teamInvitationStats, + emailOutboxStats, + projectPermissionStats, + notificationPreferenceStats, + refreshTokenStats, + connectedAccountStats, deletedRowStats, deletedRowsByTable, outgoingStatsRow, @@ -1003,6 +1166,15 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: globalStats.projectUsersStats, contact_channels: globalStats.contactChannelStats, + teams: globalStats.teamStats, + team_members: globalStats.teamMemberStats, + team_permissions: globalStats.teamPermissionStats, + team_invitations: globalStats.teamInvitationStats, + email_outboxes: globalStats.emailOutboxStats, + project_permissions: globalStats.projectPermissionStats, + notification_preferences: globalStats.notificationPreferenceStats, + refresh_tokens: globalStats.refreshTokenStats, + connected_accounts: globalStats.connectedAccountStats, deleted_rows: { ...globalStats.deletedRowStats, by_table: globalStats.deletedRowsByTable, @@ -1021,6 +1193,15 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: currentStats.projectUsersStats, contact_channels: currentStats.contactChannelStats, + teams: currentStats.teamStats, + team_members: currentStats.teamMemberStats, + team_permissions: currentStats.teamPermissionStats, + team_invitations: currentStats.teamInvitationStats, + email_outboxes: currentStats.emailOutboxStats, + project_permissions: currentStats.projectPermissionStats, + notification_preferences: currentStats.notificationPreferenceStats, + refresh_tokens: currentStats.refreshTokenStats, + connected_accounts: currentStats.connectedAccountStats, deleted_rows: { ...currentStats.deletedRowStats, by_table: currentStats.deletedRowsByTable, diff --git a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx index 08eb0d6816..99a92d1737 100644 --- a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx +++ b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -252,9 +253,9 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler id: params.provider_id, }, }, - data: { + data: withExternalDbSyncUpdate({ allowSignIn: data.allow_sign_in, - }, + }), }); if (data.allow_sign_in) { @@ -296,9 +297,9 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler id: params.provider_id, }, }, - data: { + data: withExternalDbSyncUpdate({ allowConnectedAccounts: data.allow_connected_accounts, - }, + }), }); } @@ -309,10 +310,10 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler id: params.provider_id, }, }, - data: { + data: withExternalDbSyncUpdate({ email: data.email, providerAccountId: data.account_id, - }, + }), }); const providerType = resolveProviderType(auth.tenancy, existingOAuthAccount.configOAuthProviderId) @@ -356,6 +357,12 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler }); } + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUserOAuthAccount", + tenancyId: auth.tenancy.id, + oauthAccountId: params.provider_id, + }); + await tx.projectUserOAuthAccount.delete({ where: { tenancyId_id: { diff --git a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx index 57e1a162e4..db417b7af5 100644 --- a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx +++ b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx @@ -120,10 +120,12 @@ export const POST = createSmartRouteHandler({ refreshTokenId, startedAt: new Date(firstMs), lastEventAt: new Date(newLastEventAtMs), + shouldUpdateSequenceId: true, }, update: { startedAt: new Date(newStartedAtMs), lastEventAt: new Date(newLastEventAtMs), + shouldUpdateSequenceId: true, }, }); @@ -197,6 +199,11 @@ export const POST = createSmartRouteHandler({ throw e; } + await prisma.sessionReplay.update({ + where: { tenancyId_id: { tenancyId, id: replayId } }, + data: { shouldUpdateSequenceId: true }, + }); + return { statusCode: 200, bodyType: "json", diff --git a/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx b/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx index 1e39098801..552a49484d 100644 --- a/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx +++ b/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx @@ -1,4 +1,5 @@ import { Prisma } from "@/generated/prisma/client"; +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureTeamExists, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; @@ -146,10 +147,10 @@ export const teamMemberProfilesCrudHandlers = createLazyProxy(() => createCrudHa teamId: params.team_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-member-profile-images") - }, + }), include: fullInclude, }); diff --git a/apps/backend/src/app/api/latest/team-memberships/crud.tsx b/apps/backend/src/app/api/latest/team-memberships/crud.tsx index ae11b32ef5..ae3e6f117b 100644 --- a/apps/backend/src/app/api/latest/team-memberships/crud.tsx +++ b/apps/backend/src/app/api/latest/team-memberships/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamPermissionDeletionsForTeamMember, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultTeamPermissions } from "@/lib/permissions"; import { ensureTeamExists, ensureTeamMembershipDoesNotExist, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -20,11 +21,11 @@ export async function addUserToTeam(tx: PrismaTransaction, options: { type: 'member' | 'creator', }) { await tx.teamMember.create({ - data: { + data: withExternalDbSyncUpdate({ projectUserId: options.userId, teamId: options.teamId, tenancyId: options.tenancy.id, - }, + }), }); const result = await grantDefaultTeamPermissions(tx, { @@ -138,6 +139,19 @@ export const teamMembershipsCrudHandlers = createLazyProxy(() => createCrudHandl userId: params.user_id, }); + await recordExternalDbSyncTeamPermissionDeletionsForTeamMember(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + teamId: params.team_id, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "TeamMember", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + teamId: params.team_id, + }); + await tx.teamMember.delete({ where: { tenancyId_projectUserId_teamId: { diff --git a/apps/backend/src/app/api/latest/teams/crud.tsx b/apps/backend/src/app/api/latest/teams/crud.tsx index 05c7ca1772..6ae06487fd 100644 --- a/apps/backend/src/app/api/latest/teams/crud.tsx +++ b/apps/backend/src/app/api/latest/teams/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamInvitationDeletionsForTeam, recordExternalDbSyncTeamMemberDeletionsForTeam, recordExternalDbSyncTeamPermissionDeletionsForTeam, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureTeamExists, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { sendTeamCreatedWebhook, sendTeamDeletedWebhook, sendTeamUpdatedWebhook } from "@/lib/webhooks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -73,7 +74,7 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC const db = await retryTransaction(prisma, async (tx) => { const db = await tx.team.create({ - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, mirroredProjectId: auth.project.id, mirroredBranchId: auth.branchId, @@ -81,8 +82,8 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, serverMetadata: data.server_metadata === null ? Prisma.JsonNull : data.server_metadata, - profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images") - }, + profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images"), + }), }); if (addUserId) { @@ -160,13 +161,13 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC teamId: params.team_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, serverMetadata: data.server_metadata === null ? Prisma.JsonNull : data.server_metadata, - profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images") - }, + profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images"), + }), }); }); @@ -194,6 +195,27 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC } await ensureTeamExists(tx, { tenancyId: auth.tenancy.id, teamId: params.team_id }); + await recordExternalDbSyncTeamPermissionDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + + await recordExternalDbSyncTeamInvitationDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + + await recordExternalDbSyncTeamMemberDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "Team", + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + await tx.team.delete({ where: { tenancyId_teamId: { diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index c118f09b19..23cf68199f 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,7 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; -import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncNotificationPreferenceDeletionsForUser, recordExternalDbSyncOAuthAccountDeletionsForUser, recordExternalDbSyncProjectPermissionDeletionsForUser, recordExternalDbSyncRefreshTokenDeletionsForUser, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -1133,9 +1133,9 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC }, displayName: personalTeamDefaultDisplayName, }, - data: { + data: withExternalDbSyncUpdate({ displayName: getPersonalTeamDisplayName(data.display_name ?? null, data.primary_email ?? null), - }, + }), }); } @@ -1202,6 +1202,11 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC // if user password changed, reset all refresh tokens if (passwordHash !== undefined) { + await recordExternalDbSyncRefreshTokenDeletionsForUser(globalPrismaClient, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: auth.tenancy.id, @@ -1248,6 +1253,43 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }); + await recordExternalDbSyncTeamMemberDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncTeamPermissionDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncProjectPermissionDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncNotificationPreferenceDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncRefreshTokenDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncOAuthAccountDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await tx.projectUserRefreshToken.deleteMany({ + where: { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/email-queue-step.tsx b/apps/backend/src/lib/email-queue-step.tsx index 5cdcd43277..453c1f0bd9 100644 --- a/apps/backend/src/lib/email-queue-step.tsx +++ b/apps/backend/src/lib/email-queue-step.tsx @@ -124,6 +124,7 @@ async function retryEmailsStuckInRendering(): Promise { data: { renderedByWorkerId: null, startedRenderingAt: null, + shouldUpdateSequenceId: true, }, }); if (res.length > 0) { @@ -241,7 +242,8 @@ async function claimEmailsForRendering(workerId: string): Promise UPDATE "EmailOutbox" AS e SET "renderedByWorkerId" = ${workerId}::uuid, - "startedRenderingAt" = NOW() + "startedRenderingAt" = NOW(), + "shouldUpdateSequenceId" = TRUE FROM selected WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" RETURNING e.*; @@ -327,6 +329,7 @@ async function renderTenancyEmails(workerId: string, tenancyId: string, group: E renderErrorInternalMessage: error, renderErrorInternalDetails: { error }, finishedRenderingAt: new Date(), + shouldUpdateSequenceId: true, }, }); }; @@ -347,6 +350,7 @@ async function renderTenancyEmails(workerId: string, tenancyId: string, group: E renderErrorInternalMessage: null, renderErrorInternalDetails: Prisma.DbNull, finishedRenderingAt: new Date(), + shouldUpdateSequenceId: true, }, }); }; @@ -437,7 +441,7 @@ async function queueReadyEmails(): Promise<{ queuedCount: number }> { // Query 1: Fresh emails (scheduledAt has passed, no retry pending) const freshEmails = await globalPrismaClient.$queryRaw<{ id: string }[]>` UPDATE "EmailOutbox" - SET "isQueued" = TRUE + SET "isQueued" = TRUE, "shouldUpdateSequenceId" = TRUE WHERE "isQueued" = FALSE AND "isPaused" = FALSE AND "skippedReason" IS NULL @@ -452,7 +456,7 @@ async function queueReadyEmails(): Promise<{ queuedCount: number }> { // Clear nextSendRetryAt when queuing so the email is in a clean "queued" state. const retryEmails = await globalPrismaClient.$queryRaw<{ id: string }[]>` UPDATE "EmailOutbox" - SET "isQueued" = TRUE, "nextSendRetryAt" = NULL + SET "isQueued" = TRUE, "nextSendRetryAt" = NULL, "shouldUpdateSequenceId" = TRUE WHERE "isQueued" = FALSE AND "isPaused" = FALSE AND "skippedReason" IS NULL @@ -528,7 +532,8 @@ async function claimEmailsForSending(tx: PrismaClientTransaction, tenancyId: str FOR UPDATE SKIP LOCKED ) UPDATE "EmailOutbox" AS e - SET "startedSendingAt" = NOW() + SET "startedSendingAt" = NOW(), + "shouldUpdateSequenceId" = TRUE FROM selected WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" RETURNING e.*; @@ -678,6 +683,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendRetries: newAttemptCount, nextSendRetryAt: new Date(Date.now() + backoffMs), sendAttemptErrors: updatedErrors as Prisma.InputJsonArray, + shouldUpdateSequenceId: true, }, }); } else { @@ -718,6 +724,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO failureReason, allAttemptErrors: updatedErrors as Json[], }, + shouldUpdateSequenceId: true, }, }); } @@ -738,6 +745,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendServerErrorExternalDetails: Prisma.DbNull, sendServerErrorInternalMessage: null, sendServerErrorInternalDetails: Prisma.DbNull, + shouldUpdateSequenceId: true, }, }); } @@ -758,6 +766,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendServerErrorExternalDetails: {}, sendServerErrorInternalMessage: errorToNiceString(error), sendServerErrorInternalDetails: {}, + shouldUpdateSequenceId: true, }, }); } @@ -843,6 +852,7 @@ async function markSkipped(row: EmailOutbox, reason: EmailOutboxSkippedReason, d data: { skippedReason: reason, skippedDetails: details as Prisma.InputJsonValue, + shouldUpdateSequenceId: true, }, }); } diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 62ac6536bd..d638911a86 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -41,82 +41,784 @@ type ExternalDbSyncTarget = tenancyId: string, projectUserId: string, contactChannelId: string, + } + | { + tableName: "Team", + tenancyId: string, + teamId: string, + } + | { + tableName: "TeamMember", + tenancyId: string, + projectUserId: string, + teamId: string, + } + | { + tableName: "TeamMemberDirectPermission", + tenancyId: string, + permissionDbId: string, + } + | { + tableName: "ProjectUserDirectPermission", + tenancyId: string, + permissionDbId: string, + } + | { + tableName: "UserNotificationPreference", + tenancyId: string, + notificationPreferenceId: string, + } + | { + tableName: "VerificationCode_TEAM_INVITATION", + tenancyId: string, + verificationCodeProjectId: string, + verificationCodeBranchId: string, + verificationCodeId: string, + } + | { + tableName: "ProjectUserRefreshToken", + tenancyId: string, + refreshTokenId: string, + } + | { + tableName: "ProjectUserOAuthAccount", + tenancyId: string, + oauthAccountId: string, + }; + +type ExternalDbType = NonNullable["type"]>; +type DbSyncMapping = typeof DEFAULT_DB_SYNC_MAPPINGS[keyof typeof DEFAULT_DB_SYNC_MAPPINGS]; + +export function withExternalDbSyncUpdate(data: T): T & { shouldUpdateSequenceId: true } { + return { + ...data, + shouldUpdateSequenceId: true, }; +} + +export async function markProjectUserForExternalDbSync( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + } +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + await tx.projectUser.update({ + where: { + tenancyId_projectUserId: { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, + }, + }, + data: { + shouldUpdateSequenceId: true, + }, + }); +} + +export async function recordExternalDbSyncDeletion( + tx: ExternalDbSyncClient, + target: ExternalDbSyncTarget, +): Promise { + assertUuid(target.tenancyId, "tenancyId"); + + if (target.tableName === "ProjectUser") { + assertUuid(target.projectUserId, "projectUserId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUser', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId"), + to_jsonb("ProjectUser".*), + NOW(), + TRUE + FROM "ProjectUser" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUser, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "ContactChannel") { + assertUuid(target.projectUserId, "projectUserId"); + assertUuid(target.contactChannelId, "contactChannelId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "id" = ${target.contactChannelId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "Team") { + assertUuid(target.teamId, "teamId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'Team', + jsonb_build_object('tenancyId', "tenancyId", 'teamId', "teamId"), + to_jsonb("Team".*), + NOW(), + TRUE + FROM "Team" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "teamId" = ${target.teamId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for Team, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "TeamMember") { + assertUuid(target.projectUserId, "projectUserId"); + assertUuid(target.teamId, "teamId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), + NOW(), + TRUE + FROM "TeamMember" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "teamId" = ${target.teamId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for TeamMember, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "TeamMemberDirectPermission") { + assertUuid(target.permissionDbId, "permissionDbId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "id" = ${target.permissionDbId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for TeamMemberDirectPermission, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "ProjectUserDirectPermission") { + assertUuid(target.permissionDbId, "permissionDbId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'permissionId', "permissionId" + ), + to_jsonb("ProjectUserDirectPermission".*), + NOW(), + TRUE + FROM "ProjectUserDirectPermission" + WHERE "id" = ${target.permissionDbId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserDirectPermission, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "UserNotificationPreference") { + assertUuid(target.notificationPreferenceId, "notificationPreferenceId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'UserNotificationPreference', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'id', "id" + ), + to_jsonb("UserNotificationPreference".*), + NOW(), + TRUE + FROM "UserNotificationPreference" + WHERE "id" = ${target.notificationPreferenceId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for UserNotificationPreference, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "ProjectUserRefreshToken") { + assertUuid(target.refreshTokenId, "refreshTokenId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserRefreshToken', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserRefreshToken".*), + NOW(), + TRUE + FROM "ProjectUserRefreshToken" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "id" = ${target.refreshTokenId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserRefreshToken, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "ProjectUserOAuthAccount") { + assertUuid(target.oauthAccountId, "oauthAccountId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserOAuthAccount', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserOAuthAccount".*), + NOW(), + TRUE + FROM "ProjectUserOAuthAccount" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "id" = ${target.oauthAccountId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserOAuthAccount, got ${insertedCount}.` + ); + } + return; + } + + { + const _verificationCodeTarget: { tableName: "VerificationCode_TEAM_INVITATION" } = target; + assertNonEmptyString(target.verificationCodeProjectId, "verificationCodeProjectId"); + assertNonEmptyString(target.verificationCodeBranchId, "verificationCodeBranchId"); + assertUuid(target.verificationCodeId, "verificationCodeId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "Tenancy"."id", + 'VerificationCode_TEAM_INVITATION', + jsonb_build_object('id', "VerificationCode"."id"), + to_jsonb("VerificationCode".*), + NOW(), + TRUE + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + WHERE "Tenancy"."id" = ${target.tenancyId}::uuid + AND "VerificationCode"."projectId" = ${target.verificationCodeProjectId} + AND "VerificationCode"."branchId" = ${target.verificationCodeBranchId} + AND "VerificationCode"."id" = ${target.verificationCodeId}::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + FOR UPDATE OF "VerificationCode" + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for VerificationCode_TEAM_INVITATION, got ${insertedCount}.` + ); + } + return; + } +} + +export async function recordExternalDbSyncContactChannelDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamMemberDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), + NOW(), + TRUE + FROM "TeamMember" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamPermissionDeletionsForTeamMember( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamPermissionDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); +} -type ExternalDbType = NonNullable["type"]>; -type DbSyncMapping = typeof DEFAULT_DB_SYNC_MAPPINGS[keyof typeof DEFAULT_DB_SYNC_MAPPINGS]; +export async function recordExternalDbSyncTeamPermissionDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); -export function withExternalDbSyncUpdate(data: T): T & { shouldUpdateSequenceId: true } { - return { - ...data, - shouldUpdateSequenceId: true, - }; + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); } -export async function markProjectUserForExternalDbSync( +export async function recordExternalDbSyncTeamInvitationDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "Tenancy"."id", + 'VerificationCode_TEAM_INVITATION', + jsonb_build_object('id', "VerificationCode"."id"), + to_jsonb("VerificationCode".*), + NOW(), + TRUE + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + WHERE "Tenancy"."id" = ${options.tenancyId}::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + AND "VerificationCode"."data"->>'team_id' = ${options.teamId} + FOR UPDATE OF "VerificationCode" + `); +} + +export async function recordExternalDbSyncTeamMemberDeletionsForUser( tx: ExternalDbSyncClient, options: { tenancyId: string, projectUserId: string, - } + }, ): Promise { assertUuid(options.tenancyId, "tenancyId"); assertUuid(options.projectUserId, "projectUserId"); - await tx.projectUser.update({ - where: { - tenancyId_projectUserId: { - tenancyId: options.tenancyId, - projectUserId: options.projectUserId, - }, - }, - data: { - shouldUpdateSequenceId: true, - }, - }); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), + NOW(), + TRUE + FROM "TeamMember" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); } -export async function recordExternalDbSyncDeletion( +export async function recordExternalDbSyncProjectPermissionDeletionsForUser( tx: ExternalDbSyncClient, - target: ExternalDbSyncTarget, + options: { + tenancyId: string, + projectUserId: string, + }, ): Promise { - assertUuid(target.tenancyId, "tenancyId"); - assertUuid(target.projectUserId, "projectUserId"); + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); - if (target.tableName === "ProjectUser") { - const insertedCount = await tx.$executeRaw(Prisma.sql` - INSERT INTO "DeletedRow" ( - "id", - "tenancyId", - "tableName", - "primaryKey", - "data", - "deletedAt", - "shouldUpdateSequenceId" - ) - SELECT - gen_random_uuid(), - "tenancyId", - 'ProjectUser', - jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId"), - to_jsonb("ProjectUser".*), - NOW(), - TRUE - FROM "ProjectUser" - WHERE "tenancyId" = ${target.tenancyId}::uuid - AND "projectUserId" = ${target.projectUserId}::uuid - FOR UPDATE - `); + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'permissionId', "permissionId" + ), + to_jsonb("ProjectUserDirectPermission".*), + NOW(), + TRUE + FROM "ProjectUserDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} - if (insertedCount !== 1) { - throw new StackAssertionError( - `Expected to insert 1 DeletedRow entry for ProjectUser, got ${insertedCount}.` - ); - } - return; - } +export async function recordExternalDbSyncNotificationPreferenceDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); - assertUuid(target.contactChannelId, "contactChannelId"); - const insertedCount = await tx.$executeRaw(Prisma.sql` + await tx.$executeRaw(Prisma.sql` INSERT INTO "DeletedRow" ( "id", "tenancyId", @@ -129,33 +831,63 @@ export async function recordExternalDbSyncDeletion( SELECT gen_random_uuid(), "tenancyId", - 'ContactChannel', + 'UserNotificationPreference', jsonb_build_object( - 'tenancyId', - "tenancyId", - 'projectUserId', - "projectUserId", - 'id', - "id" + 'tenancyId', "tenancyId", + 'id', "id" ), - to_jsonb("ContactChannel".*), + to_jsonb("UserNotificationPreference".*), NOW(), TRUE - FROM "ContactChannel" - WHERE "tenancyId" = ${target.tenancyId}::uuid - AND "projectUserId" = ${target.projectUserId}::uuid - AND "id" = ${target.contactChannelId}::uuid + FROM "UserNotificationPreference" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid FOR UPDATE `); +} - if (insertedCount !== 1) { - throw new StackAssertionError( - `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` - ); - } +export async function recordExternalDbSyncRefreshTokenDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + excludeRefreshToken?: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + const excludeCondition = options.excludeRefreshToken + ? Prisma.sql`AND "refreshToken" != ${options.excludeRefreshToken}` + : Prisma.sql``; + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserRefreshToken', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserRefreshToken".*), + NOW(), + TRUE + FROM "ProjectUserRefreshToken" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + ${excludeCondition} + FOR UPDATE + `); } -export async function recordExternalDbSyncContactChannelDeletionsForUser( +export async function recordExternalDbSyncOAuthAccountDeletionsForUser( tx: ExternalDbSyncClient, options: { tenancyId: string, @@ -178,19 +910,12 @@ export async function recordExternalDbSyncContactChannelDeletionsForUser( SELECT gen_random_uuid(), "tenancyId", - 'ContactChannel', - jsonb_build_object( - 'tenancyId', - "tenancyId", - 'projectUserId', - "projectUserId", - 'id', - "id" - ), - to_jsonb("ContactChannel".*), + 'ProjectUserOAuthAccount', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserOAuthAccount".*), NOW(), TRUE - FROM "ContactChannel" + FROM "ProjectUserOAuthAccount" WHERE "tenancyId" = ${options.tenancyId}::uuid AND "projectUserId" = ${options.projectUserId}::uuid FOR UPDATE @@ -325,7 +1050,7 @@ async function pushRowsToExternalDb( } } -function getInternalDbFetchQuery(mapping: DbSyncMapping, dbType: ExternalDbType) { +function getInternalDbFetchQuery(mapping: DbSyncMapping) { return mapping.internalDbFetchQuery; } @@ -343,11 +1068,18 @@ function normalizeClickhouseBoolean(value: unknown, label: string): number { throw new StackAssertionError(`${label} must be a boolean or 0/1. Received: ${JSON.stringify(value)}`); } +function normalizeClickhouseNullableBoolean(value: unknown, label: string): number | null { + if (value === null || value === undefined) { + return null; + } + return normalizeClickhouseBoolean(value, label); +} + function parseSequenceId(value: unknown, mappingId: string): number | null { if (value == null) { return null; } - const seqNum = typeof value === "bigint" ? Number(value) : Number(value); + const seqNum = Number(value); if (!Number.isFinite(seqNum)) { throw new StackAssertionError( `Invalid sequence_id for mapping ${mappingId}: ${JSON.stringify(value)}` @@ -372,6 +1104,64 @@ async function ensureClickhouseSchema( } } +// Map of target table name -> column normalizers for ClickHouse +// 'json' columns get JSON.stringify, 'boolean' columns get normalizeClickhouseBoolean, 'bigint' columns get Number() +export const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { + users: { + client_metadata: 'json', + client_read_only_metadata: 'json', + server_metadata: 'json', + primary_email_verified: 'boolean', + is_anonymous: 'boolean', + restricted_by_admin: 'boolean', + sync_is_deleted: 'boolean', + }, + contact_channels: { + is_primary: 'boolean', + is_verified: 'boolean', + used_for_auth: 'boolean', + sync_is_deleted: 'boolean', + }, + teams: { + client_metadata: 'json', + client_read_only_metadata: 'json', + server_metadata: 'json', + sync_is_deleted: 'boolean', + }, + team_member_profiles: { + sync_is_deleted: 'boolean', + }, + team_permissions: { + sync_is_deleted: 'boolean', + }, + team_invitations: { + expires_at_millis: 'bigint', + sync_is_deleted: 'boolean', + }, + email_outboxes: { + is_high_priority: 'boolean', + is_transactional: 'nullable_boolean', + can_have_delivery_info: 'nullable_boolean', + skipped_details: 'json', + is_paused: 'boolean', + sync_is_deleted: 'boolean', + }, + project_permissions: { + sync_is_deleted: 'boolean', + }, + notification_preferences: { + enabled: 'boolean', + sync_is_deleted: 'boolean', + }, + refresh_tokens: { + is_impersonation: 'boolean', + sync_is_deleted: 'boolean', + }, + connected_accounts: { + sync_is_deleted: 'boolean', + }, +}; + async function pushRowsToClickhouse( client: ClickHouseClient, tableName: string, @@ -390,6 +1180,10 @@ async function pushRowsToClickhouse( const sampleRow = newRows[0] ?? throwErr("Expected at least one row for ClickHouse sync."); const orderedKeys = Object.keys(omit(sampleRow, ["tenancyId"])); + // Derive the target table name from the full tableName (e.g. "analytics_internal.users" -> "users") + const targetTable = tableName.includes('.') ? tableName.split('.').pop()! : tableName; + const normalizers = CLICKHOUSE_COLUMN_NORMALIZERS[targetTable] ?? {}; + const normalizedRows = newRows.map((row) => { const tenancyIdValue = row.tenancyId; if (typeof tenancyIdValue !== "string") { @@ -427,17 +1221,27 @@ async function pushRowsToClickhouse( `sync_sequence_id must be defined for ClickHouse sync. Mapping: ${mappingId}` ); } - return { + + const normalized: Record = { ...rest, sync_sequence_id: sequenceId, - client_metadata: JSON.stringify(rest.client_metadata), - client_read_only_metadata: JSON.stringify(rest.client_read_only_metadata), - server_metadata: JSON.stringify(rest.server_metadata), - primary_email_verified: normalizeClickhouseBoolean(rest.primary_email_verified, "primary_email_verified"), - is_anonymous: normalizeClickhouseBoolean(rest.is_anonymous, "is_anonymous"), - restricted_by_admin: normalizeClickhouseBoolean(rest.restricted_by_admin, "restricted_by_admin"), - sync_is_deleted: normalizeClickhouseBoolean(rest.sync_is_deleted, "sync_is_deleted"), }; + + for (const [col, type] of Object.entries(normalizers)) { + if (col in normalized) { + if (type === 'json') { + normalized[col] = JSON.stringify(normalized[col]); + } else if (type === 'nullable_boolean') { + normalized[col] = normalizeClickhouseNullableBoolean(normalized[col], col); + } else if (type === 'bigint') { + normalized[col] = Number(normalized[col]); + } else { + normalized[col] = normalizeClickhouseBoolean(normalized[col], col); + } + } + } + + return normalized; }); await client.insert({ @@ -523,7 +1327,7 @@ async function syncPostgresMapping( assertNonEmptyString(mappingId, "mappingId"); assertNonEmptyString(mapping.targetTable, "mapping.targetTable"); assertUuid(tenancyId, "tenancyId"); - const fetchQuery = getInternalDbFetchQuery(mapping, "postgres"); + const fetchQuery = getInternalDbFetchQuery(mapping); const updateQuery = mapping.externalDbUpdateQueries.postgres; const tableName = mapping.targetTable; assertNonEmptyString(fetchQuery, "internalDbFetchQuery"); @@ -628,8 +1432,6 @@ async function syncClickhouseMapping( } const clickhouseTableName = `analytics_internal.${mapping.targetTable}`; - await ensureClickhouseSchema(client, tableSchema, clickhouseTableName); - let lastSequenceId = await getClickhouseLastSyncedSequenceId(client, tenancyId, mappingId); const BATCH_LIMIT = 1000; diff --git a/apps/backend/src/lib/permissions.tsx b/apps/backend/src/lib/permissions.tsx index 30d5441bf0..7429ead3a6 100644 --- a/apps/backend/src/lib/permissions.tsx +++ b/apps/backend/src/lib/permissions.tsx @@ -7,6 +7,7 @@ import { groupBy } from "@stackframe/stack-shared/dist/utils/arrays"; import { getOrUndefined, has, typedEntries, typedFromEntries } from "@stackframe/stack-shared/dist/utils/objects"; import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings"; import { overrideEnvironmentConfigOverride } from "./config"; +import { recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "./external-db-sync"; import { Tenancy } from "./tenancies"; import { PrismaTransaction } from "./types"; @@ -122,13 +123,13 @@ export async function grantTeamPermission( permissionId: options.permissionId, }, }, - create: { + create: withExternalDbSyncUpdate({ tenancyId: options.tenancy.id, projectUserId: options.userId, teamId: options.teamId, permissionId: options.permissionId, - }, - update: {}, + }), + update: withExternalDbSyncUpdate({}), }); return { @@ -147,6 +148,24 @@ export async function revokeTeamPermission( permissionId: string, } ) { + const permissionRecord = await tx.teamMemberDirectPermission.findUniqueOrThrow({ + where: { + tenancyId_projectUserId_teamId_permissionId: { + tenancyId: options.tenancy.id, + projectUserId: options.userId, + teamId: options.teamId, + permissionId: options.permissionId, + }, + }, + select: { id: true }, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "TeamMemberDirectPermission", + tenancyId: options.tenancy.id, + permissionDbId: permissionRecord.id, + }); + await tx.teamMemberDirectPermission.delete({ where: { tenancyId_projectUserId_teamId_permissionId: { @@ -313,9 +332,9 @@ export async function updatePermissionDefinition( tenancyId: options.tenancy.id, permissionId: options.oldId, }, - data: { + data: withExternalDbSyncUpdate({ permissionId: newId, - }, + }), }); await sourceOfTruthTx.projectUserDirectPermission.updateMany({ @@ -323,9 +342,9 @@ export async function updatePermissionDefinition( tenancyId: options.tenancy.id, permissionId: options.oldId, }, - data: { + data: withExternalDbSyncUpdate({ permissionId: newId, - }, + }), }); return { @@ -419,6 +438,20 @@ export async function deletePermissionDefinition( }, }); } else { + const projectPermissions = await sourceOfTruthTx.projectUserDirectPermission.findMany({ + where: { + tenancyId: options.tenancy.id, + permissionId: options.permissionId, + }, + select: { id: true }, + }); + for (const perm of projectPermissions) { + await recordExternalDbSyncDeletion(sourceOfTruthTx, { + tableName: "ProjectUserDirectPermission", + tenancyId: options.tenancy.id, + permissionDbId: perm.id, + }); + } await sourceOfTruthTx.projectUserDirectPermission.deleteMany({ where: { tenancyId: options.tenancy.id, @@ -452,12 +485,12 @@ export async function grantProjectPermission( permissionId: options.permissionId, }, }, - create: { + create: withExternalDbSyncUpdate({ permissionId: options.permissionId, projectUserId: options.userId, tenancyId: options.tenancy.id, - }, - update: {}, + }), + update: withExternalDbSyncUpdate({}), }); return { @@ -474,6 +507,23 @@ export async function revokeProjectPermission( permissionId: string, } ) { + const permissionRecord = await tx.projectUserDirectPermission.findUniqueOrThrow({ + where: { + tenancyId_projectUserId_permissionId: { + tenancyId: options.tenancy.id, + projectUserId: options.userId, + permissionId: options.permissionId, + }, + }, + select: { id: true }, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUserDirectPermission", + tenancyId: options.tenancy.id, + permissionDbId: permissionRecord.id, + }); + await tx.projectUserDirectPermission.delete({ where: { tenancyId_projectUserId_permissionId: { diff --git a/apps/backend/src/lib/tokens.tsx b/apps/backend/src/lib/tokens.tsx index e9c817f797..a7dcfd4101 100644 --- a/apps/backend/src/lib/tokens.tsx +++ b/apps/backend/src/lib/tokens.tsx @@ -267,10 +267,10 @@ export async function generateAccessTokenFromRefreshTokenIfValid(options: Refres id: options.refreshTokenObj.id, }, }, - data: { + data: withExternalDbSyncUpdate({ lastActiveAt: now, lastActiveAtIpInfo: ipInfo ?? undefined, - }, + }), }), ]); diff --git a/apps/backend/src/oauth/model.tsx b/apps/backend/src/oauth/model.tsx index 4953c2331b..e83af3a229 100644 --- a/apps/backend/src/oauth/model.tsx +++ b/apps/backend/src/oauth/model.tsx @@ -1,6 +1,7 @@ import { createMfaRequiredError } from "@/app/api/latest/auth/mfa/sign-in/verification-code-handler"; import { usersCrudHandlers } from "@/app/api/latest/users/crud"; import { Prisma } from "@/generated/prisma/client"; +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { checkApiKeySet } from "@/lib/internal-api-keys"; import { isAcceptedNativeAppUrl, validateRedirectUrl } from "@/lib/redirect-urls"; import { getSoleTenancyFromProjectBranch, getTenancy } from "@/lib/tenancies"; @@ -178,10 +179,10 @@ export class OAuthModel implements AuthorizationCodeModel { id: user.refreshTokenId, }, }, - update: { + update: withExternalDbSyncUpdate({ refreshToken: token.refreshToken, expiresAt: token.refreshTokenExpiresAt, - }, + }), create: { refreshToken: token.refreshToken, tenancyId: tenancy.id, diff --git a/apps/backend/src/route-handlers/verification-code-handler.tsx b/apps/backend/src/route-handlers/verification-code-handler.tsx index 3bb55f18d5..5444e436a1 100644 --- a/apps/backend/src/route-handlers/verification-code-handler.tsx +++ b/apps/backend/src/route-handlers/verification-code-handler.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { validateRedirectUrl } from "@/lib/redirect-urls"; import { getSoleTenancyFromProjectBranch, getTenancy, Tenancy } from "@/lib/tenancies"; import { globalPrismaClient } from "@/prisma-client"; @@ -272,16 +273,27 @@ export function createVerificationCodeHandler< return codes.map(code => createCodeObjectFromPrismaCode(code)); }, - async revokeCode(options) { - const { project, branchId } = parseProjectBranchCombo(options); + async revokeCode(revokeOptions) { + const { project, branchId } = parseProjectBranchCombo(revokeOptions); const tenancy = await getSoleTenancyFromProjectBranch(project.id, branchId); + // Record deletion for external DB sync if this is a TEAM_INVITATION code + if (options.type === 'TEAM_INVITATION') { + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "VerificationCode_TEAM_INVITATION", + tenancyId: tenancy.id, + verificationCodeProjectId: project.id, + verificationCodeBranchId: branchId, + verificationCodeId: revokeOptions.id, + }); + } + await globalPrismaClient.verificationCode.delete({ where: { projectId_branchId_id: { projectId: project.id, branchId, - id: options.id, + id: revokeOptions.id, }, }, }); diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx index c1e7d57951..02c2bd30a5 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx @@ -80,6 +80,15 @@ type ExternalDbSyncStatus = { sequencer: { project_users: SequenceStats, contact_channels: SequenceStats, + teams: SequenceStats, + team_members: SequenceStats, + team_permissions: SequenceStats, + team_invitations: SequenceStats, + email_outboxes: SequenceStats, + project_permissions: SequenceStats, + notification_preferences: SequenceStats, + refresh_tokens: SequenceStats, + connected_accounts: SequenceStats, deleted_rows: DeletedRowStats, }, poller: PollerStats, @@ -95,6 +104,15 @@ type ExternalDbSyncStatus = { sequencer: { project_users: SequenceStats, contact_channels: SequenceStats, + teams: SequenceStats, + team_members: SequenceStats, + team_permissions: SequenceStats, + team_invitations: SequenceStats, + email_outboxes: SequenceStats, + project_permissions: SequenceStats, + notification_preferences: SequenceStats, + refresh_tokens: SequenceStats, + connected_accounts: SequenceStats, deleted_rows: DeletedRowStats, }, poller: PollerStats, @@ -400,6 +418,15 @@ export default function PageClient() { const sequencerPending = sumBigIntStrings([ summarySource.sequencer.project_users.pending, summarySource.sequencer.contact_channels.pending, + summarySource.sequencer.teams.pending, + summarySource.sequencer.team_members.pending, + summarySource.sequencer.team_permissions.pending, + summarySource.sequencer.team_invitations.pending, + summarySource.sequencer.email_outboxes.pending, + summarySource.sequencer.project_permissions.pending, + summarySource.sequencer.notification_preferences.pending, + summarySource.sequencer.refresh_tokens.pending, + summarySource.sequencer.connected_accounts.pending, summarySource.sequencer.deleted_rows.pending, ]); const mappingPending = sumBigIntStrings( @@ -518,7 +545,7 @@ export default function PageClient() { -
ProjectUser + ContactChannel + DeletedRow rows waiting for sequence IDs.
+
All synced table rows waiting for sequence IDs.
Throughput {loadingState ? "—" : formatThroughput(throughputStats?.sequencer ?? null)} @@ -578,30 +605,29 @@ export default function PageClient() { - - ProjectUser - - - - - - - - ContactChannel - - - - - - - - DeletedRow - - - - - - + {([ + ["ProjectUser", status?.sequencer.project_users], + ["ContactChannel", status?.sequencer.contact_channels], + ["Team", status?.sequencer.teams], + ["TeamMember", status?.sequencer.team_members], + ["TeamPermission", status?.sequencer.team_permissions], + ["TeamInvitation", status?.sequencer.team_invitations], + ["EmailOutbox", status?.sequencer.email_outboxes], + ["ProjectPermission", status?.sequencer.project_permissions], + ["NotificationPref", status?.sequencer.notification_preferences], + ["RefreshToken", status?.sequencer.refresh_tokens], + ["ConnectedAccount", status?.sequencer.connected_accounts], + ["DeletedRow", status?.sequencer.deleted_rows], + ] as const).map(([name, stats]) => ( + + {name} + + + + + + + ))} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index 01d0242388..afcdc93243 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -522,7 +522,17 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON SQLite FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON URL FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.connected_accounts TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.notification_preferences TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.project_permissions TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.refresh_tokens TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_permissions TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.teams TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.users TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.aggregate_function_combinators TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.collations TO limited_user" }, @@ -561,10 +571,50 @@ it("can see only some tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { + "database": "default", + "name": "connected_accounts", + }, + { + "database": "default", + "name": "contact_channels", + }, + { + "database": "default", + "name": "email_outboxes", + }, { "database": "default", "name": "events", }, + { + "database": "default", + "name": "notification_preferences", + }, + { + "database": "default", + "name": "project_permissions", + }, + { + "database": "default", + "name": "refresh_tokens", + }, + { + "database": "default", + "name": "team_invitations", + }, + { + "database": "default", + "name": "team_member_profiles", + }, + { + "database": "default", + "name": "team_permissions", + }, + { + "database": "default", + "name": "teams", + }, { "database": "default", "name": "users", @@ -586,7 +636,17 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { "name": "connected_accounts" }, + { "name": "contact_channels" }, + { "name": "email_outboxes" }, { "name": "events" }, + { "name": "notification_preferences" }, + { "name": "project_permissions" }, + { "name": "refresh_tokens" }, + { "name": "team_invitations" }, + { "name": "team_member_profiles" }, + { "name": "team_permissions" }, + { "name": "teams" }, { "name": "users" }, ], }, @@ -1068,7 +1128,17 @@ it("shows grants", async ({ expect }) => { "status": 200, "body": { "result": [ + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.connected_accounts TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.notification_preferences TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.project_permissions TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.refresh_tokens TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_permissions TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.teams TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.users TO limited_user" }, ], }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 31aaf597a1..191c31068d 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -2,15 +2,38 @@ import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors" import { wait } from "@stackframe/stack-shared/dist/utils/promises"; import { afterAll, beforeAll, describe, expect } from 'vitest'; import { test } from '../../../../helpers'; -import { InternalApiKey, Project, User, niceBackendFetch } from '../../../backend-helpers'; +import { withPortPrefix } from '../../../../helpers/ports'; +import { Auth, backendContext, InternalApiKey, Project, User, niceBackendFetch } from '../../../backend-helpers'; +import { randomUUID } from 'node:crypto'; import { TEST_TIMEOUT, TestDbManager, createProjectWithExternalDb as createProjectWithExternalDbRaw, verifyInExternalDb, verifyNotInExternalDb, + waitForSyncedContactChannel, + waitForSyncedContactChannelDeletion, + waitForSyncedConnectedAccount, + waitForSyncedConnectedAccountDeletion, waitForSyncedData, waitForSyncedDeletion, + waitForSyncedEmailOutbox, + waitForSyncedEmailOutboxByStatus, + waitForSyncedRefreshToken, + waitForSyncedRefreshTokenDeletion, + + waitForSyncedTeam, + waitForSyncedTeamDeletion, + waitForSyncedTeamInvitation, + waitForSyncedTeamInvitationDeletion, + waitForSyncedTeamMember, + waitForSyncedTeamMemberDeletion, + waitForSyncedTeamPermission, + waitForSyncedTeamPermissionDeletion, + waitForSyncedProjectPermission, + waitForSyncedProjectPermissionDeletion, + waitForCondition, + waitForSyncedNotificationPreference, waitForTable } from './external-db-sync-utils'; @@ -81,7 +104,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { let dbManager: TestDbManager; const createProjectWithExternalDb = ( externalDatabases: any, - projectOptions?: { display_name?: string, description?: string } + projectOptions?: { display_name?: string, description?: string, config?: Record } ) => { return createProjectWithExternalDbRaw( externalDatabases, @@ -577,6 +600,893 @@ describe.sequential('External DB Sync - Basic Tests', () => { }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a team, verifies it in the external DB, updates it, verifies the update, + * deletes it, and verifies the removal. + */ + test('Team CRUD sync (Postgres)', async () => { + const dbName = 'team_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + // Create a team + const createResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Sync Test Team' }, + }); + expect(createResponse.status).toBe(201); + const teamId = createResponse.body.id; + + await waitForSyncedTeam(client, 'Sync Test Team'); + + const res1 = await client.query(`SELECT * FROM "teams" WHERE "id" = $1`, [teamId]); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].display_name).toBe('Sync Test Team'); + + // Update the team + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Team Name' }, + }); + + await waitForSyncedTeam(client, 'Updated Team Name'); + + const res2 = await client.query(`SELECT * FROM "teams" WHERE "id" = $1`, [teamId]); + expect(res2.rows[0].display_name).toBe('Updated Team Name'); + + // Delete the team + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team and verifies it appears via the ClickHouse analytics query API. + */ + test('Team sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const createResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'CH Team Test' }, + }); + expect(createResponse.status).toBe(201); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT display_name FROM teams WHERE display_name = {name:String}", + params: { name: 'CH Team Test' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].display_name).toBe('CH Team Test'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user and team, adds the user as a member, verifies in external DB, + * removes the member, and verifies removal. + */ + test('TeamMember CRUD sync (Postgres)', async () => { + const dbName = 'team_member_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'tm-crud@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TM CRUD Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Add user as team member + const addMemberResponse = await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(addMemberResponse.status).toBe(201); + + await waitForSyncedTeamMember(client, teamId, user.userId); + + const res1 = await client.query(`SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, user.userId]); + expect(res1.rows.length).toBe(1); + + // Remove member + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user with a primary email and verifies the contact channel appears + * in the external DB contact_channels table. + */ + test('ContactChannel sync (Postgres)', async () => { + const dbName = 'contact_channel_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cc-sync@example.com' }); + + await waitForSyncedContactChannel(client, 'cc-sync@example.com'); + + const res = await client.query(`SELECT * FROM "contact_channels" WHERE "value" = $1`, ['cc-sync@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].user_id).toBe(user.userId); + expect(res.rows[0].is_primary).toBe(true); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user in a team, deletes the user, and verifies the team_member is gone. + */ + test('Cascade: User delete removes team members from external DB', async () => { + const dbName = 'cascade_user_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-user-del@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade User Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamMember(client, teamId, user.userId); + + // Delete the user — should cascade-delete the team member + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team with a member, deletes the team, and verifies both team and member are gone. + */ + test('Cascade: Team delete removes team and members from external DB', async () => { + const dbName = 'cascade_team_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-team-del@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamMember(client, teamId, user.userId); + await waitForSyncedTeam(client, 'Cascade Team'); + + // Delete the team — should cascade-delete the member too + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team, adds a member, grants a permission, verifies in external DB, + * revokes the permission, and verifies removal. + */ + test('TeamPermission CRUD sync (Postgres)', async () => { + const dbName = 'team_permission_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'tp-crud@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TP CRUD Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Add user as team member + const addMemberResponse = await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(addMemberResponse.status).toBe(201); + + // Grant a permission + const grantResponse = await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(grantResponse.status).toBe(201); + + await waitForSyncedTeamPermission(client, teamId, user.userId, '$read_members'); + + const res1 = await client.query(`SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, user.userId, '$read_members']); + expect(res1.rows.length).toBe(1); + + // Revoke the permission + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamPermissionDeletion(client, teamId, user.userId, '$read_members'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team + member + permission, queries ClickHouse analytics API to verify. + */ + test('TeamPermission sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const user = await User.create({ primary_email: 'tp-ch@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TP CH Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT team_id, user_id, id FROM team_permissions WHERE id = {perm:String}", + params: { perm: '$read_members' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].id).toBe('$read_members'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user, grants a project permission, verifies in external DB, + * revokes the permission, and verifies removal. + */ + test('ProjectPermission CRUD sync (Postgres)', async () => { + const dbName = 'project_permission_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + // Create a project permission definition via config + await Project.updateConfig({ + "rbac.permissions": { "test_perm": { scope: "project" } }, + }); + + const user = await User.create({ primary_email: 'pp-crud@example.com' }); + + // Grant a project permission + const grantResponse = await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/test_perm`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(grantResponse.status).toBe(201); + + await waitForSyncedProjectPermission(client, user.userId, 'test_perm'); + + const res1 = await client.query(`SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [user.userId, 'test_perm']); + expect(res1.rows.length).toBe(1); + + // Revoke the permission + await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/test_perm`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedProjectPermissionDeletion(client, user.userId, 'test_perm'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user + project permission, queries ClickHouse analytics API to verify. + */ + test('ProjectPermission sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + // Create a project permission definition via config + await Project.updateConfig({ + "rbac.permissions": { "ch_test_perm": { scope: "project" } }, + }); + + const user = await User.create({ primary_email: 'pp-ch@example.com' }); + + await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/ch_test_perm`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT user_id, id FROM project_permissions WHERE id = {perm:String}", + params: { perm: 'ch_test_perm' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].id).toBe('ch_test_perm'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user, updates a notification preference, verifies in external DB. + */ + test('NotificationPreference sync (Postgres)', async () => { + const dbName = 'notification_pref_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'np-crud@example.com' }); + + // Update a notification preference + const updateResponse = await niceBackendFetch(`/api/v1/emails/notification-preference/${user.userId}/4f6f8873-3d04-46bd-8bef-18338b1a1b4c`, { + accessType: 'admin', + method: 'PATCH', + body: { enabled: false }, + }); + expect(updateResponse.status).toBe(200); + + await waitForSyncedNotificationPreference(client, user.userId, '4f6f8873-3d04-46bd-8bef-18338b1a1b4c'); + + const res1 = await client.query(`SELECT * FROM "notification_preferences" WHERE "user_id" = $1 AND "notification_category_id" = $2`, [user.userId, '4f6f8873-3d04-46bd-8bef-18338b1a1b4c']); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].enabled).toBe(false); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user + notification preference, queries ClickHouse analytics API to verify. + */ + test('NotificationPreference sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const user = await User.create({ primary_email: 'np-ch@example.com' }); + + await niceBackendFetch(`/api/v1/emails/notification-preference/${user.userId}/4f6f8873-3d04-46bd-8bef-18338b1a1b4c`, { + accessType: 'admin', + method: 'PATCH', + body: { enabled: false }, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT user_id, notification_category_id, enabled FROM notification_preferences WHERE notification_category_id = {cat:String}", + params: { cat: '4f6f8873-3d04-46bd-8bef-18338b1a1b4c' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].notification_category_id).toBe('4f6f8873-3d04-46bd-8bef-18338b1a1b4c'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends a team invitation, verifies in external DB, revokes it, verifies removal. + */ + test('TeamInvitation sync (Postgres)', async () => { + const dbName = 'team_invitation_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { display_name: 'Invitation Test Project' }); + + const client = dbManager.getClient(dbName); + + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Invitation Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Send a team invitation + const inviteResponse = await niceBackendFetch('/api/v1/team-invitations/send-code', { + accessType: 'admin', + method: 'POST', + body: { team_id: teamId, email: 'invited@example.com', callback_url: 'http://localhost:12345/callback' }, + }); + expect(inviteResponse.status).toBe(200); + + await waitForSyncedTeamInvitation(client, 'invited@example.com'); + + const res1 = await client.query(`SELECT * FROM "team_invitations" WHERE "recipient_email" = $1`, ['invited@example.com']); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].team_display_name).toBe('Invitation Team'); + const invitationId = res1.rows[0].id; + + // Revoke the invitation + await niceBackendFetch(`/api/v1/team-invitations/${invitationId}?team_id=${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamInvitationDeletion(client, invitationId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends a team invitation, queries ClickHouse analytics API to verify. + */ + test('TeamInvitation sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'CH Invitation Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch('/api/v1/team-invitations/send-code', { + accessType: 'admin', + method: 'POST', + body: { team_id: teamId, email: 'ch-invited@example.com', callback_url: 'http://localhost:12345/callback' }, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT recipient_email, team_display_name FROM team_invitations WHERE recipient_email = {email:String}", + params: { email: 'ch-invited@example.com' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].recipient_email).toBe('ch-invited@example.com'); + expect(response!.body.result[0].team_display_name).toBe('CH Invitation Team'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team with a member and permission, deletes the team, + * verifies team, member, and permissions are all gone. + */ + test('Cascade: Team delete removes permissions and invitations from external DB', async () => { + const dbName = 'cascade_team_perm_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-perm@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade Perm Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamPermission(client, teamId, user.userId, '$read_members'); + await waitForSyncedTeam(client, 'Cascade Perm Team'); + + // Delete the team — should cascade-delete permissions too + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + await waitForSyncedTeamPermissionDeletion(client, teamId, user.userId, '$read_members'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a project with email config, sends an email, and verifies + * the email outbox row is synced to the external Postgres DB. + */ + test('EmailOutbox sync (Postgres)', async () => { + const dbName = 'email_outbox_pg_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: 'Email Outbox Sync Test', + config: { + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + // Create a user + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + // Send an email + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

Sync test email

", + subject: "DB Sync Test Email", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + // Poll the outbox API until the email appears + let emailId!: string; + await waitForCondition( + async () => { + const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { + method: "GET", + accessType: "server", + }); + if (listResponse.status !== 200 || listResponse.body.items.length === 0) return false; + emailId = listResponse.body.items[0].id; + return true; + }, + { timeoutMs: 30_000, intervalMs: 500, description: 'email to appear in outbox' } + ); + + const client = dbManager.getClient(dbName); + + // Wait for the email outbox row to appear in external DB + await waitForSyncedEmailOutbox(client, emailId); + + // Verify the synced row has expected columns + const res = await client.query(`SELECT * FROM "email_outboxes" WHERE "id" = $1`, [emailId]); + expect(res.rows.length).toBe(1); + const row = res.rows[0]; + expect(row.created_with).toBe('PROGRAMMATIC_CALL'); + expect(row.is_high_priority).toBe(false); + expect(row.is_paused).toBe(false); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a project, sends an email, and verifies the email outbox row + * is synced to ClickHouse. + */ + test('EmailOutbox sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ + config: { + magic_link_enabled: true, + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + // Create a user + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + // Send an email + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

ClickHouse sync test email

", + subject: "CH Sync Test Email", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + await InternalApiKey.createAndSetProjectKeys(); + + // Poll ClickHouse until the email_outboxes row appears + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, status, simple_status, created_with, is_high_priority FROM email_outboxes LIMIT 10", + }); + expect(response.status).toBe(200); + if (response.body.result.length >= 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBeGreaterThanOrEqual(1); + const row = response!.body.result[0]; + expect(row.created_with).toBe('programmatic-call'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends an email, waits for it to reach a terminal state, then verifies + * the status update is reflected in the external Postgres DB. + */ + test('EmailOutbox status updates are synced (Postgres)', async () => { + const dbName = 'email_outbox_status_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + config: { + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

Status sync test

", + subject: "Status Sync Test", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + const client = dbManager.getClient(dbName); + + // The email should eventually reach SENT status in the external DB + await waitForSyncedEmailOutboxByStatus(client, 'SENT'); + + const res = await client.query(`SELECT * FROM "email_outboxes" WHERE "status" = 'SENT'`); + expect(res.rows.length).toBeGreaterThanOrEqual(1); + const row = res.rows[0]; + expect(row.simple_status).toBe('OK'); + expect(row.finished_sending_at).not.toBeNull(); + expect(row.sent_at).not.toBeNull(); + expect(row.send_retries).toBe(0); + }, TEST_TIMEOUT); + /** * What it does: * - Reads the external DB sync fusebox settings. @@ -614,6 +1524,202 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); }, TEST_TIMEOUT); -}); + /** + * What it does: + * - Signs up a user (which creates a refresh token), waits for it to sync to the external DB. + * + * Why it matters: + * - Validates that refresh tokens are synced to external databases. + */ + test('Refresh token sync to external DB', async ({ expect }) => { + const dbName = 'refresh_token_sync'; + const connectionString = await dbManager.createDatabase(dbName); + await createProjectWithExternalDb({ + main: { + type: "postgres", + connectionString, + }, + }, { config: { magic_link_enabled: true } }); + + const signUpRes = await Auth.Otp.signIn(); + + // List sessions to get the session (refresh token) ID + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + expect(listRes.body.items.length).toBeGreaterThanOrEqual(1); + const sessionId = listRes.body.items[0].id; + + const client = dbManager.getClient(dbName); + await waitForSyncedRefreshToken(client, sessionId); + + const res = await client.query(`SELECT * FROM "refresh_tokens" WHERE "id" = $1`, [sessionId]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].user_id).toBe(signUpRes.userId); + expect(res.rows[0].is_impersonation).toBe(false); + expect(res.rows[0].created_at).toBeInstanceOf(Date); + expect(res.rows[0].last_used_at).toBeInstanceOf(Date); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, revokes the session, and waits for the deletion to sync. + * + * Why it matters: + * - Validates that refresh token deletions are synced to external databases. + */ + test('Refresh token deletion sync to external DB', async ({ expect }) => { + const dbName = 'refresh_token_delete_sync'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: "postgres", + connectionString, + }, + }, { config: { magic_link_enabled: true } }); + + const signUpRes = await Auth.Otp.signIn(); + + // Create a second session so we can revoke one + const newSession = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "server", + method: "POST", + body: { user_id: signUpRes.userId }, + }); + expect(newSession.status).toBe(200); + + // List sessions to find the second session ID + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + const nonCurrentSession = listRes.body.items.find((s: any) => !s.is_current_session); + expect(nonCurrentSession).toBeDefined(); + + const client = dbManager.getClient(dbName); + await waitForSyncedRefreshToken(client, nonCurrentSession.id); + + // Revoke the non-current session + const deleteRes = await niceBackendFetch(`/api/v1/auth/sessions/${nonCurrentSession.id}`, { + accessType: "client", + method: "DELETE", + query: { user_id: signUpRes.userId }, + }); + expect(deleteRes.status).toBe(200); + + await waitForSyncedRefreshTokenDeletion(client, nonCurrentSession.id); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, verifies refresh token appears in ClickHouse. + * + * Why it matters: + * - Validates ClickHouse refresh_tokens table sync. + */ + test('Refresh token sync to ClickHouse', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + await InternalApiKey.createAndSetProjectKeys(); + + const signUpRes = await Auth.Otp.signIn(); + + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + const sessionId = listRes.body.items[0].id; + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, user_id, is_impersonation FROM refresh_tokens WHERE id = {session_id:UUID}", + params: { session_id: sessionId }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + expect(response.body.result[0]).toMatchObject({ + id: sessionId, + user_id: signUpRes.userId, + is_impersonation: 0, + }); + return; + } + await wait(intervalMs); + } + throw new StackAssertionError(`Timed out waiting for ClickHouse refresh token to sync.`, { response }); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, verifies connected account appears in ClickHouse. + * + * Why it matters: + * - Validates ClickHouse connected_accounts table sync. + */ + test('Connected account sync to ClickHouse', async ({ expect }) => { + // Use default project (has spotify configured) with analytics keys + await Auth.OAuth.signIn(); + await InternalApiKey.createAndSetProjectKeys(); + + // Get the user ID + const userRes = await niceBackendFetch("/api/v1/users/me", { + accessType: "client", + method: "GET", + }); + expect(userRes.status).toBe(200); + const userId = userRes.body.id; + // Create an additional connected account via the oauth-providers API so we have a known ID + const createRes = await niceBackendFetch("/api/v1/oauth-providers", { + accessType: "server", + method: "POST", + body: { + user_id: userId, + provider_config_id: "spotify", + account_id: "ch-test-account-12345", + email: "chuser@example.com", + allow_sign_in: false, + allow_connected_accounts: true, + }, + }); + expect(createRes.status).toBe(201); + const accountId = createRes.body.id; + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT user_id, provider, provider_account_id FROM connected_accounts WHERE provider_account_id = {account_id:String} AND user_id = {user_id:UUID}", + params: { account_id: "ch-test-account-12345", user_id: userId }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + expect(response.body.result[0]).toMatchObject({ + user_id: userId, + provider: "spotify", + provider_account_id: "ch-test-account-12345", + }); + return; + } + await wait(intervalMs); + } + throw new StackAssertionError(`Timed out waiting for ClickHouse connected account to sync.`, { response }); + }, TEST_TIMEOUT); + +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 45281add2c..4c91ae93a1 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -150,59 +150,66 @@ export async function waitForCondition( } /** - * Wait for data to appear in external DB (relies on automatic cron job) + * Generic helper to wait for a row to appear or disappear in the external DB. + * Handles the common pattern of catching "table does not exist" (42P01) errors. */ -export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { - +async function waitForExternalDbRow( + client: Client, + query: string, + params: unknown[], + opts: { shouldExist: boolean, description: string, checkRow?: (row: Record) => boolean }, +) { await waitForCondition( async () => { let res; try { - res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + res = await client.query(query, params); } catch (err: any) { if (err && err.code === '42P01') { return false; } throw err; } - if (res.rows.length === 0) { - return false; + if (opts.shouldExist) { + if (res.rows.length === 0) return false; + if (opts.checkRow && !opts.checkRow(res.rows[0])) return false; + return true; } - if (expectedName && res.rows[0].display_name !== expectedName) { - return false; - } - return true; + return res.rows.length === 0; }, { - description: `data for ${email} to appear in external DB`, + description: opts.description, timeoutMs: 180000, intervalMs: 500, } ); } +/** + * Wait for data to appear in external DB (relies on automatic cron job) + */ +export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [email], + { + shouldExist: true, + description: `data for ${email} to appear in external DB`, + checkRow: expectedName ? (row) => row.display_name === expectedName : undefined, + }, + ); +} + /** * Wait for data to be removed from external DB (relies on automatic cron job) */ export async function waitForSyncedDeletion(client: Client, email: string) { - await waitForCondition( - async () => { - let res; - try { - res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); - } catch (err: any) { - if (err && err.code === '42P01') { - return false; - } - throw err; - } - return res.rows.length === 0; - }, - { - description: `data for ${email} to be removed from external DB`, - timeoutMs: 180000, - intervalMs: 500, - } + await waitForExternalDbRow( + client, + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [email], + { shouldExist: false, description: `data for ${email} to be removed from external DB` }, ); } @@ -214,7 +221,7 @@ export async function waitForTable(client: Client, tableName: string) { async () => { const res = await client.query(` SELECT EXISTS ( - SELECT FROM information_schema.tables + SELECT FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1 ); @@ -265,13 +272,197 @@ export async function countUsersInExternalDb(client: Client): Promise { } } +export async function waitForSyncedTeam(client: Client, displayName: string) { + await waitForExternalDbRow(client, `SELECT * FROM "teams" WHERE "display_name" = $1`, [displayName], { + shouldExist: true, + description: `team "${displayName}" to appear in external DB`, + }); +} + +export async function waitForSyncedTeamDeletion(client: Client, teamId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "teams" WHERE "id" = $1`, [teamId], { + shouldExist: false, + description: `team ${teamId} to be removed from external DB`, + }); +} + +export async function waitForSyncedTeamMember(client: Client, teamId: string, userId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + shouldExist: true, + description: `team member (team=${teamId}, user=${userId}) to appear in external DB`, + }); +} + +export async function waitForSyncedTeamMemberDeletion(client: Client, teamId: string, userId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + shouldExist: false, + description: `team member (team=${teamId}, user=${userId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedContactChannel(client: Client, value: string) { + await waitForExternalDbRow(client, `SELECT * FROM "contact_channels" WHERE "value" = $1`, [value], { + shouldExist: true, + description: `contact channel "${value}" to appear in external DB`, + }); +} + +export async function waitForSyncedContactChannelDeletion(client: Client, value: string) { + await waitForExternalDbRow(client, `SELECT * FROM "contact_channels" WHERE "value" = $1`, [value], { + shouldExist: false, + description: `contact channel "${value}" to be removed from external DB`, + }); +} + +export async function waitForSyncedTeamPermission(client: Client, teamId: string, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, userId, permissionId], { + shouldExist: true, + description: `team permission (team=${teamId}, user=${userId}, perm=${permissionId}) to appear in external DB`, + }); +} + +export async function waitForSyncedTeamPermissionDeletion(client: Client, teamId: string, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, userId, permissionId], { + shouldExist: false, + description: `team permission (team=${teamId}, user=${userId}, perm=${permissionId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedTeamInvitation(client: Client, recipientEmail: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_invitations" WHERE "recipient_email" = $1`, [recipientEmail], { + shouldExist: true, + description: `team invitation for "${recipientEmail}" to appear in external DB`, + }); +} + +export async function waitForSyncedTeamInvitationDeletion(client: Client, invitationId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_invitations" WHERE "id" = $1`, [invitationId], { + shouldExist: false, + description: `team invitation ${invitationId} to be removed from external DB`, + }); +} + +export async function waitForSyncedEmailOutbox(client: Client, emailId: string, expectedStatus?: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "email_outboxes" WHERE "id" = $1`, + [emailId], + { + shouldExist: true, + description: `email outbox "${emailId}" to appear in external DB`, + checkRow: expectedStatus ? (row) => row.status === expectedStatus : undefined, + }, + ); +} + +export async function waitForSyncedSessionReplay(client: Client, replayId: string, expectedChunkCount?: number) { + await waitForExternalDbRow( + client, + `SELECT * FROM "session_replays" WHERE "id" = $1`, + [replayId], + { + shouldExist: true, + description: `session replay "${replayId}" to appear in external DB`, + checkRow: expectedChunkCount == null ? undefined : (row) => Number(row.chunk_count) === expectedChunkCount, + }, + ); +} + +export async function waitForSyncedProjectPermission(client: Client, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [userId, permissionId], { + shouldExist: true, + description: `project permission (user=${userId}, perm=${permissionId}) to appear in external DB`, + }); +} + +export async function waitForSyncedProjectPermissionDeletion(client: Client, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [userId, permissionId], { + shouldExist: false, + description: `project permission (user=${userId}, perm=${permissionId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedNotificationPreference(client: Client, userId: string, notificationCategoryId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "notification_preferences" WHERE "user_id" = $1 AND "notification_category_id" = $2`, [userId, notificationCategoryId], { + shouldExist: true, + description: `notification preference (user=${userId}, category=${notificationCategoryId}) to appear in external DB`, + }); +} + +export async function waitForSyncedNotificationPreferenceDeletion(client: Client, notificationPreferenceId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "notification_preferences" WHERE "id" = $1`, [notificationPreferenceId], { + shouldExist: false, + description: `notification preference ${notificationPreferenceId} to be removed from external DB`, + }); +} + +export async function waitForSyncedRefreshToken(client: Client, refreshTokenId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "refresh_tokens" WHERE "id" = $1`, + [refreshTokenId], + { + shouldExist: true, + description: `refresh token "${refreshTokenId}" to appear in external DB`, + }, + ); +} + +export async function waitForSyncedRefreshTokenDeletion(client: Client, refreshTokenId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "refresh_tokens" WHERE "id" = $1`, + [refreshTokenId], + { + shouldExist: false, + description: `refresh token "${refreshTokenId}" to be removed from external DB`, + }, + ); +} + +export async function waitForSyncedConnectedAccount(client: Client, accountId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "connected_accounts" WHERE "id" = $1`, + [accountId], + { + shouldExist: true, + description: `connected account "${accountId}" to appear in external DB`, + }, + ); +} + +export async function waitForSyncedConnectedAccountDeletion(client: Client, accountId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "connected_accounts" WHERE "id" = $1`, + [accountId], + { + shouldExist: false, + description: `connected account "${accountId}" to be removed from external DB`, + }, + ); +} + +export async function waitForSyncedEmailOutboxByStatus(client: Client, status: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "email_outboxes" WHERE "status" = $1`, + [status], + { + shouldExist: true, + description: `email outbox with status "${status}" to appear in external DB`, + }, + ); +} + /** * Helper to create a project and update its config with external DB settings. * Tracks the project for cleanup later. */ export async function createProjectWithExternalDb( externalDatabases: any, - projectOptions?: { display_name?: string, description?: string }, + projectOptions?: { display_name?: string, description?: string, config?: Record }, options?: { projectTracker?: ProjectContext[] } ) { const project = await Project.createAndSwitch(projectOptions); diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 31039fd759..50e754d22c 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -275,4 +275,2019 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "contact_channels": { + sourceTables: { "ContactChannel": "ContactChannel" }, + targetTable: "contact_channels", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "contact_channels" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "type" text NOT NULL, + "value" text NOT NULL, + "is_primary" boolean NOT NULL DEFAULT false, + "is_verified" boolean NOT NULL DEFAULT false, + "used_for_auth" boolean NOT NULL DEFAULT false, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "contact_channels" FROM PUBLIC; + GRANT SELECT ON "contact_channels" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.contact_channels ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + type LowCardinality(String), + value String, + is_primary UInt8, + is_verified UInt8, + used_for_auth UInt8, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ContactChannel"."id" AS "id", + "ContactChannel"."projectUserId" AS "user_id", + "ContactChannel"."type"::text AS "type", + "ContactChannel"."value" AS "value", + CASE WHEN "ContactChannel"."isPrimary" = 'TRUE' THEN true ELSE false END AS "is_primary", + "ContactChannel"."isVerified" AS "is_verified", + CASE WHEN "ContactChannel"."usedForAuth" = 'TRUE' THEN true ELSE false END AS "used_for_auth", + "ContactChannel"."createdAt" AS "created_at", + "ContactChannel"."sequenceId" AS "sync_sequence_id", + "ContactChannel"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ContactChannel" + JOIN "Tenancy" ON "Tenancy"."id" = "ContactChannel"."tenancyId" + WHERE "ContactChannel"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "type", + NULL::text AS "value", + false AS "is_primary", + false AS "is_verified", + false AS "used_for_auth", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ContactChannel' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ContactChannel"."id" AS "id", + "ContactChannel"."projectUserId" AS "user_id", + "ContactChannel"."type"::text AS "type", + "ContactChannel"."value" AS "value", + CASE WHEN "ContactChannel"."isPrimary" = 'TRUE' THEN true ELSE false END AS "is_primary", + "ContactChannel"."isVerified" AS "is_verified", + CASE WHEN "ContactChannel"."usedForAuth" = 'TRUE' THEN true ELSE false END AS "used_for_auth", + "ContactChannel"."createdAt" AS "created_at", + "ContactChannel"."sequenceId" AS "sequence_id", + "ContactChannel"."tenancyId", + false AS "is_deleted" + FROM "ContactChannel" + WHERE "ContactChannel"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "type", + NULL::text AS "value", + false AS "is_primary", + false AS "is_verified", + false AS "used_for_auth", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ContactChannel' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "type", + $4::text AS "value", + $5::boolean AS "is_primary", + $6::boolean AS "is_verified", + $7::boolean AS "used_for_auth", + $8::timestamp without time zone AS "created_at", + $9::bigint AS "sequence_id", + $10::boolean AS "is_deleted", + $11::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "contact_channels" c + USING params p + WHERE p."is_deleted" = true AND c."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "contact_channels" ( + "id", + "user_id", + "type", + "value", + "is_primary", + "is_verified", + "used_for_auth", + "created_at" + ) + SELECT + p."id", + p."user_id", + p."type", + p."value", + p."is_primary", + p."is_verified", + p."used_for_auth", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "type" = EXCLUDED."type", + "value" = EXCLUDED."value", + "is_primary" = EXCLUDED."is_primary", + "is_verified" = EXCLUDED."is_verified", + "used_for_auth" = EXCLUDED."used_for_auth", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "teams": { + sourceTables: { "Team": "Team" }, + targetTable: "teams", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "teams" ( + "id" uuid PRIMARY KEY NOT NULL, + "display_name" text NOT NULL, + "profile_image_url" text, + "created_at" timestamp without time zone NOT NULL, + "client_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "client_read_only_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "server_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb + ); + REVOKE ALL ON "teams" FROM PUBLIC; + GRANT SELECT ON "teams" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.teams ( + project_id String, + branch_id String, + id UUID, + display_name String, + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + client_metadata String, + client_read_only_metadata String, + server_metadata String, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "Team"."teamId" AS "id", + "Team"."displayName" AS "display_name", + "Team"."profileImageUrl" AS "profile_image_url", + "Team"."createdAt" AS "created_at", + COALESCE("Team"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("Team"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("Team"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "Team"."sequenceId" AS "sync_sequence_id", + "Team"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "Team" + JOIN "Tenancy" ON "Tenancy"."id" = "Team"."tenancyId" + WHERE "Team"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'Team' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "Team"."teamId" AS "id", + "Team"."displayName" AS "display_name", + "Team"."profileImageUrl" AS "profile_image_url", + "Team"."createdAt" AS "created_at", + COALESCE("Team"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("Team"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("Team"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "Team"."sequenceId" AS "sequence_id", + "Team"."tenancyId", + false AS "is_deleted" + FROM "Team" + WHERE "Team"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'Team' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "display_name", + $3::text AS "profile_image_url", + $4::timestamp without time zone AS "created_at", + $5::jsonb AS "client_metadata", + $6::jsonb AS "client_read_only_metadata", + $7::jsonb AS "server_metadata", + $8::bigint AS "sequence_id", + $9::boolean AS "is_deleted", + $10::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "teams" t + USING params p + WHERE p."is_deleted" = true AND t."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "teams" ( + "id", + "display_name", + "profile_image_url", + "created_at", + "client_metadata", + "client_read_only_metadata", + "server_metadata" + ) + SELECT + p."id", + p."display_name", + p."profile_image_url", + p."created_at", + p."client_metadata", + p."client_read_only_metadata", + p."server_metadata" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "created_at" = EXCLUDED."created_at", + "client_metadata" = EXCLUDED."client_metadata", + "client_read_only_metadata" = EXCLUDED."client_read_only_metadata", + "server_metadata" = EXCLUDED."server_metadata" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "team_member_profiles": { + sourceTables: { "TeamMember": "TeamMember" }, + targetTable: "team_member_profiles", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_member_profiles" ( + "team_id" uuid NOT NULL, + "user_id" uuid NOT NULL, + "display_name" text, + "profile_image_url" text, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("team_id", "user_id") + ); + REVOKE ALL ON "team_member_profiles" FROM PUBLIC; + GRANT SELECT ON "team_member_profiles" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_member_profiles ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + display_name Nullable(String), + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, team_id, user_id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "TeamMember"."teamId" AS "team_id", + "TeamMember"."projectUserId" AS "user_id", + "TeamMember"."displayName" AS "display_name", + "TeamMember"."profileImageUrl" AS "profile_image_url", + "TeamMember"."createdAt" AS "created_at", + "TeamMember"."sequenceId" AS "sync_sequence_id", + "TeamMember"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "TeamMember" + JOIN "Tenancy" ON "Tenancy"."id" = "TeamMember"."tenancyId" + WHERE "TeamMember"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMember' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "TeamMember"."teamId" AS "team_id", + "TeamMember"."projectUserId" AS "user_id", + "TeamMember"."displayName" AS "display_name", + "TeamMember"."profileImageUrl" AS "profile_image_url", + "TeamMember"."createdAt" AS "created_at", + "TeamMember"."sequenceId" AS "sequence_id", + "TeamMember"."tenancyId", + false AS "is_deleted" + FROM "TeamMember" + WHERE "TeamMember"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMember' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "team_id", + $2::uuid AS "user_id", + $3::text AS "display_name", + $4::text AS "profile_image_url", + $5::timestamp without time zone AS "created_at", + $6::bigint AS "sequence_id", + $7::boolean AS "is_deleted", + $8::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_member_profiles" tm + USING params p + WHERE p."is_deleted" = true AND tm."team_id" = p."team_id" AND tm."user_id" = p."user_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_member_profiles" ( + "team_id", + "user_id", + "display_name", + "profile_image_url", + "created_at" + ) + SELECT + p."team_id", + p."user_id", + p."display_name", + p."profile_image_url", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("team_id", "user_id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "team_permissions": { + sourceTables: { "TeamMemberDirectPermission": "TeamMemberDirectPermission" }, + targetTable: "team_permissions", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_permissions" ( + "team_id" uuid NOT NULL, + "user_id" uuid NOT NULL, + "permission_id" text NOT NULL, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("team_id", "user_id", "permission_id") + ); + REVOKE ALL ON "team_permissions" FROM PUBLIC; + GRANT SELECT ON "team_permissions" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, team_id, user_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "TeamMemberDirectPermission"."teamId" AS "team_id", + "TeamMemberDirectPermission"."projectUserId" AS "user_id", + "TeamMemberDirectPermission"."permissionId" AS "id", + "TeamMemberDirectPermission"."createdAt" AS "created_at", + "TeamMemberDirectPermission"."sequenceId" AS "sync_sequence_id", + "TeamMemberDirectPermission"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "TeamMemberDirectPermission" + JOIN "Tenancy" ON "Tenancy"."id" = "TeamMemberDirectPermission"."tenancyId" + WHERE "TeamMemberDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMemberDirectPermission' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "TeamMemberDirectPermission"."teamId" AS "team_id", + "TeamMemberDirectPermission"."projectUserId" AS "user_id", + "TeamMemberDirectPermission"."permissionId" AS "permission_id", + "TeamMemberDirectPermission"."createdAt" AS "created_at", + "TeamMemberDirectPermission"."sequenceId" AS "sequence_id", + "TeamMemberDirectPermission"."tenancyId", + false AS "is_deleted" + FROM "TeamMemberDirectPermission" + WHERE "TeamMemberDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMemberDirectPermission' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "team_id", + $2::uuid AS "user_id", + $3::text AS "permission_id", + $4::timestamp without time zone AS "created_at", + $5::bigint AS "sequence_id", + $6::boolean AS "is_deleted", + $7::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_permissions" tp + USING params p + WHERE p."is_deleted" = true AND tp."team_id" = p."team_id" AND tp."user_id" = p."user_id" AND tp."permission_id" = p."permission_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_permissions" ( + "team_id", + "user_id", + "permission_id", + "created_at" + ) + SELECT + p."team_id", + p."user_id", + p."permission_id", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("team_id", "user_id", "permission_id") DO UPDATE SET + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "team_invitations": { + sourceTables: { "VerificationCode": "VerificationCode" }, + targetTable: "team_invitations", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_invitations" ( + "id" uuid PRIMARY KEY NOT NULL, + "team_id" uuid NOT NULL, + "team_display_name" text NOT NULL, + "recipient_email" text NOT NULL, + "expires_at_millis" bigint NOT NULL, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "team_invitations" FROM PUBLIC; + GRANT SELECT ON "team_invitations" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_invitations ( + project_id String, + branch_id String, + id UUID, + team_id UUID, + team_display_name String, + recipient_email String, + expires_at_millis Int64, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "VerificationCode"."id"::uuid AS "id", + ("VerificationCode"."data"->>'team_id')::uuid AS "team_id", + "Team"."displayName" AS "team_display_name", + "VerificationCode"."method"->>'email' AS "recipient_email", + FLOOR(EXTRACT(EPOCH FROM "VerificationCode"."expiresAt") * 1000)::bigint AS "expires_at_millis", + "VerificationCode"."createdAt" AS "created_at", + "VerificationCode"."sequenceId" AS "sync_sequence_id", + "Tenancy"."id" AS "tenancyId", + false AS "sync_is_deleted" + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + LEFT JOIN "Team" ON "Team"."teamId" = ("VerificationCode"."data"->>'team_id')::uuid + AND "Team"."tenancyId" = "Tenancy"."id" + WHERE "Tenancy"."id" = $1::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + '00000000-0000-0000-0000-000000000000'::uuid AS "team_id", + ''::text AS "team_display_name", + ''::text AS "recipient_email", + 0::bigint AS "expires_at_millis", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'VerificationCode_TEAM_INVITATION' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "VerificationCode"."id"::uuid AS "id", + ("VerificationCode"."data"->>'team_id')::uuid AS "team_id", + "Team"."displayName" AS "team_display_name", + "VerificationCode"."method"->>'email' AS "recipient_email", + FLOOR(EXTRACT(EPOCH FROM "VerificationCode"."expiresAt") * 1000)::bigint AS "expires_at_millis", + "VerificationCode"."createdAt" AS "created_at", + "VerificationCode"."sequenceId" AS "sequence_id", + "Tenancy"."id" AS "tenancyId", + false AS "is_deleted" + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + LEFT JOIN "Team" ON "Team"."teamId" = ("VerificationCode"."data"->>'team_id')::uuid + AND "Team"."tenancyId" = "Tenancy"."id" + WHERE "Tenancy"."id" = $1::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + '00000000-0000-0000-0000-000000000000'::uuid AS "team_id", + ''::text AS "team_display_name", + ''::text AS "recipient_email", + 0::bigint AS "expires_at_millis", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'VerificationCode_TEAM_INVITATION' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "team_id", + $3::text AS "team_display_name", + $4::text AS "recipient_email", + $5::bigint AS "expires_at_millis", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_invitations" ti + USING params p + WHERE p."is_deleted" = true AND ti."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_invitations" ( + "id", + "team_id", + "team_display_name", + "recipient_email", + "expires_at_millis", + "created_at" + ) + SELECT + p."id", + p."team_id", + p."team_display_name", + p."recipient_email", + p."expires_at_millis", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "team_id" = EXCLUDED."team_id", + "team_display_name" = EXCLUDED."team_display_name", + "recipient_email" = EXCLUDED."recipient_email", + "expires_at_millis" = EXCLUDED."expires_at_millis", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "email_outboxes": { + sourceTables: { "EmailOutbox": "EmailOutbox" }, + targetTable: "email_outboxes", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "email_outboxes" ( + "id" uuid PRIMARY KEY NOT NULL, + "status" text NOT NULL, + "simple_status" text NOT NULL, + "created_with" text NOT NULL, + "email_draft_id" text, + "email_programmatic_call_template_id" text, + "theme_id" text, + "is_high_priority" boolean NOT NULL DEFAULT false, + "rendered_is_transactional" boolean, + "rendered_subject" text, + "rendered_notification_category_id" text, + "started_rendering_at" timestamp without time zone, + "finished_rendering_at" timestamp without time zone, + "render_error" text, + "scheduled_at" timestamp without time zone NOT NULL, + "created_at" timestamp without time zone NOT NULL, + "started_sending_at" timestamp without time zone, + "finished_sending_at" timestamp without time zone, + "server_error" text, + "sent_at" timestamp without time zone, + "delivered_at" timestamp without time zone, + "opened_at" timestamp without time zone, + "clicked_at" timestamp without time zone, + "unsubscribed_at" timestamp without time zone, + "marked_as_spam_at" timestamp without time zone, + "bounced_at" timestamp without time zone, + "delivery_delayed_at" timestamp without time zone, + "can_have_delivery_info" boolean, + "skipped_reason" text, + "skipped_details" jsonb, + "send_retries" integer NOT NULL DEFAULT 0, + "is_paused" boolean NOT NULL DEFAULT false + ); + REVOKE ALL ON "email_outboxes" FROM PUBLIC; + GRANT SELECT ON "email_outboxes" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( + project_id String, + branch_id String, + id UUID, + status LowCardinality(String), + simple_status LowCardinality(String), + created_with LowCardinality(String), + email_draft_id Nullable(String), + email_programmatic_call_template_id Nullable(String), + theme_id Nullable(String), + is_high_priority UInt8, + is_transactional Nullable(UInt8), + subject Nullable(String), + notification_category_id Nullable(String), + started_rendering_at Nullable(DateTime64(3, 'UTC')), + rendered_at Nullable(DateTime64(3, 'UTC')), + render_error Nullable(String), + scheduled_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + updated_at DateTime64(3, 'UTC'), + started_sending_at Nullable(DateTime64(3, 'UTC')), + server_error Nullable(String), + delivered_at Nullable(DateTime64(3, 'UTC')), + opened_at Nullable(DateTime64(3, 'UTC')), + clicked_at Nullable(DateTime64(3, 'UTC')), + unsubscribed_at Nullable(DateTime64(3, 'UTC')), + marked_as_spam_at Nullable(DateTime64(3, 'UTC')), + bounced_at Nullable(DateTime64(3, 'UTC')), + delivery_delayed_at Nullable(DateTime64(3, 'UTC')), + can_have_delivery_info Nullable(UInt8), + skipped_reason LowCardinality(Nullable(String)), + skipped_details Nullable(String), + send_retries Int32, + is_paused UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "EmailOutbox"."id" AS "id", + LOWER(REPLACE("EmailOutbox"."status"::text, '_', '-')) AS "status", + LOWER(REPLACE("EmailOutbox"."simpleStatus"::text, '_', '-')) AS "simple_status", + CASE WHEN "EmailOutbox"."createdWith"::text = 'DRAFT' THEN 'draft' ELSE 'programmatic-call' END AS "created_with", + "EmailOutbox"."emailDraftId" AS "email_draft_id", + "EmailOutbox"."emailProgrammaticCallTemplateId" AS "email_programmatic_call_template_id", + "EmailOutbox"."themeId" AS "theme_id", + "EmailOutbox"."isHighPriority" AS "is_high_priority", + "EmailOutbox"."renderedIsTransactional" AS "is_transactional", + "EmailOutbox"."renderedSubject" AS "subject", + "EmailOutbox"."renderedNotificationCategoryId" AS "notification_category_id", + "EmailOutbox"."startedRenderingAt" AS "started_rendering_at", + "EmailOutbox"."finishedRenderingAt" AS "rendered_at", + "EmailOutbox"."renderErrorExternalMessage" AS "render_error", + "EmailOutbox"."scheduledAt" AS "scheduled_at", + "EmailOutbox"."createdAt" AS "created_at", + "EmailOutbox"."updatedAt" AS "updated_at", + "EmailOutbox"."startedSendingAt" AS "started_sending_at", + "EmailOutbox"."sendServerErrorExternalMessage" AS "server_error", + "EmailOutbox"."deliveredAt" AS "delivered_at", + "EmailOutbox"."openedAt" AS "opened_at", + "EmailOutbox"."clickedAt" AS "clicked_at", + "EmailOutbox"."unsubscribedAt" AS "unsubscribed_at", + "EmailOutbox"."markedAsSpamAt" AS "marked_as_spam_at", + "EmailOutbox"."bouncedAt" AS "bounced_at", + "EmailOutbox"."deliveryDelayedAt" AS "delivery_delayed_at", + "EmailOutbox"."canHaveDeliveryInfo" AS "can_have_delivery_info", + LOWER(REPLACE("EmailOutbox"."skippedReason"::text, '_', '-')) AS "skipped_reason", + "EmailOutbox"."skippedDetails" AS "skipped_details", + "EmailOutbox"."sendRetries" AS "send_retries", + "EmailOutbox"."isPaused" AS "is_paused", + "EmailOutbox"."sequenceId" AS "sync_sequence_id", + "EmailOutbox"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "EmailOutbox" + JOIN "Tenancy" ON "Tenancy"."id" = "EmailOutbox"."tenancyId" + WHERE "EmailOutbox"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ''::text AS "status", + ''::text AS "simple_status", + ''::text AS "created_with", + NULL::text AS "email_draft_id", + NULL::text AS "email_programmatic_call_template_id", + NULL::text AS "theme_id", + false AS "is_high_priority", + NULL::boolean AS "is_transactional", + NULL::text AS "subject", + NULL::text AS "notification_category_id", + NULL::timestamp without time zone AS "started_rendering_at", + NULL::timestamp without time zone AS "rendered_at", + NULL::text AS "render_error", + "DeletedRow"."deletedAt"::timestamp without time zone AS "scheduled_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "updated_at", + NULL::timestamp without time zone AS "started_sending_at", + NULL::text AS "server_error", + NULL::timestamp without time zone AS "delivered_at", + NULL::timestamp without time zone AS "opened_at", + NULL::timestamp without time zone AS "clicked_at", + NULL::timestamp without time zone AS "unsubscribed_at", + NULL::timestamp without time zone AS "marked_as_spam_at", + NULL::timestamp without time zone AS "bounced_at", + NULL::timestamp without time zone AS "delivery_delayed_at", + NULL::boolean AS "can_have_delivery_info", + NULL::text AS "skipped_reason", + NULL::jsonb AS "skipped_details", + 0 AS "send_retries", + false AS "is_paused", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'EmailOutbox' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "EmailOutbox"."id" AS "id", + "EmailOutbox"."status"::text AS "status", + "EmailOutbox"."simpleStatus"::text AS "simple_status", + "EmailOutbox"."createdWith"::text AS "created_with", + "EmailOutbox"."emailDraftId" AS "email_draft_id", + "EmailOutbox"."emailProgrammaticCallTemplateId" AS "email_programmatic_call_template_id", + "EmailOutbox"."themeId" AS "theme_id", + "EmailOutbox"."isHighPriority" AS "is_high_priority", + "EmailOutbox"."renderedIsTransactional" AS "rendered_is_transactional", + "EmailOutbox"."renderedSubject" AS "rendered_subject", + "EmailOutbox"."renderedNotificationCategoryId" AS "rendered_notification_category_id", + "EmailOutbox"."startedRenderingAt" AS "started_rendering_at", + "EmailOutbox"."finishedRenderingAt" AS "finished_rendering_at", + "EmailOutbox"."renderErrorExternalMessage" AS "render_error", + "EmailOutbox"."scheduledAt" AS "scheduled_at", + "EmailOutbox"."createdAt" AS "created_at", + "EmailOutbox"."startedSendingAt" AS "started_sending_at", + "EmailOutbox"."finishedSendingAt" AS "finished_sending_at", + "EmailOutbox"."sendServerErrorExternalMessage" AS "server_error", + "EmailOutbox"."sentAt" AS "sent_at", + "EmailOutbox"."deliveredAt" AS "delivered_at", + "EmailOutbox"."openedAt" AS "opened_at", + "EmailOutbox"."clickedAt" AS "clicked_at", + "EmailOutbox"."unsubscribedAt" AS "unsubscribed_at", + "EmailOutbox"."markedAsSpamAt" AS "marked_as_spam_at", + "EmailOutbox"."bouncedAt" AS "bounced_at", + "EmailOutbox"."deliveryDelayedAt" AS "delivery_delayed_at", + "EmailOutbox"."canHaveDeliveryInfo" AS "can_have_delivery_info", + "EmailOutbox"."skippedReason"::text AS "skipped_reason", + "EmailOutbox"."skippedDetails" AS "skipped_details", + "EmailOutbox"."sendRetries" AS "send_retries", + "EmailOutbox"."isPaused" AS "is_paused", + "EmailOutbox"."sequenceId" AS "sequence_id", + "EmailOutbox"."tenancyId", + false AS "is_deleted" + FROM "EmailOutbox" + WHERE "EmailOutbox"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ''::text AS "status", + ''::text AS "simple_status", + ''::text AS "created_with", + NULL::text AS "email_draft_id", + NULL::text AS "email_programmatic_call_template_id", + NULL::text AS "theme_id", + false AS "is_high_priority", + NULL::boolean AS "rendered_is_transactional", + NULL::text AS "rendered_subject", + NULL::text AS "rendered_notification_category_id", + NULL::timestamp without time zone AS "started_rendering_at", + NULL::timestamp without time zone AS "finished_rendering_at", + NULL::text AS "render_error", + "DeletedRow"."deletedAt"::timestamp without time zone AS "scheduled_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + NULL::timestamp without time zone AS "started_sending_at", + NULL::timestamp without time zone AS "finished_sending_at", + NULL::text AS "server_error", + NULL::timestamp without time zone AS "sent_at", + NULL::timestamp without time zone AS "delivered_at", + NULL::timestamp without time zone AS "opened_at", + NULL::timestamp without time zone AS "clicked_at", + NULL::timestamp without time zone AS "unsubscribed_at", + NULL::timestamp without time zone AS "marked_as_spam_at", + NULL::timestamp without time zone AS "bounced_at", + NULL::timestamp without time zone AS "delivery_delayed_at", + NULL::boolean AS "can_have_delivery_info", + NULL::text AS "skipped_reason", + NULL::jsonb AS "skipped_details", + 0 AS "send_retries", + false AS "is_paused", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'EmailOutbox' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "status", + $3::text AS "simple_status", + $4::text AS "created_with", + $5::text AS "email_draft_id", + $6::text AS "email_programmatic_call_template_id", + $7::text AS "theme_id", + $8::boolean AS "is_high_priority", + $9::boolean AS "rendered_is_transactional", + $10::text AS "rendered_subject", + $11::text AS "rendered_notification_category_id", + $12::timestamp without time zone AS "started_rendering_at", + $13::timestamp without time zone AS "finished_rendering_at", + $14::text AS "render_error", + $15::timestamp without time zone AS "scheduled_at", + $16::timestamp without time zone AS "created_at", + $17::timestamp without time zone AS "started_sending_at", + $18::timestamp without time zone AS "finished_sending_at", + $19::text AS "server_error", + $20::timestamp without time zone AS "sent_at", + $21::timestamp without time zone AS "delivered_at", + $22::timestamp without time zone AS "opened_at", + $23::timestamp without time zone AS "clicked_at", + $24::timestamp without time zone AS "unsubscribed_at", + $25::timestamp without time zone AS "marked_as_spam_at", + $26::timestamp without time zone AS "bounced_at", + $27::timestamp without time zone AS "delivery_delayed_at", + $28::boolean AS "can_have_delivery_info", + $29::text AS "skipped_reason", + $30::jsonb AS "skipped_details", + $31::integer AS "send_retries", + $32::boolean AS "is_paused", + $33::bigint AS "sequence_id", + $34::boolean AS "is_deleted", + $35::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "email_outboxes" eo + USING params p + WHERE p."is_deleted" = true AND eo."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "email_outboxes" ( + "id", + "status", + "simple_status", + "created_with", + "email_draft_id", + "email_programmatic_call_template_id", + "theme_id", + "is_high_priority", + "rendered_is_transactional", + "rendered_subject", + "rendered_notification_category_id", + "started_rendering_at", + "finished_rendering_at", + "render_error", + "scheduled_at", + "created_at", + "started_sending_at", + "finished_sending_at", + "server_error", + "sent_at", + "delivered_at", + "opened_at", + "clicked_at", + "unsubscribed_at", + "marked_as_spam_at", + "bounced_at", + "delivery_delayed_at", + "can_have_delivery_info", + "skipped_reason", + "skipped_details", + "send_retries", + "is_paused" + ) + SELECT + p."id", + p."status", + p."simple_status", + p."created_with", + p."email_draft_id", + p."email_programmatic_call_template_id", + p."theme_id", + p."is_high_priority", + p."rendered_is_transactional", + p."rendered_subject", + p."rendered_notification_category_id", + p."started_rendering_at", + p."finished_rendering_at", + p."render_error", + p."scheduled_at", + p."created_at", + p."started_sending_at", + p."finished_sending_at", + p."server_error", + p."sent_at", + p."delivered_at", + p."opened_at", + p."clicked_at", + p."unsubscribed_at", + p."marked_as_spam_at", + p."bounced_at", + p."delivery_delayed_at", + p."can_have_delivery_info", + p."skipped_reason", + p."skipped_details", + p."send_retries", + p."is_paused" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "status" = EXCLUDED."status", + "simple_status" = EXCLUDED."simple_status", + "created_with" = EXCLUDED."created_with", + "email_draft_id" = EXCLUDED."email_draft_id", + "email_programmatic_call_template_id" = EXCLUDED."email_programmatic_call_template_id", + "theme_id" = EXCLUDED."theme_id", + "is_high_priority" = EXCLUDED."is_high_priority", + "rendered_is_transactional" = EXCLUDED."rendered_is_transactional", + "rendered_subject" = EXCLUDED."rendered_subject", + "rendered_notification_category_id" = EXCLUDED."rendered_notification_category_id", + "started_rendering_at" = EXCLUDED."started_rendering_at", + "finished_rendering_at" = EXCLUDED."finished_rendering_at", + "render_error" = EXCLUDED."render_error", + "scheduled_at" = EXCLUDED."scheduled_at", + "created_at" = EXCLUDED."created_at", + "started_sending_at" = EXCLUDED."started_sending_at", + "finished_sending_at" = EXCLUDED."finished_sending_at", + "server_error" = EXCLUDED."server_error", + "sent_at" = EXCLUDED."sent_at", + "delivered_at" = EXCLUDED."delivered_at", + "opened_at" = EXCLUDED."opened_at", + "clicked_at" = EXCLUDED."clicked_at", + "unsubscribed_at" = EXCLUDED."unsubscribed_at", + "marked_as_spam_at" = EXCLUDED."marked_as_spam_at", + "bounced_at" = EXCLUDED."bounced_at", + "delivery_delayed_at" = EXCLUDED."delivery_delayed_at", + "can_have_delivery_info" = EXCLUDED."can_have_delivery_info", + "skipped_reason" = EXCLUDED."skipped_reason", + "skipped_details" = EXCLUDED."skipped_details", + "send_retries" = EXCLUDED."send_retries", + "is_paused" = EXCLUDED."is_paused" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "project_permissions": { + sourceTables: { "ProjectUserDirectPermission": "ProjectUserDirectPermission" }, + targetTable: "project_permissions", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "project_permissions" ( + "user_id" uuid NOT NULL, + "permission_id" text NOT NULL, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("user_id", "permission_id") + ); + REVOKE ALL ON "project_permissions" FROM PUBLIC; + GRANT SELECT ON "project_permissions" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( + project_id String, + branch_id String, + user_id UUID, + id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, user_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserDirectPermission"."projectUserId" AS "user_id", + "ProjectUserDirectPermission"."permissionId" AS "id", + "ProjectUserDirectPermission"."createdAt" AS "created_at", + "ProjectUserDirectPermission"."sequenceId" AS "sync_sequence_id", + "ProjectUserDirectPermission"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserDirectPermission" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserDirectPermission"."tenancyId" + WHERE "ProjectUserDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserDirectPermission' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserDirectPermission"."projectUserId" AS "user_id", + "ProjectUserDirectPermission"."permissionId" AS "permission_id", + "ProjectUserDirectPermission"."createdAt" AS "created_at", + "ProjectUserDirectPermission"."sequenceId" AS "sequence_id", + "ProjectUserDirectPermission"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserDirectPermission" + WHERE "ProjectUserDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserDirectPermission' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "user_id", + $2::text AS "permission_id", + $3::timestamp without time zone AS "created_at", + $4::bigint AS "sequence_id", + $5::boolean AS "is_deleted", + $6::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "project_permissions" pp + USING params p + WHERE p."is_deleted" = true AND pp."user_id" = p."user_id" AND pp."permission_id" = p."permission_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "project_permissions" ( + "user_id", + "permission_id", + "created_at" + ) + SELECT + p."user_id", + p."permission_id", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("user_id", "permission_id") DO UPDATE SET + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "notification_preferences": { + sourceTables: { "UserNotificationPreference": "UserNotificationPreference" }, + targetTable: "notification_preferences", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "notification_preferences" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "notification_category_id" text NOT NULL, + "enabled" boolean NOT NULL DEFAULT true + ); + REVOKE ALL ON "notification_preferences" FROM PUBLIC; + GRANT SELECT ON "notification_preferences" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( + project_id String, + branch_id String, + user_id UUID, + notification_category_id String, + enabled UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + ORDER BY (project_id, branch_id, user_id, notification_category_id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "UserNotificationPreference"."projectUserId" AS "user_id", + "UserNotificationPreference"."notificationCategoryId" AS "notification_category_id", + "UserNotificationPreference"."enabled" AS "enabled", + "UserNotificationPreference"."sequenceId" AS "sync_sequence_id", + "UserNotificationPreference"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "UserNotificationPreference" + JOIN "Tenancy" ON "Tenancy"."id" = "UserNotificationPreference"."tenancyId" + WHERE "UserNotificationPreference"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + ("DeletedRow"."data"->>'notificationCategoryId')::uuid AS "notification_category_id", + ("DeletedRow"."data"->>'enabled')::boolean AS "enabled", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'UserNotificationPreference' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "UserNotificationPreference"."id" AS "id", + "UserNotificationPreference"."projectUserId" AS "user_id", + "UserNotificationPreference"."notificationCategoryId" AS "notification_category_id", + "UserNotificationPreference"."enabled" AS "enabled", + "UserNotificationPreference"."sequenceId" AS "sequence_id", + "UserNotificationPreference"."tenancyId", + false AS "is_deleted" + FROM "UserNotificationPreference" + WHERE "UserNotificationPreference"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + ("DeletedRow"."data"->>'notificationCategoryId')::uuid AS "notification_category_id", + ("DeletedRow"."data"->>'enabled')::boolean AS "enabled", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'UserNotificationPreference' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "notification_category_id", + $4::boolean AS "enabled", + $5::bigint AS "sequence_id", + $6::boolean AS "is_deleted", + $7::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "notification_preferences" np + USING params p + WHERE p."is_deleted" = true AND np."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "notification_preferences" ( + "id", + "user_id", + "notification_category_id", + "enabled" + ) + SELECT + p."id", + p."user_id", + p."notification_category_id", + p."enabled" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "notification_category_id" = EXCLUDED."notification_category_id", + "enabled" = EXCLUDED."enabled" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "refresh_tokens": { + sourceTables: { "ProjectUserRefreshToken": "ProjectUserRefreshToken" }, + targetTable: "refresh_tokens", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "refresh_tokens" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "created_at" timestamp without time zone NOT NULL, + "last_used_at" timestamp without time zone NOT NULL, + "is_impersonation" boolean NOT NULL DEFAULT false, + "expires_at" timestamp without time zone + ); + REVOKE ALL ON "refresh_tokens" FROM PUBLIC; + GRANT SELECT ON "refresh_tokens" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.refresh_tokens ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + created_at DateTime64(3, 'UTC'), + last_used_at DateTime64(3, 'UTC'), + is_impersonation UInt8, + expires_at Nullable(DateTime64(3, 'UTC')), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + + CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata ( + tenancy_id UUID, + mapping_name String, + last_synced_sequence_id Int64, + updated_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(updated_at) + ORDER BY (tenancy_id, mapping_name); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserRefreshToken"."id" AS "id", + "ProjectUserRefreshToken"."projectUserId" AS "user_id", + "ProjectUserRefreshToken"."createdAt" AS "created_at", + "ProjectUserRefreshToken"."lastActiveAt" AS "last_used_at", + "ProjectUserRefreshToken"."isImpersonation" AS "is_impersonation", + "ProjectUserRefreshToken"."expiresAt" AS "expires_at", + "ProjectUserRefreshToken"."sequenceId" AS "sync_sequence_id", + "ProjectUserRefreshToken"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserRefreshToken" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserRefreshToken"."tenancyId" + WHERE "ProjectUserRefreshToken"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "last_used_at", + false AS "is_impersonation", + NULL::timestamp without time zone AS "expires_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserRefreshToken' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserRefreshToken"."id" AS "id", + "ProjectUserRefreshToken"."projectUserId" AS "user_id", + "ProjectUserRefreshToken"."createdAt" AS "created_at", + "ProjectUserRefreshToken"."lastActiveAt" AS "last_used_at", + "ProjectUserRefreshToken"."isImpersonation" AS "is_impersonation", + "ProjectUserRefreshToken"."expiresAt" AS "expires_at", + "ProjectUserRefreshToken"."sequenceId" AS "sequence_id", + "ProjectUserRefreshToken"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserRefreshToken" + WHERE "ProjectUserRefreshToken"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "last_used_at", + false AS "is_impersonation", + NULL::timestamp without time zone AS "expires_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserRefreshToken' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::timestamp without time zone AS "created_at", + $4::timestamp without time zone AS "last_used_at", + $5::boolean AS "is_impersonation", + $6::timestamp without time zone AS "expires_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "refresh_tokens" rt + USING params p + WHERE p."is_deleted" = true AND rt."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "refresh_tokens" ( + "id", + "user_id", + "created_at", + "last_used_at", + "is_impersonation", + "expires_at" + ) + SELECT + p."id", + p."user_id", + p."created_at", + p."last_used_at", + p."is_impersonation", + p."expires_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "created_at" = EXCLUDED."created_at", + "last_used_at" = EXCLUDED."last_used_at", + "is_impersonation" = EXCLUDED."is_impersonation", + "expires_at" = EXCLUDED."expires_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "connected_accounts": { + sourceTables: { "ProjectUserOAuthAccount": "ProjectUserOAuthAccount" }, + targetTable: "connected_accounts", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "connected_accounts" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "provider" text NOT NULL, + "provider_account_id" text NOT NULL, + "email" text, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "connected_accounts" FROM PUBLIC; + GRANT SELECT ON "connected_accounts" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( + project_id String, + branch_id String, + user_id UUID, + provider String, + provider_account_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, user_id, provider, provider_account_id); + + CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata ( + tenancy_id UUID, + mapping_name String, + last_synced_sequence_id Int64, + updated_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(updated_at) + ORDER BY (tenancy_id, mapping_name); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserOAuthAccount"."projectUserId" AS "user_id", + "ProjectUserOAuthAccount"."configOAuthProviderId" AS "provider", + "ProjectUserOAuthAccount"."providerAccountId" AS "provider_account_id", + "ProjectUserOAuthAccount"."createdAt" AS "created_at", + "ProjectUserOAuthAccount"."sequenceId" AS "sync_sequence_id", + "ProjectUserOAuthAccount"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserOAuthAccount" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserOAuthAccount"."tenancyId" + WHERE "ProjectUserOAuthAccount"."tenancyId" = $1::uuid + AND "ProjectUserOAuthAccount"."projectUserId" IS NOT NULL + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."data"->>'configOAuthProviderId' AS "provider", + "DeletedRow"."data"->>'providerAccountId' AS "provider_account_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserOAuthAccount' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserOAuthAccount"."id" AS "id", + "ProjectUserOAuthAccount"."projectUserId" AS "user_id", + "ProjectUserOAuthAccount"."configOAuthProviderId" AS "provider", + "ProjectUserOAuthAccount"."providerAccountId" AS "provider_account_id", + "ProjectUserOAuthAccount"."email" AS "email", + "ProjectUserOAuthAccount"."createdAt" AS "created_at", + "ProjectUserOAuthAccount"."sequenceId" AS "sequence_id", + "ProjectUserOAuthAccount"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserOAuthAccount" + WHERE "ProjectUserOAuthAccount"."tenancyId" = $1::uuid + AND "ProjectUserOAuthAccount"."projectUserId" IS NOT NULL + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "provider", + NULL::text AS "provider_account_id", + NULL::text AS "email", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserOAuthAccount' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "provider", + $4::text AS "provider_account_id", + $5::text AS "email", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "connected_accounts" ca + USING params p + WHERE p."is_deleted" = true AND ca."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "connected_accounts" ( + "id", + "user_id", + "provider", + "provider_account_id", + "email", + "created_at" + ) + SELECT + p."id", + p."user_id", + p."provider", + p."provider_account_id", + p."email", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "provider" = EXCLUDED."provider", + "provider_account_id" = EXCLUDED."provider_account_id", + "email" = EXCLUDED."email", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const;