From 3c889504191eba9038beccfee1a9cd12dd21fa07 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 16 Mar 2026 10:53:20 -0700 Subject: [PATCH 01/14] clickhouse contact channel, team, and team member tables --- .../migration.sql | 25 + apps/backend/prisma/schema.prisma | 10 + apps/backend/scripts/clickhouse-migrations.ts | 133 +++++ .../external-db-sync/sequencer/route.ts | 59 +- .../app/api/latest/team-memberships/crud.tsx | 12 +- .../backend/src/app/api/latest/teams/crud.tsx | 24 +- .../backend/src/app/api/latest/users/crud.tsx | 7 +- apps/backend/src/lib/external-db-sync.ts | 257 +++++++- .../api/v1/external-db-sync-basics.test.ts | 260 ++++++++ .../api/v1/external-db-sync-utils.ts | 109 +++- .../src/config/db-sync-mappings.ts | 557 ++++++++++++++++++ 11 files changed, 1382 insertions(+), 71 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql diff --git a/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql new file mode 100644 index 0000000000..358cf0514c --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql @@ -0,0 +1,25 @@ +-- AlterTable +ALTER TABLE "Team" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "Team_sequenceId_key" ON "Team"("sequenceId"); + +-- CreateIndex +CREATE INDEX "Team_tenancyId_sequenceId_idx" ON "Team"("tenancyId", "sequenceId"); + +-- CreateIndex +CREATE INDEX "Team_shouldUpdateSequenceId_idx" ON "Team"("shouldUpdateSequenceId", "tenancyId"); + +-- AlterTable +ALTER TABLE "TeamMember" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "TeamMember_sequenceId_key" ON "TeamMember"("sequenceId"); + +-- CreateIndex +CREATE INDEX "TeamMember_tenancyId_sequenceId_idx" ON "TeamMember"("tenancyId", "sequenceId"); + +-- CreateIndex +CREATE INDEX "TeamMember_shouldUpdateSequenceId_idx" ON "TeamMember"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 0d408ff0ba..d6077f3039 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -179,11 +179,16 @@ model Team { serverMetadata Json? profileImageUrl String? + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + teamMembers TeamMember[] projectApiKey ProjectApiKey[] @@id([tenancyId, teamId]) @@unique([mirroredProjectId, mirroredBranchId, teamId]) + @@index([tenancyId, sequenceId], name: "Team_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "Team_shouldUpdateSequenceId_idx") } // This is used for fields that are boolean but only the true value is part of a unique constraint. @@ -205,6 +210,9 @@ model TeamMember { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) team Team @relation(fields: [tenancyId, teamId], references: [tenancyId, teamId], onDelete: Cascade) isSelected BooleanTrue? @@ -213,6 +221,8 @@ model TeamMember { @@id([tenancyId, projectUserId, teamId]) @@unique([tenancyId, projectUserId, isSelected]) @@index([tenancyId, projectUserId, isSelected], map: "TeamMember_projectUserId_isSelected_idx") + @@index([tenancyId, sequenceId], name: "TeamMember_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMember_shouldUpdateSequenceId_idx") } model ProjectUserDirectPermission { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index d94f84baa0..f0b533c034 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -16,6 +16,12 @@ export async function runClickhouseMigrations() { await client.exec({ query: EVENTS_VIEW_SQL }); await client.exec({ query: USERS_TABLE_BASE_SQL }); await client.exec({ query: USERS_VIEW_SQL }); + await client.exec({ query: CONTACT_CHANNELS_TABLE_BASE_SQL }); + await client.exec({ query: CONTACT_CHANNELS_VIEW_SQL }); + await client.exec({ query: TEAMS_TABLE_BASE_SQL }); + await client.exec({ query: TEAMS_VIEW_SQL }); + await client.exec({ query: TEAM_MEMBERS_TABLE_BASE_SQL }); + await client.exec({ query: TEAM_MEMBERS_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -27,6 +33,9 @@ export async function runClickhouseMigrations() { "REVOKE ALL FROM limited_user;", "GRANT SELECT ON default.events TO limited_user;", "GRANT SELECT ON default.users TO limited_user;", + "GRANT SELECT ON default.contact_channels TO limited_user;", + "GRANT SELECT ON default.teams TO limited_user;", + "GRANT SELECT ON default.team_members TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -34,6 +43,15 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS users_project_isolation ON default.users FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS contact_channels_project_isolation ON default.contact_channels FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS teams_project_isolation ON default.teams FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS team_members_project_isolation ON default.team_members FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -197,6 +215,121 @@ WHERE event_type = '$token-refresh' AND data.refresh_token_id::Nullable(String) IS NOT NULL; `; +const CONTACT_CHANNELS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.contact_channels ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + type LowCardinality(String), + value String, + is_primary UInt8, + is_verified UInt8, + used_for_auth UInt8, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const CONTACT_CHANNELS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.contact_channels +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + type, + value, + is_primary, + is_verified, + used_for_auth, + created_at +FROM analytics_internal.contact_channels +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAMS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.teams ( + project_id String, + branch_id String, + id UUID, + display_name String, + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + client_metadata String, + client_read_only_metadata String, + server_metadata String, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const TEAMS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.teams +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + display_name, + profile_image_url, + created_at, + client_metadata, + client_read_only_metadata, + server_metadata +FROM analytics_internal.teams +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAM_MEMBERS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_members ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + display_name Nullable(String), + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, team_id, user_id); +`; + +const TEAM_MEMBERS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_members +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + team_id, + user_id, + display_name, + profile_image_url, + created_at +FROM analytics_internal.team_members +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index c7808fb53b..78bbb99ffd 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -109,6 +109,63 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const teamTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "teamId" + FROM "Team" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "Team" t + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE t."tenancyId" = r."tenancyId" + AND t."teamId" = r."teamId" + RETURNING t."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-tenants", teamTenants.length); + + if (teamTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const teamMemberTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId", "teamId" + FROM "TeamMember" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "TeamMember" tm + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE tm."tenancyId" = r."tenancyId" + AND tm."projectUserId" = r."projectUserId" + AND tm."teamId" = r."teamId" + RETURNING tm."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-member-tenants", teamMemberTenants.length); + + if (teamMemberTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamMemberTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -138,7 +195,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/team-memberships/crud.tsx b/apps/backend/src/app/api/latest/team-memberships/crud.tsx index ae11b32ef5..ae5dbfd43a 100644 --- a/apps/backend/src/app/api/latest/team-memberships/crud.tsx +++ b/apps/backend/src/app/api/latest/team-memberships/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultTeamPermissions } from "@/lib/permissions"; import { ensureTeamExists, ensureTeamMembershipDoesNotExist, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -20,11 +21,11 @@ export async function addUserToTeam(tx: PrismaTransaction, options: { type: 'member' | 'creator', }) { await tx.teamMember.create({ - data: { + data: withExternalDbSyncUpdate({ projectUserId: options.userId, teamId: options.teamId, tenancyId: options.tenancy.id, - }, + }), }); const result = await grantDefaultTeamPermissions(tx, { @@ -138,6 +139,13 @@ export const teamMembershipsCrudHandlers = createLazyProxy(() => createCrudHandl userId: params.user_id, }); + await recordExternalDbSyncDeletion(tx, { + tableName: "TeamMember", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + teamId: params.team_id, + }); + await tx.teamMember.delete({ where: { tenancyId_projectUserId_teamId: { diff --git a/apps/backend/src/app/api/latest/teams/crud.tsx b/apps/backend/src/app/api/latest/teams/crud.tsx index 05c7ca1772..aab670506e 100644 --- a/apps/backend/src/app/api/latest/teams/crud.tsx +++ b/apps/backend/src/app/api/latest/teams/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForTeam, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureTeamExists, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { sendTeamCreatedWebhook, sendTeamDeletedWebhook, sendTeamUpdatedWebhook } from "@/lib/webhooks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -73,7 +74,7 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC const db = await retryTransaction(prisma, async (tx) => { const db = await tx.team.create({ - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, mirroredProjectId: auth.project.id, mirroredBranchId: auth.branchId, @@ -81,8 +82,8 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, serverMetadata: data.server_metadata === null ? Prisma.JsonNull : data.server_metadata, - profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images") - }, + profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images"), + }), }); if (addUserId) { @@ -160,13 +161,13 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC teamId: params.team_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, serverMetadata: data.server_metadata === null ? Prisma.JsonNull : data.server_metadata, - profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images") - }, + profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-profile-images"), + }), }); }); @@ -194,6 +195,17 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC } await ensureTeamExists(tx, { tenancyId: auth.tenancy.id, teamId: params.team_id }); + await recordExternalDbSyncTeamMemberDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "Team", + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + await tx.team.delete({ where: { tenancyId_teamId: { diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index 3a16996d3d..fc7e56006c 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,7 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; -import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -1202,6 +1202,11 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }); + await recordExternalDbSyncTeamMemberDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 62ac6536bd..bd9b0b1cab 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -41,6 +41,17 @@ type ExternalDbSyncTarget = tenancyId: string, projectUserId: string, contactChannelId: string, + } + | { + tableName: "Team", + tenancyId: string, + teamId: string, + } + | { + tableName: "TeamMember", + tenancyId: string, + projectUserId: string, + teamId: string, }; type ExternalDbType = NonNullable["type"]>; @@ -80,9 +91,9 @@ export async function recordExternalDbSyncDeletion( target: ExternalDbSyncTarget, ): Promise { assertUuid(target.tenancyId, "tenancyId"); - assertUuid(target.projectUserId, "projectUserId"); if (target.tableName === "ProjectUser") { + assertUuid(target.projectUserId, "projectUserId"); const insertedCount = await tx.$executeRaw(Prisma.sql` INSERT INTO "DeletedRow" ( "id", @@ -115,8 +126,132 @@ export async function recordExternalDbSyncDeletion( return; } - assertUuid(target.contactChannelId, "contactChannelId"); - const insertedCount = await tx.$executeRaw(Prisma.sql` + if (target.tableName === "ContactChannel") { + assertUuid(target.projectUserId, "projectUserId"); + assertUuid(target.contactChannelId, "contactChannelId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "id" = ${target.contactChannelId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "Team") { + assertUuid(target.teamId, "teamId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'Team', + jsonb_build_object('tenancyId', "tenancyId", 'teamId', "teamId"), + to_jsonb("Team".*), + NOW(), + TRUE + FROM "Team" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "teamId" = ${target.teamId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for Team, got ${insertedCount}.` + ); + } + return; + } + + { + const _teamMemberTarget: { tableName: "TeamMember" } = target; + assertUuid(target.projectUserId, "projectUserId"); + assertUuid(target.teamId, "teamId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), + NOW(), + TRUE + FROM "TeamMember" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "teamId" = ${target.teamId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for TeamMember, got ${insertedCount}.` + ); + } + return; + } +} + +export async function recordExternalDbSyncContactChannelDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` INSERT INTO "DeletedRow" ( "id", "tenancyId", @@ -142,20 +277,48 @@ export async function recordExternalDbSyncDeletion( NOW(), TRUE FROM "ContactChannel" - WHERE "tenancyId" = ${target.tenancyId}::uuid - AND "projectUserId" = ${target.projectUserId}::uuid - AND "id" = ${target.contactChannelId}::uuid + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid FOR UPDATE `); +} - if (insertedCount !== 1) { - throw new StackAssertionError( - `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` - ); - } +export async function recordExternalDbSyncTeamMemberDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), + NOW(), + TRUE + FROM "TeamMember" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); } -export async function recordExternalDbSyncContactChannelDeletionsForUser( +export async function recordExternalDbSyncTeamMemberDeletionsForUser( tx: ExternalDbSyncClient, options: { tenancyId: string, @@ -178,19 +341,12 @@ export async function recordExternalDbSyncContactChannelDeletionsForUser( SELECT gen_random_uuid(), "tenancyId", - 'ContactChannel', - jsonb_build_object( - 'tenancyId', - "tenancyId", - 'projectUserId', - "projectUserId", - 'id', - "id" - ), - to_jsonb("ContactChannel".*), + 'TeamMember', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId", 'teamId', "teamId"), + to_jsonb("TeamMember".*), NOW(), TRUE - FROM "ContactChannel" + FROM "TeamMember" WHERE "tenancyId" = ${options.tenancyId}::uuid AND "projectUserId" = ${options.projectUserId}::uuid FOR UPDATE @@ -372,6 +528,35 @@ async function ensureClickhouseSchema( } } +// Map of target table name -> column normalizers for ClickHouse +// 'json' columns get JSON.stringify, 'boolean' columns get normalizeClickhouseBoolean +const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { + users: { + client_metadata: 'json', + client_read_only_metadata: 'json', + server_metadata: 'json', + primary_email_verified: 'boolean', + is_anonymous: 'boolean', + restricted_by_admin: 'boolean', + sync_is_deleted: 'boolean', + }, + contact_channels: { + is_primary: 'boolean', + is_verified: 'boolean', + used_for_auth: 'boolean', + sync_is_deleted: 'boolean', + }, + teams: { + client_metadata: 'json', + client_read_only_metadata: 'json', + server_metadata: 'json', + sync_is_deleted: 'boolean', + }, + team_members: { + sync_is_deleted: 'boolean', + }, +}; + async function pushRowsToClickhouse( client: ClickHouseClient, tableName: string, @@ -390,6 +575,10 @@ async function pushRowsToClickhouse( const sampleRow = newRows[0] ?? throwErr("Expected at least one row for ClickHouse sync."); const orderedKeys = Object.keys(omit(sampleRow, ["tenancyId"])); + // Derive the target table name from the full tableName (e.g. "analytics_internal.users" -> "users") + const targetTable = tableName.includes('.') ? tableName.split('.').pop()! : tableName; + const normalizers = CLICKHOUSE_COLUMN_NORMALIZERS[targetTable] ?? {}; + const normalizedRows = newRows.map((row) => { const tenancyIdValue = row.tenancyId; if (typeof tenancyIdValue !== "string") { @@ -427,17 +616,23 @@ async function pushRowsToClickhouse( `sync_sequence_id must be defined for ClickHouse sync. Mapping: ${mappingId}` ); } - return { + + const normalized: Record = { ...rest, sync_sequence_id: sequenceId, - client_metadata: JSON.stringify(rest.client_metadata), - client_read_only_metadata: JSON.stringify(rest.client_read_only_metadata), - server_metadata: JSON.stringify(rest.server_metadata), - primary_email_verified: normalizeClickhouseBoolean(rest.primary_email_verified, "primary_email_verified"), - is_anonymous: normalizeClickhouseBoolean(rest.is_anonymous, "is_anonymous"), - restricted_by_admin: normalizeClickhouseBoolean(rest.restricted_by_admin, "restricted_by_admin"), - sync_is_deleted: normalizeClickhouseBoolean(rest.sync_is_deleted, "sync_is_deleted"), }; + + for (const [col, type] of Object.entries(normalizers)) { + if (col in normalized) { + if (type === 'json') { + normalized[col] = JSON.stringify(normalized[col]); + } else { + normalized[col] = normalizeClickhouseBoolean(normalized[col], col); + } + } + } + + return normalized; }); await client.insert({ diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 31aaf597a1..4c7bceee8d 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -9,8 +9,14 @@ import { createProjectWithExternalDb as createProjectWithExternalDbRaw, verifyInExternalDb, verifyNotInExternalDb, + waitForSyncedContactChannel, + waitForSyncedContactChannelDeletion, waitForSyncedData, waitForSyncedDeletion, + waitForSyncedTeam, + waitForSyncedTeamDeletion, + waitForSyncedTeamMember, + waitForSyncedTeamMemberDeletion, waitForTable } from './external-db-sync-utils'; @@ -577,6 +583,260 @@ describe.sequential('External DB Sync - Basic Tests', () => { }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a team, verifies it in the external DB, updates it, verifies the update, + * deletes it, and verifies the removal. + */ + test('Team CRUD sync (Postgres)', async () => { + const dbName = 'team_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + // Create a team + const createResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Sync Test Team' }, + }); + expect(createResponse.status).toBe(201); + const teamId = createResponse.body.id; + + await waitForSyncedTeam(client, 'Sync Test Team'); + + const res1 = await client.query(`SELECT * FROM "teams" WHERE "id" = $1`, [teamId]); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].display_name).toBe('Sync Test Team'); + + // Update the team + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Team Name' }, + }); + + await waitForSyncedTeam(client, 'Updated Team Name'); + + const res2 = await client.query(`SELECT * FROM "teams" WHERE "id" = $1`, [teamId]); + expect(res2.rows[0].display_name).toBe('Updated Team Name'); + + // Delete the team + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team and verifies it appears via the ClickHouse analytics query API. + */ + test('Team sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const createResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'CH Team Test' }, + }); + expect(createResponse.status).toBe(201); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT display_name FROM teams WHERE display_name = {name:String}", + params: { name: 'CH Team Test' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].display_name).toBe('CH Team Test'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user and team, adds the user as a member, verifies in external DB, + * removes the member, and verifies removal. + */ + test('TeamMember CRUD sync (Postgres)', async () => { + const dbName = 'team_member_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'tm-crud@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TM CRUD Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Add user as team member + const addMemberResponse = await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(addMemberResponse.status).toBe(201); + + await waitForSyncedTeamMember(client, teamId, user.userId); + + const res1 = await client.query(`SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, user.userId]); + expect(res1.rows.length).toBe(1); + + // Remove member + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user with a primary email and verifies the contact channel appears + * in the external DB contact_channels table. + */ + test('ContactChannel sync (Postgres)', async () => { + const dbName = 'contact_channel_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cc-sync@example.com' }); + + await waitForSyncedContactChannel(client, 'cc-sync@example.com'); + + const res = await client.query(`SELECT * FROM "contact_channels" WHERE "value" = $1`, ['cc-sync@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].user_id).toBe(user.userId); + expect(res.rows[0].is_primary).toBe(true); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user in a team, deletes the user, and verifies the team_member is gone. + */ + test('Cascade: User delete removes team members from external DB', async () => { + const dbName = 'cascade_user_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-user-del@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade User Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamMember(client, teamId, user.userId); + + // Delete the user — should cascade-delete the team member + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team with a member, deletes the team, and verifies both team and member are gone. + */ + test('Cascade: Team delete removes team and members from external DB', async () => { + const dbName = 'cascade_team_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-team-del@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamMember(client, teamId, user.userId); + await waitForSyncedTeam(client, 'Cascade Team'); + + // Delete the team — should cascade-delete the member too + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); + }, TEST_TIMEOUT); + /** * What it does: * - Reads the external DB sync fusebox settings. diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 45281add2c..ae830bbe67 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -150,59 +150,66 @@ export async function waitForCondition( } /** - * Wait for data to appear in external DB (relies on automatic cron job) + * Generic helper to wait for a row to appear or disappear in the external DB. + * Handles the common pattern of catching "table does not exist" (42P01) errors. */ -export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { - +async function waitForExternalDbRow( + client: Client, + query: string, + params: unknown[], + opts: { shouldExist: boolean, description: string, checkRow?: (row: Record) => boolean }, +) { await waitForCondition( async () => { let res; try { - res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + res = await client.query(query, params); } catch (err: any) { if (err && err.code === '42P01') { return false; } throw err; } - if (res.rows.length === 0) { - return false; + if (opts.shouldExist) { + if (res.rows.length === 0) return false; + if (opts.checkRow && !opts.checkRow(res.rows[0])) return false; + return true; } - if (expectedName && res.rows[0].display_name !== expectedName) { - return false; - } - return true; + return res.rows.length === 0; }, { - description: `data for ${email} to appear in external DB`, + description: opts.description, timeoutMs: 180000, intervalMs: 500, } ); } +/** + * Wait for data to appear in external DB (relies on automatic cron job) + */ +export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [email], + { + shouldExist: true, + description: `data for ${email} to appear in external DB`, + checkRow: expectedName ? (row) => row.display_name === expectedName : undefined, + }, + ); +} + /** * Wait for data to be removed from external DB (relies on automatic cron job) */ export async function waitForSyncedDeletion(client: Client, email: string) { - await waitForCondition( - async () => { - let res; - try { - res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); - } catch (err: any) { - if (err && err.code === '42P01') { - return false; - } - throw err; - } - return res.rows.length === 0; - }, - { - description: `data for ${email} to be removed from external DB`, - timeoutMs: 180000, - intervalMs: 500, - } + await waitForExternalDbRow( + client, + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [email], + { shouldExist: false, description: `data for ${email} to be removed from external DB` }, ); } @@ -214,7 +221,7 @@ export async function waitForTable(client: Client, tableName: string) { async () => { const res = await client.query(` SELECT EXISTS ( - SELECT FROM information_schema.tables + SELECT FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1 ); @@ -265,6 +272,48 @@ export async function countUsersInExternalDb(client: Client): Promise { } } +export async function waitForSyncedTeam(client: Client, displayName: string) { + await waitForExternalDbRow(client, `SELECT * FROM "teams" WHERE "display_name" = $1`, [displayName], { + shouldExist: true, + description: `team "${displayName}" to appear in external DB`, + }); +} + +export async function waitForSyncedTeamDeletion(client: Client, teamId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "teams" WHERE "id" = $1`, [teamId], { + shouldExist: false, + description: `team ${teamId} to be removed from external DB`, + }); +} + +export async function waitForSyncedTeamMember(client: Client, teamId: string, userId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + shouldExist: true, + description: `team member (team=${teamId}, user=${userId}) to appear in external DB`, + }); +} + +export async function waitForSyncedTeamMemberDeletion(client: Client, teamId: string, userId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + shouldExist: false, + description: `team member (team=${teamId}, user=${userId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedContactChannel(client: Client, value: string) { + await waitForExternalDbRow(client, `SELECT * FROM "contact_channels" WHERE "value" = $1`, [value], { + shouldExist: true, + description: `contact channel "${value}" to appear in external DB`, + }); +} + +export async function waitForSyncedContactChannelDeletion(client: Client, value: string) { + await waitForExternalDbRow(client, `SELECT * FROM "contact_channels" WHERE "value" = $1`, [value], { + shouldExist: false, + description: `contact channel "${value}" to be removed from external DB`, + }); +} + /** * Helper to create a project and update its config with external DB settings. * Tracks the project for cleanup later. diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 65e839446c..92d6399970 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -275,4 +275,561 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "contact_channels": { + sourceTables: { "ContactChannel": "ContactChannel" }, + targetTable: "contact_channels", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "contact_channels" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "type" text NOT NULL, + "value" text NOT NULL, + "is_primary" boolean NOT NULL DEFAULT false, + "is_verified" boolean NOT NULL DEFAULT false, + "used_for_auth" boolean NOT NULL DEFAULT false, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "contact_channels" FROM PUBLIC; + GRANT SELECT ON "contact_channels" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.contact_channels ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + type LowCardinality(String), + value String, + is_primary UInt8, + is_verified UInt8, + used_for_auth UInt8, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ContactChannel"."id" AS "id", + "ContactChannel"."projectUserId" AS "user_id", + "ContactChannel"."type"::text AS "type", + "ContactChannel"."value" AS "value", + CASE WHEN "ContactChannel"."isPrimary" = 'TRUE' THEN true ELSE false END AS "is_primary", + "ContactChannel"."isVerified" AS "is_verified", + CASE WHEN "ContactChannel"."usedForAuth" = 'TRUE' THEN true ELSE false END AS "used_for_auth", + "ContactChannel"."createdAt" AS "created_at", + "ContactChannel"."sequenceId" AS "sync_sequence_id", + "ContactChannel"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ContactChannel" + JOIN "Tenancy" ON "Tenancy"."id" = "ContactChannel"."tenancyId" + WHERE "ContactChannel"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "type", + NULL::text AS "value", + false AS "is_primary", + false AS "is_verified", + false AS "used_for_auth", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ContactChannel' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ContactChannel"."id" AS "id", + "ContactChannel"."projectUserId" AS "user_id", + "ContactChannel"."type"::text AS "type", + "ContactChannel"."value" AS "value", + CASE WHEN "ContactChannel"."isPrimary" = 'TRUE' THEN true ELSE false END AS "is_primary", + "ContactChannel"."isVerified" AS "is_verified", + CASE WHEN "ContactChannel"."usedForAuth" = 'TRUE' THEN true ELSE false END AS "used_for_auth", + "ContactChannel"."createdAt" AS "created_at", + "ContactChannel"."sequenceId" AS "sequence_id", + "ContactChannel"."tenancyId", + false AS "is_deleted" + FROM "ContactChannel" + WHERE "ContactChannel"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "type", + NULL::text AS "value", + false AS "is_primary", + false AS "is_verified", + false AS "used_for_auth", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ContactChannel' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "type", + $4::text AS "value", + $5::boolean AS "is_primary", + $6::boolean AS "is_verified", + $7::boolean AS "used_for_auth", + $8::timestamp without time zone AS "created_at", + $9::bigint AS "sequence_id", + $10::boolean AS "is_deleted", + $11::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "contact_channels" c + USING params p + WHERE p."is_deleted" = true AND c."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "contact_channels" ( + "id", + "user_id", + "type", + "value", + "is_primary", + "is_verified", + "used_for_auth", + "created_at" + ) + SELECT + p."id", + p."user_id", + p."type", + p."value", + p."is_primary", + p."is_verified", + p."used_for_auth", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "type" = EXCLUDED."type", + "value" = EXCLUDED."value", + "is_primary" = EXCLUDED."is_primary", + "is_verified" = EXCLUDED."is_verified", + "used_for_auth" = EXCLUDED."used_for_auth", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "teams": { + sourceTables: { "Team": "Team" }, + targetTable: "teams", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "teams" ( + "id" uuid PRIMARY KEY NOT NULL, + "display_name" text NOT NULL, + "profile_image_url" text, + "created_at" timestamp without time zone NOT NULL, + "client_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "client_read_only_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "server_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb + ); + REVOKE ALL ON "teams" FROM PUBLIC; + GRANT SELECT ON "teams" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.teams ( + project_id String, + branch_id String, + id UUID, + display_name String, + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + client_metadata String, + client_read_only_metadata String, + server_metadata String, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "Team"."teamId" AS "id", + "Team"."displayName" AS "display_name", + "Team"."profileImageUrl" AS "profile_image_url", + "Team"."createdAt" AS "created_at", + COALESCE("Team"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("Team"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("Team"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "Team"."sequenceId" AS "sync_sequence_id", + "Team"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "Team" + JOIN "Tenancy" ON "Tenancy"."id" = "Team"."tenancyId" + WHERE "Team"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'Team' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "Team"."teamId" AS "id", + "Team"."displayName" AS "display_name", + "Team"."profileImageUrl" AS "profile_image_url", + "Team"."createdAt" AS "created_at", + COALESCE("Team"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("Team"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("Team"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "Team"."sequenceId" AS "sequence_id", + "Team"."tenancyId", + false AS "is_deleted" + FROM "Team" + WHERE "Team"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'Team' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "display_name", + $3::text AS "profile_image_url", + $4::timestamp without time zone AS "created_at", + $5::jsonb AS "client_metadata", + $6::jsonb AS "client_read_only_metadata", + $7::jsonb AS "server_metadata", + $8::bigint AS "sequence_id", + $9::boolean AS "is_deleted", + $10::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "teams" t + USING params p + WHERE p."is_deleted" = true AND t."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "teams" ( + "id", + "display_name", + "profile_image_url", + "created_at", + "client_metadata", + "client_read_only_metadata", + "server_metadata" + ) + SELECT + p."id", + p."display_name", + p."profile_image_url", + p."created_at", + p."client_metadata", + p."client_read_only_metadata", + p."server_metadata" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "created_at" = EXCLUDED."created_at", + "client_metadata" = EXCLUDED."client_metadata", + "client_read_only_metadata" = EXCLUDED."client_read_only_metadata", + "server_metadata" = EXCLUDED."server_metadata" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "team_members": { + sourceTables: { "TeamMember": "TeamMember" }, + targetTable: "team_members", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_members" ( + "team_id" uuid NOT NULL, + "user_id" uuid NOT NULL, + "display_name" text, + "profile_image_url" text, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("team_id", "user_id") + ); + REVOKE ALL ON "team_members" FROM PUBLIC; + GRANT SELECT ON "team_members" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_members ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + display_name Nullable(String), + profile_image_url Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, team_id, user_id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "TeamMember"."teamId" AS "team_id", + "TeamMember"."projectUserId" AS "user_id", + "TeamMember"."displayName" AS "display_name", + "TeamMember"."profileImageUrl" AS "profile_image_url", + "TeamMember"."createdAt" AS "created_at", + "TeamMember"."sequenceId" AS "sync_sequence_id", + "TeamMember"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "TeamMember" + JOIN "Tenancy" ON "Tenancy"."id" = "TeamMember"."tenancyId" + WHERE "TeamMember"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMember' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "TeamMember"."teamId" AS "team_id", + "TeamMember"."projectUserId" AS "user_id", + "TeamMember"."displayName" AS "display_name", + "TeamMember"."profileImageUrl" AS "profile_image_url", + "TeamMember"."createdAt" AS "created_at", + "TeamMember"."sequenceId" AS "sequence_id", + "TeamMember"."tenancyId", + false AS "is_deleted" + FROM "TeamMember" + WHERE "TeamMember"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMember' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "team_id", + $2::uuid AS "user_id", + $3::text AS "display_name", + $4::text AS "profile_image_url", + $5::timestamp without time zone AS "created_at", + $6::bigint AS "sequence_id", + $7::boolean AS "is_deleted", + $8::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_members" tm + USING params p + WHERE p."is_deleted" = true AND tm."team_id" = p."team_id" AND tm."user_id" = p."user_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_members" ( + "team_id", + "user_id", + "display_name", + "profile_image_url", + "created_at" + ) + SELECT + p."team_id", + p."user_id", + p."display_name", + p."profile_image_url", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("team_id", "user_id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From 9f9c9a46dcea6657e77fb137915437e85cd8c2ca Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 16 Mar 2026 12:36:07 -0700 Subject: [PATCH 02/14] clickhouse sync email outbox table --- .../migration.sql | 12 + apps/backend/prisma/schema.prisma | 5 + apps/backend/scripts/clickhouse-migrations.ts | 81 +++++ .../src/app/api/latest/emails/outbox/crud.tsx | 5 + .../external-db-sync/sequencer/route.ts | 30 +- .../internal/external-db-sync/status/route.ts | 27 +- apps/backend/src/lib/email-queue-step.tsx | 12 +- apps/backend/src/lib/external-db-sync.ts | 18 +- .../api/v1/external-db-sync-basics.test.ts | 228 +++++++++++++- .../api/v1/external-db-sync-utils.ts | 27 +- .../src/config/db-sync-mappings.ts | 288 ++++++++++++++++++ 11 files changed, 725 insertions(+), 8 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql diff --git a/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql new file mode 100644 index 0000000000..74b5681c01 --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql @@ -0,0 +1,12 @@ +-- AlterTable +ALTER TABLE "EmailOutbox" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "EmailOutbox_sequenceId_key" ON "EmailOutbox"("sequenceId"); + +-- CreateIndex +CREATE INDEX "EmailOutbox_tenancyId_sequenceId_idx" ON "EmailOutbox"("tenancyId", "sequenceId"); + +-- CreateIndex +CREATE INDEX "EmailOutbox_shouldUpdateSequenceId_idx" ON "EmailOutbox"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index d6077f3039..dad91817f7 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -1004,6 +1004,9 @@ model EmailOutbox { unsubscribedAt DateTime? markedAsSpamAt DateTime? + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@id([tenancyId, id]) @@ -1011,6 +1014,8 @@ model EmailOutbox { @@index([tenancyId, simpleStatus], map: "EmailOutbox_simple_status_tenancy_idx") @@index([tenancyId, status], map: "EmailOutbox_status_tenancy_idx") @@index([isQueued], map: "EmailOutbox_isQueued_idx") + @@index([tenancyId, sequenceId], name: "EmailOutbox_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "EmailOutbox_shouldUpdateSequenceId_idx") } model EmailOutboxProcessingMetadata { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index f0b533c034..f56870c00d 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -22,6 +22,8 @@ export async function runClickhouseMigrations() { await client.exec({ query: TEAMS_VIEW_SQL }); await client.exec({ query: TEAM_MEMBERS_TABLE_BASE_SQL }); await client.exec({ query: TEAM_MEMBERS_VIEW_SQL }); + await client.exec({ query: EMAIL_OUTBOXES_TABLE_BASE_SQL }); + await client.exec({ query: EMAIL_OUTBOXES_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -36,6 +38,7 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.contact_channels TO limited_user;", "GRANT SELECT ON default.teams TO limited_user;", "GRANT SELECT ON default.team_members TO limited_user;", + "GRANT SELECT ON default.email_outboxes TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -52,6 +55,9 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS team_members_project_isolation ON default.team_members FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS email_outboxes_project_isolation ON default.email_outboxes FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -330,6 +336,81 @@ FINAL WHERE sync_is_deleted = 0; `; +const EMAIL_OUTBOXES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( + project_id String, + branch_id String, + id UUID, + status LowCardinality(String), + simple_status LowCardinality(String), + created_with LowCardinality(String), + email_draft_id Nullable(String), + email_programmatic_call_template_id Nullable(String), + is_high_priority UInt8, + rendered_is_transactional Nullable(UInt8), + rendered_subject Nullable(String), + rendered_notification_category_id Nullable(String), + scheduled_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + started_sending_at Nullable(DateTime64(3, 'UTC')), + finished_sending_at Nullable(DateTime64(3, 'UTC')), + sent_at Nullable(DateTime64(3, 'UTC')), + delivered_at Nullable(DateTime64(3, 'UTC')), + opened_at Nullable(DateTime64(3, 'UTC')), + clicked_at Nullable(DateTime64(3, 'UTC')), + unsubscribed_at Nullable(DateTime64(3, 'UTC')), + marked_as_spam_at Nullable(DateTime64(3, 'UTC')), + bounced_at Nullable(DateTime64(3, 'UTC')), + can_have_delivery_info Nullable(UInt8), + skipped_reason LowCardinality(Nullable(String)), + send_retries Int32, + is_paused UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const EMAIL_OUTBOXES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.email_outboxes +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + status, + simple_status, + created_with, + email_draft_id, + email_programmatic_call_template_id, + is_high_priority, + rendered_is_transactional, + rendered_subject, + rendered_notification_category_id, + scheduled_at, + created_at, + started_sending_at, + finished_sending_at, + sent_at, + delivered_at, + opened_at, + clicked_at, + unsubscribed_at, + marked_as_spam_at, + bounced_at, + can_have_delivery_info, + skipped_reason, + send_retries, + is_paused +FROM analytics_internal.email_outboxes +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/src/app/api/latest/emails/outbox/crud.tsx b/apps/backend/src/app/api/latest/emails/outbox/crud.tsx index 2d36b499d8..73ec227026 100644 --- a/apps/backend/src/app/api/latest/emails/outbox/crud.tsx +++ b/apps/backend/src/app/api/latest/emails/outbox/crud.tsx @@ -447,6 +447,9 @@ export const emailOutboxCrudHandlers = createLazyProxy(() => createCrudHandlers( set("updatedAt", Prisma.sql`NOW()`); } + // Mark for external DB sync + set("shouldUpdateSequenceId", Prisma.sql`TRUE`); + const updateQuery: RawQuery = { supportedPrismaClients: ["global"], readOnlyQuery: false, @@ -543,6 +546,8 @@ function parseEmailOutboxFromJson(j: Record): EmailOutbox { clickedAt: dateOrNull("clickedAt"), unsubscribedAt: dateOrNull("unsubscribedAt"), markedAsSpamAt: dateOrNull("markedAsSpamAt"), + sequenceId: j.sequenceId != null ? BigInt(j.sequenceId as string | number) : null, + shouldUpdateSequenceId: j.shouldUpdateSequenceId as boolean, }; } diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index 78bbb99ffd..a7d30bdc1e 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -166,6 +166,34 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const emailOutboxTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "EmailOutbox" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "EmailOutbox" eo + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE eo."tenancyId" = r."tenancyId" + AND eo."id" = r."id" + RETURNING eo."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.email-outbox-tenants", emailOutboxTenants.length); + + if (emailOutboxTenants.length > 0) { + await enqueueExternalDbSyncBatch(emailOutboxTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -195,7 +223,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, EO=${emailOutboxTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts index 701c8818a0..f8de41b24f 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts @@ -87,6 +87,7 @@ const globalSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + email_outboxes: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -119,6 +120,7 @@ const responseSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + email_outboxes: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -233,6 +235,7 @@ function maxBigIntString(values: Array): string | nul function buildMappingInternalStats( projectUsersStats: SequenceStats, + emailOutboxStats: SequenceStats, deletedRowsByTable: DeletedRowSummary[], ) { const deletedProjectUserStats = deletedRowsByTable.find((row) => row.table_name === "ProjectUser") ?? null; @@ -264,6 +267,13 @@ function buildMappingInternalStats( internal_pending_count: usersMappingPending, }); + mappingInternalStats.set("email_outboxes", { + mapping_id: "email_outboxes", + internal_min_sequence_id: emailOutboxStats.min_sequence_id, + internal_max_sequence_id: emailOutboxStats.max_sequence_id, + internal_pending_count: emailOutboxStats.pending, + }); + const mappings = Array.from(mappingInternalStats.values()); const mappingStatuses = mappings.map((mapping) => ({ mapping_id: mapping.mapping_id, @@ -300,6 +310,17 @@ async function fetchInternalStats(tenancyId: string | null) { ${tenancyWhere} `).at(0) ?? throwErr("Contact channel stats query returned no rows."); + const emailOutboxStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "EmailOutbox" + ${tenancyWhere} + `).at(0) ?? throwErr("Email outbox stats query returned no rows."); + const deletedRowStatsRow = (await globalPrismaClient.$queryRaw` SELECT COUNT(*)::bigint AS "total", @@ -346,6 +367,7 @@ async function fetchInternalStats(tenancyId: string | null) { const projectUsersStats = formatSequenceStats(projectUserStatsRow); const contactChannelStats = formatSequenceStats(contactChannelStatsRow); + const emailOutboxStats = formatSequenceStats(emailOutboxStatsRow); const deletedRowStats = formatSequenceStats(deletedRowStatsRow); const deletedRowsByTable = deletedRowsByTableRows.map((row) => ({ @@ -353,11 +375,12 @@ async function fetchInternalStats(tenancyId: string | null) { ...formatSequenceStats(row), })); - const { mappings, mappingStatuses } = buildMappingInternalStats(projectUsersStats, deletedRowsByTable); + const { mappings, mappingStatuses } = buildMappingInternalStats(projectUsersStats, emailOutboxStats, deletedRowsByTable); return { projectUsersStats, contactChannelStats, + emailOutboxStats, deletedRowStats, deletedRowsByTable, outgoingStatsRow, @@ -1003,6 +1026,7 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: globalStats.projectUsersStats, contact_channels: globalStats.contactChannelStats, + email_outboxes: globalStats.emailOutboxStats, deleted_rows: { ...globalStats.deletedRowStats, by_table: globalStats.deletedRowsByTable, @@ -1021,6 +1045,7 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: currentStats.projectUsersStats, contact_channels: currentStats.contactChannelStats, + email_outboxes: currentStats.emailOutboxStats, deleted_rows: { ...currentStats.deletedRowStats, by_table: currentStats.deletedRowsByTable, diff --git a/apps/backend/src/lib/email-queue-step.tsx b/apps/backend/src/lib/email-queue-step.tsx index b98ebb812c..17456c7728 100644 --- a/apps/backend/src/lib/email-queue-step.tsx +++ b/apps/backend/src/lib/email-queue-step.tsx @@ -195,6 +195,7 @@ async function retryEmailsStuckInRendering(): Promise { data: { renderedByWorkerId: null, startedRenderingAt: null, + shouldUpdateSequenceId: true, }, }); if (res.length > 0) { @@ -398,6 +399,7 @@ async function renderTenancyEmails(workerId: string, tenancyId: string, group: E renderErrorInternalMessage: error, renderErrorInternalDetails: { error }, finishedRenderingAt: new Date(), + shouldUpdateSequenceId: true, }, }); }; @@ -418,6 +420,7 @@ async function renderTenancyEmails(workerId: string, tenancyId: string, group: E renderErrorInternalMessage: null, renderErrorInternalDetails: Prisma.DbNull, finishedRenderingAt: new Date(), + shouldUpdateSequenceId: true, }, }); }; @@ -508,7 +511,7 @@ async function queueReadyEmails(): Promise<{ queuedCount: number }> { // Query 1: Fresh emails (scheduledAt has passed, no retry pending) const freshEmails = await globalPrismaClient.$queryRaw<{ id: string }[]>` UPDATE "EmailOutbox" - SET "isQueued" = TRUE + SET "isQueued" = TRUE, "shouldUpdateSequenceId" = TRUE WHERE "isQueued" = FALSE AND "isPaused" = FALSE AND "skippedReason" IS NULL @@ -523,7 +526,7 @@ async function queueReadyEmails(): Promise<{ queuedCount: number }> { // Clear nextSendRetryAt when queuing so the email is in a clean "queued" state. const retryEmails = await globalPrismaClient.$queryRaw<{ id: string }[]>` UPDATE "EmailOutbox" - SET "isQueued" = TRUE, "nextSendRetryAt" = NULL + SET "isQueued" = TRUE, "nextSendRetryAt" = NULL, "shouldUpdateSequenceId" = TRUE WHERE "isQueued" = FALSE AND "isPaused" = FALSE AND "skippedReason" IS NULL @@ -749,6 +752,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendRetries: newAttemptCount, nextSendRetryAt: new Date(Date.now() + backoffMs), sendAttemptErrors: updatedErrors as Prisma.InputJsonArray, + shouldUpdateSequenceId: true, }, }); } else { @@ -789,6 +793,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO failureReason, allAttemptErrors: updatedErrors as Json[], }, + shouldUpdateSequenceId: true, }, }); } @@ -809,6 +814,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendServerErrorExternalDetails: Prisma.DbNull, sendServerErrorInternalMessage: null, sendServerErrorInternalDetails: Prisma.DbNull, + shouldUpdateSequenceId: true, }, }); } @@ -829,6 +835,7 @@ async function processSingleEmail(context: TenancyProcessingContext, row: EmailO sendServerErrorExternalDetails: {}, sendServerErrorInternalMessage: errorToNiceString(error), sendServerErrorInternalDetails: {}, + shouldUpdateSequenceId: true, }, }); } @@ -914,6 +921,7 @@ async function markSkipped(row: EmailOutbox, reason: EmailOutboxSkippedReason, d data: { skippedReason: reason, skippedDetails: details as Prisma.InputJsonValue, + shouldUpdateSequenceId: true, }, }); } diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index bd9b0b1cab..519930537e 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -499,6 +499,13 @@ function normalizeClickhouseBoolean(value: unknown, label: string): number { throw new StackAssertionError(`${label} must be a boolean or 0/1. Received: ${JSON.stringify(value)}`); } +function normalizeClickhouseNullableBoolean(value: unknown, label: string): number | null { + if (value === null || value === undefined) { + return null; + } + return normalizeClickhouseBoolean(value, label); +} + function parseSequenceId(value: unknown, mappingId: string): number | null { if (value == null) { return null; @@ -530,7 +537,7 @@ async function ensureClickhouseSchema( // Map of target table name -> column normalizers for ClickHouse // 'json' columns get JSON.stringify, 'boolean' columns get normalizeClickhouseBoolean -const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { +const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { users: { client_metadata: 'json', client_read_only_metadata: 'json', @@ -555,6 +562,13 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record { let dbManager: TestDbManager; const createProjectWithExternalDb = ( externalDatabases: any, - projectOptions?: { display_name?: string, description?: string } + projectOptions?: { display_name?: string, description?: string, config?: Record } ) => { return createProjectWithExternalDbRaw( externalDatabases, @@ -837,6 +840,227 @@ describe.sequential('External DB Sync - Basic Tests', () => { await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a project with email config, sends an email, and verifies + * the email outbox row is synced to the external Postgres DB. + */ + test('EmailOutbox sync (Postgres)', async () => { + const dbName = 'email_outbox_pg_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: 'Email Outbox Sync Test', + config: { + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + // Create a user + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + // Send an email + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

Sync test email

", + subject: "DB Sync Test Email", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + // Wait for the email to be processed (rendered + sent) + await wait(8_000); + + // Get the email ID from the outbox API + const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { + method: "GET", + accessType: "server", + }); + expect(listResponse.status).toBe(200); + expect(listResponse.body.items.length).toBeGreaterThanOrEqual(1); + const emailId = listResponse.body.items[0].id; + + const client = dbManager.getClient(dbName); + + // Wait for the email outbox row to appear in external DB + await waitForSyncedEmailOutbox(client, emailId); + + // Verify the synced row has expected columns + const res = await client.query(`SELECT * FROM "email_outboxes" WHERE "id" = $1`, [emailId]); + expect(res.rows.length).toBe(1); + const row = res.rows[0]; + expect(row.created_with).toBe('PROGRAMMATIC_CALL'); + expect(row.is_high_priority).toBe(false); + expect(row.is_paused).toBe(false); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a project, sends an email, and verifies the email outbox row + * is synced to ClickHouse. + */ + test('EmailOutbox sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ + config: { + magic_link_enabled: true, + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + // Create a user + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + // Send an email + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

ClickHouse sync test email

", + subject: "CH Sync Test Email", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + // Wait for the email to be processed + await wait(8_000); + + await InternalApiKey.createAndSetProjectKeys(); + + // Poll ClickHouse until the email_outboxes row appears + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, status, simple_status, created_with, is_high_priority FROM email_outboxes LIMIT 10", + }); + expect(response.status).toBe(200); + if (response.body.result.length >= 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBeGreaterThanOrEqual(1); + const row = response!.body.result[0]; + expect(row.created_with).toBe('PROGRAMMATIC_CALL'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends an email, waits for it to reach a terminal state, then verifies + * the status update is reflected in the external Postgres DB. + */ + test('EmailOutbox status updates are synced (Postgres)', async () => { + const dbName = 'email_outbox_status_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + config: { + email_config: { + type: "standard", + host: "localhost", + port: Number(withPortPrefix("29")), + username: "test", + password: "test", + sender_name: "Test Project", + sender_email: "test@example.com", + }, + }, + }); + + const createUserResponse = await niceBackendFetch("/api/v1/users", { + method: "POST", + accessType: "server", + body: { + primary_email: backendContext.value.mailbox.emailAddress, + primary_email_verified: true, + }, + }); + expect(createUserResponse.status).toBe(201); + const userId = createUserResponse.body.id; + + const sendResponse = await niceBackendFetch("/api/v1/emails/send-email", { + method: "POST", + accessType: "server", + body: { + user_ids: [userId], + html: "

Status sync test

", + subject: "Status Sync Test", + notification_category_name: "Transactional", + }, + }); + expect(sendResponse.status).toBe(200); + + // Wait for the email to finish sending + await wait(8_000); + + const client = dbManager.getClient(dbName); + + // The email should eventually reach SENT status in the external DB + await waitForSyncedEmailOutboxByStatus(client, 'SENT'); + + const res = await client.query(`SELECT * FROM "email_outboxes" WHERE "status" = 'SENT'`); + expect(res.rows.length).toBeGreaterThanOrEqual(1); + const row = res.rows[0]; + expect(row.simple_status).toBe('OK'); + expect(row.finished_sending_at).not.toBeNull(); + expect(row.sent_at).not.toBeNull(); + expect(row.send_retries).toBe(0); + }, TEST_TIMEOUT); + /** * What it does: * - Reads the external DB sync fusebox settings. diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index ae830bbe67..73a2f29b91 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -314,13 +314,38 @@ export async function waitForSyncedContactChannelDeletion(client: Client, value: }); } +export async function waitForSyncedEmailOutbox(client: Client, emailId: string, expectedStatus?: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "email_outboxes" WHERE "id" = $1`, + [emailId], + { + shouldExist: true, + description: `email outbox "${emailId}" to appear in external DB`, + checkRow: expectedStatus ? (row) => row.status === expectedStatus : undefined, + }, + ); +} + +export async function waitForSyncedEmailOutboxByStatus(client: Client, status: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "email_outboxes" WHERE "status" = $1`, + [status], + { + shouldExist: true, + description: `email outbox with status "${status}" to appear in external DB`, + }, + ); +} + /** * Helper to create a project and update its config with external DB settings. * Tracks the project for cleanup later. */ export async function createProjectWithExternalDb( externalDatabases: any, - projectOptions?: { display_name?: string, description?: string }, + projectOptions?: { display_name?: string, description?: string, config?: Record }, options?: { projectTracker?: ProjectContext[] } ) { const project = await Project.createAndSwitch(projectOptions); diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 92d6399970..d77641e1a4 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -832,4 +832,292 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "email_outboxes": { + sourceTables: { "EmailOutbox": "EmailOutbox" }, + targetTable: "email_outboxes", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "email_outboxes" ( + "id" uuid PRIMARY KEY NOT NULL, + "status" text NOT NULL, + "simple_status" text NOT NULL, + "created_with" text NOT NULL, + "email_draft_id" text, + "email_programmatic_call_template_id" text, + "is_high_priority" boolean NOT NULL DEFAULT false, + "rendered_is_transactional" boolean, + "rendered_subject" text, + "rendered_notification_category_id" text, + "scheduled_at" timestamp without time zone NOT NULL, + "created_at" timestamp without time zone NOT NULL, + "started_sending_at" timestamp without time zone, + "finished_sending_at" timestamp without time zone, + "sent_at" timestamp without time zone, + "delivered_at" timestamp without time zone, + "opened_at" timestamp without time zone, + "clicked_at" timestamp without time zone, + "unsubscribed_at" timestamp without time zone, + "marked_as_spam_at" timestamp without time zone, + "bounced_at" timestamp without time zone, + "can_have_delivery_info" boolean, + "skipped_reason" text, + "send_retries" integer NOT NULL DEFAULT 0, + "is_paused" boolean NOT NULL DEFAULT false + ); + REVOKE ALL ON "email_outboxes" FROM PUBLIC; + GRANT SELECT ON "email_outboxes" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( + project_id String, + branch_id String, + id UUID, + status LowCardinality(String), + simple_status LowCardinality(String), + created_with LowCardinality(String), + email_draft_id Nullable(String), + email_programmatic_call_template_id Nullable(String), + is_high_priority UInt8, + rendered_is_transactional Nullable(UInt8), + rendered_subject Nullable(String), + rendered_notification_category_id Nullable(String), + scheduled_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + started_sending_at Nullable(DateTime64(3, 'UTC')), + finished_sending_at Nullable(DateTime64(3, 'UTC')), + sent_at Nullable(DateTime64(3, 'UTC')), + delivered_at Nullable(DateTime64(3, 'UTC')), + opened_at Nullable(DateTime64(3, 'UTC')), + clicked_at Nullable(DateTime64(3, 'UTC')), + unsubscribed_at Nullable(DateTime64(3, 'UTC')), + marked_as_spam_at Nullable(DateTime64(3, 'UTC')), + bounced_at Nullable(DateTime64(3, 'UTC')), + can_have_delivery_info Nullable(UInt8), + skipped_reason LowCardinality(Nullable(String)), + send_retries Int32, + is_paused UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "EmailOutbox"."id" AS "id", + "EmailOutbox"."status"::text AS "status", + "EmailOutbox"."simpleStatus"::text AS "simple_status", + "EmailOutbox"."createdWith"::text AS "created_with", + "EmailOutbox"."emailDraftId" AS "email_draft_id", + "EmailOutbox"."emailProgrammaticCallTemplateId" AS "email_programmatic_call_template_id", + "EmailOutbox"."isHighPriority" AS "is_high_priority", + "EmailOutbox"."renderedIsTransactional" AS "rendered_is_transactional", + "EmailOutbox"."renderedSubject" AS "rendered_subject", + "EmailOutbox"."renderedNotificationCategoryId" AS "rendered_notification_category_id", + "EmailOutbox"."scheduledAt" AS "scheduled_at", + "EmailOutbox"."createdAt" AS "created_at", + "EmailOutbox"."startedSendingAt" AS "started_sending_at", + "EmailOutbox"."finishedSendingAt" AS "finished_sending_at", + "EmailOutbox"."sentAt" AS "sent_at", + "EmailOutbox"."deliveredAt" AS "delivered_at", + "EmailOutbox"."openedAt" AS "opened_at", + "EmailOutbox"."clickedAt" AS "clicked_at", + "EmailOutbox"."unsubscribedAt" AS "unsubscribed_at", + "EmailOutbox"."markedAsSpamAt" AS "marked_as_spam_at", + "EmailOutbox"."bouncedAt" AS "bounced_at", + "EmailOutbox"."canHaveDeliveryInfo" AS "can_have_delivery_info", + "EmailOutbox"."skippedReason"::text AS "skipped_reason", + "EmailOutbox"."sendRetries" AS "send_retries", + "EmailOutbox"."isPaused" AS "is_paused", + "EmailOutbox"."sequenceId" AS "sync_sequence_id", + "EmailOutbox"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "EmailOutbox" + JOIN "Tenancy" ON "Tenancy"."id" = "EmailOutbox"."tenancyId" + WHERE "EmailOutbox"."tenancyId" = $1::uuid + AND "EmailOutbox"."sequenceId" IS NOT NULL + AND "EmailOutbox"."sequenceId" > $2::bigint + ORDER BY "EmailOutbox"."sequenceId" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT + "EmailOutbox"."id" AS "id", + "EmailOutbox"."status"::text AS "status", + "EmailOutbox"."simpleStatus"::text AS "simple_status", + "EmailOutbox"."createdWith"::text AS "created_with", + "EmailOutbox"."emailDraftId" AS "email_draft_id", + "EmailOutbox"."emailProgrammaticCallTemplateId" AS "email_programmatic_call_template_id", + "EmailOutbox"."isHighPriority" AS "is_high_priority", + "EmailOutbox"."renderedIsTransactional" AS "rendered_is_transactional", + "EmailOutbox"."renderedSubject" AS "rendered_subject", + "EmailOutbox"."renderedNotificationCategoryId" AS "rendered_notification_category_id", + "EmailOutbox"."scheduledAt" AS "scheduled_at", + "EmailOutbox"."createdAt" AS "created_at", + "EmailOutbox"."startedSendingAt" AS "started_sending_at", + "EmailOutbox"."finishedSendingAt" AS "finished_sending_at", + "EmailOutbox"."sentAt" AS "sent_at", + "EmailOutbox"."deliveredAt" AS "delivered_at", + "EmailOutbox"."openedAt" AS "opened_at", + "EmailOutbox"."clickedAt" AS "clicked_at", + "EmailOutbox"."unsubscribedAt" AS "unsubscribed_at", + "EmailOutbox"."markedAsSpamAt" AS "marked_as_spam_at", + "EmailOutbox"."bouncedAt" AS "bounced_at", + "EmailOutbox"."canHaveDeliveryInfo" AS "can_have_delivery_info", + "EmailOutbox"."skippedReason"::text AS "skipped_reason", + "EmailOutbox"."sendRetries" AS "send_retries", + "EmailOutbox"."isPaused" AS "is_paused", + "EmailOutbox"."sequenceId" AS "sequence_id", + "EmailOutbox"."tenancyId", + false AS "is_deleted" + FROM "EmailOutbox" + WHERE "EmailOutbox"."tenancyId" = $1::uuid + AND "EmailOutbox"."sequenceId" IS NOT NULL + AND "EmailOutbox"."sequenceId" > $2::bigint + ORDER BY "EmailOutbox"."sequenceId" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "status", + $3::text AS "simple_status", + $4::text AS "created_with", + $5::text AS "email_draft_id", + $6::text AS "email_programmatic_call_template_id", + $7::boolean AS "is_high_priority", + $8::boolean AS "rendered_is_transactional", + $9::text AS "rendered_subject", + $10::text AS "rendered_notification_category_id", + $11::timestamp without time zone AS "scheduled_at", + $12::timestamp without time zone AS "created_at", + $13::timestamp without time zone AS "started_sending_at", + $14::timestamp without time zone AS "finished_sending_at", + $15::timestamp without time zone AS "sent_at", + $16::timestamp without time zone AS "delivered_at", + $17::timestamp without time zone AS "opened_at", + $18::timestamp without time zone AS "clicked_at", + $19::timestamp without time zone AS "unsubscribed_at", + $20::timestamp without time zone AS "marked_as_spam_at", + $21::timestamp without time zone AS "bounced_at", + $22::boolean AS "can_have_delivery_info", + $23::text AS "skipped_reason", + $24::integer AS "send_retries", + $25::boolean AS "is_paused", + $26::bigint AS "sequence_id", + $27::boolean AS "is_deleted", + $28::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "email_outboxes" eo + USING params p + WHERE p."is_deleted" = true AND eo."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "email_outboxes" ( + "id", + "status", + "simple_status", + "created_with", + "email_draft_id", + "email_programmatic_call_template_id", + "is_high_priority", + "rendered_is_transactional", + "rendered_subject", + "rendered_notification_category_id", + "scheduled_at", + "created_at", + "started_sending_at", + "finished_sending_at", + "sent_at", + "delivered_at", + "opened_at", + "clicked_at", + "unsubscribed_at", + "marked_as_spam_at", + "bounced_at", + "can_have_delivery_info", + "skipped_reason", + "send_retries", + "is_paused" + ) + SELECT + p."id", + p."status", + p."simple_status", + p."created_with", + p."email_draft_id", + p."email_programmatic_call_template_id", + p."is_high_priority", + p."rendered_is_transactional", + p."rendered_subject", + p."rendered_notification_category_id", + p."scheduled_at", + p."created_at", + p."started_sending_at", + p."finished_sending_at", + p."sent_at", + p."delivered_at", + p."opened_at", + p."clicked_at", + p."unsubscribed_at", + p."marked_as_spam_at", + p."bounced_at", + p."can_have_delivery_info", + p."skipped_reason", + p."send_retries", + p."is_paused" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "status" = EXCLUDED."status", + "simple_status" = EXCLUDED."simple_status", + "created_with" = EXCLUDED."created_with", + "email_draft_id" = EXCLUDED."email_draft_id", + "email_programmatic_call_template_id" = EXCLUDED."email_programmatic_call_template_id", + "is_high_priority" = EXCLUDED."is_high_priority", + "rendered_is_transactional" = EXCLUDED."rendered_is_transactional", + "rendered_subject" = EXCLUDED."rendered_subject", + "rendered_notification_category_id" = EXCLUDED."rendered_notification_category_id", + "scheduled_at" = EXCLUDED."scheduled_at", + "created_at" = EXCLUDED."created_at", + "started_sending_at" = EXCLUDED."started_sending_at", + "finished_sending_at" = EXCLUDED."finished_sending_at", + "sent_at" = EXCLUDED."sent_at", + "delivered_at" = EXCLUDED."delivered_at", + "opened_at" = EXCLUDED."opened_at", + "clicked_at" = EXCLUDED."clicked_at", + "unsubscribed_at" = EXCLUDED."unsubscribed_at", + "marked_as_spam_at" = EXCLUDED."marked_as_spam_at", + "bounced_at" = EXCLUDED."bounced_at", + "can_have_delivery_info" = EXCLUDED."can_have_delivery_info", + "skipped_reason" = EXCLUDED."skipped_reason", + "send_retries" = EXCLUDED."send_retries", + "is_paused" = EXCLUDED."is_paused" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From 008c6b083f2571acd5ff4da8a5f31a237310280d Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 16 Mar 2026 13:47:35 -0700 Subject: [PATCH 03/14] clickhouse sync session replays --- .../migration.sql | 12 ++ apps/backend/prisma/schema.prisma | 5 + apps/backend/scripts/clickhouse-migrations.ts | 36 +++++ .../external-db-sync/sequencer/route.ts | 30 +++- .../latest/session-replays/batch/route.tsx | 2 + apps/backend/src/lib/external-db-sync.ts | 3 + .../api/v1/external-db-sync-basics.test.ts | 127 +++++++++++++++- .../api/v1/external-db-sync-utils.ts | 12 ++ .../src/config/db-sync-mappings.ts | 136 ++++++++++++++++++ 9 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql diff --git a/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql new file mode 100644 index 0000000000..88e0751fd3 --- /dev/null +++ b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql @@ -0,0 +1,12 @@ +-- AlterTable +ALTER TABLE "SessionReplay" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "SessionReplay_sequenceId_key" ON "SessionReplay"("sequenceId"); + +-- CreateIndex +CREATE INDEX "SessionReplay_tenancyId_sequenceId_idx" ON "SessionReplay"("tenancyId", "sequenceId"); + +-- CreateIndex +CREATE INDEX "SessionReplay_shouldUpdateSequenceId_idx" ON "SessionReplay"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index dad91817f7..b795705977 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -356,6 +356,9 @@ model SessionReplay { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@ -367,6 +370,8 @@ model SessionReplay { @@index([tenancyId, lastEventAt]) // index by updatedAt instead of lastEventAt because event timing can be spoofed @@index([tenancyId, refreshTokenId, updatedAt]) + @@index([tenancyId, sequenceId], name: "SessionReplay_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "SessionReplay_shouldUpdateSequenceId_idx") } model SessionReplayChunk { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index f56870c00d..f5cd50d1d8 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -24,6 +24,8 @@ export async function runClickhouseMigrations() { await client.exec({ query: TEAM_MEMBERS_VIEW_SQL }); await client.exec({ query: EMAIL_OUTBOXES_TABLE_BASE_SQL }); await client.exec({ query: EMAIL_OUTBOXES_VIEW_SQL }); + await client.exec({ query: SESSION_REPLAYS_TABLE_BASE_SQL }); + await client.exec({ query: SESSION_REPLAYS_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -39,6 +41,7 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.teams TO limited_user;", "GRANT SELECT ON default.team_members TO limited_user;", "GRANT SELECT ON default.email_outboxes TO limited_user;", + "GRANT SELECT ON default.session_replays TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -58,6 +61,9 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS email_outboxes_project_isolation ON default.email_outboxes FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS session_replays_project_isolation ON default.session_replays FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -411,6 +417,36 @@ FINAL WHERE sync_is_deleted = 0; `; +const SESSION_REPLAYS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.session_replays ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + refresh_token_id String, + started_at DateTime64(3, 'UTC'), + last_event_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(started_at) +ORDER BY (project_id, branch_id, id); +`; + +const SESSION_REPLAYS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.session_replays +SQL SECURITY DEFINER +AS +SELECT project_id, branch_id, id, user_id, refresh_token_id, + started_at, last_event_at, created_at +FROM analytics_internal.session_replays +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index a7d30bdc1e..e9d60ce88b 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -194,6 +194,34 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const sessionReplayTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "SessionReplay" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "SessionReplay" sr + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE sr."tenancyId" = r."tenancyId" + AND sr."id" = r."id" + RETURNING sr."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.session-replay-tenants", sessionReplayTenants.length); + + if (sessionReplayTenants.length > 0) { + await enqueueExternalDbSyncBatch(sessionReplayTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -223,7 +251,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, EO=${emailOutboxTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, EO=${emailOutboxTenants.length}, SR=${sessionReplayTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx index 57e1a162e4..25dccb4c78 100644 --- a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx +++ b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx @@ -120,10 +120,12 @@ export const POST = createSmartRouteHandler({ refreshTokenId, startedAt: new Date(firstMs), lastEventAt: new Date(newLastEventAtMs), + shouldUpdateSequenceId: true, }, update: { startedAt: new Date(newStartedAtMs), lastEventAt: new Date(newLastEventAtMs), + shouldUpdateSequenceId: true, }, }); diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 519930537e..ffd24b6870 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -569,6 +569,9 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record { expect(row.send_retries).toBe(0); }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a project with analytics, signs in a user, uploads a session replay batch, + * and verifies the session replay row is synced to ClickHouse. + */ + test('SessionReplay sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ + config: { + magic_link_enabled: true, + }, + }); + await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } }); + await Auth.Otp.signIn(); + + const now = Date.now(); + const browserSessionId = randomUUID(); + const batchId = randomUUID(); + + const uploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { + method: "POST", + accessType: "client", + body: { + browser_session_id: browserSessionId, + session_replay_segment_id: randomUUID(), + batch_id: batchId, + started_at_ms: now, + sent_at_ms: now + 500, + events: [ + { timestamp: now + 100, type: 2 }, + { timestamp: now + 200, type: 3 }, + ], + }, + }); + expect(uploadRes.status).toBe(200); + expect(uploadRes.body.deduped).toBe(false); + + await InternalApiKey.createAndSetProjectKeys(); + + // Poll ClickHouse until the session_replays row appears + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, user_id, refresh_token_id, started_at, last_event_at FROM session_replays LIMIT 10", + }); + expect(response.status).toBe(200); + if (response.body.result.length >= 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBeGreaterThanOrEqual(1); + const row = response!.body.result[0]; + expect(row.user_id).toBeDefined(); + expect(row.refresh_token_id).toBeDefined(); + expect(row.started_at).toBeDefined(); + expect(row.last_event_at).toBeDefined(); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a project with an external Postgres DB, signs in a user, + * uploads a session replay batch, and verifies the row is synced to external Postgres. + */ + test('SessionReplay sync (Postgres)', async () => { + const dbName = 'session_replay_pg_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: 'Session Replay Sync Test', + config: { + magic_link_enabled: true, + }, + }); + await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } }); + await Auth.Otp.signIn(); + + const now = Date.now(); + const browserSessionId = randomUUID(); + const batchId = randomUUID(); + + const uploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { + method: "POST", + accessType: "client", + body: { + browser_session_id: browserSessionId, + session_replay_segment_id: randomUUID(), + batch_id: batchId, + started_at_ms: now, + sent_at_ms: now + 500, + events: [ + { timestamp: now + 100, type: 2 }, + { timestamp: now + 200, type: 3 }, + ], + }, + }); + expect(uploadRes.status).toBe(200); + const replayId = uploadRes.body.session_replay_id; + + const client = dbManager.getClient(dbName); + + // Wait for the session replay row to appear in external DB + await waitForSyncedSessionReplay(client, replayId); + + // Verify the synced row has expected columns + const res = await client.query(`SELECT * FROM "session_replays" WHERE "id" = $1`, [replayId]); + expect(res.rows.length).toBe(1); + const row = res.rows[0]; + expect(row.user_id).toBeDefined(); + expect(row.refresh_token_id).toBeDefined(); + expect(row.started_at).toBeDefined(); + expect(row.last_event_at).toBeDefined(); + }, TEST_TIMEOUT); + /** * What it does: * - Reads the external DB sync fusebox settings. diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 73a2f29b91..f594691f1b 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -327,6 +327,18 @@ export async function waitForSyncedEmailOutbox(client: Client, emailId: string, ); } +export async function waitForSyncedSessionReplay(client: Client, replayId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "session_replays" WHERE "id" = $1`, + [replayId], + { + shouldExist: true, + description: `session replay "${replayId}" to appear in external DB`, + }, + ); +} + export async function waitForSyncedEmailOutboxByStatus(client: Client, status: string) { await waitForExternalDbRow( client, diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index d77641e1a4..b9b206f7e2 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -1120,4 +1120,140 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "session_replays": { + sourceTables: { "SessionReplay": "SessionReplay" }, + targetTable: "session_replays", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "session_replays" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "refresh_token_id" text NOT NULL, + "started_at" timestamp without time zone NOT NULL, + "last_event_at" timestamp without time zone NOT NULL, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "session_replays" FROM PUBLIC; + GRANT SELECT ON "session_replays" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.session_replays ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + refresh_token_id String, + started_at DateTime64(3, 'UTC'), + last_event_at DateTime64(3, 'UTC'), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(started_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "SessionReplay"."id" AS "id", + "SessionReplay"."projectUserId" AS "user_id", + "SessionReplay"."refreshTokenId" AS "refresh_token_id", + "SessionReplay"."startedAt" AS "started_at", + "SessionReplay"."lastEventAt" AS "last_event_at", + "SessionReplay"."createdAt" AS "created_at", + "SessionReplay"."sequenceId" AS "sync_sequence_id", + "SessionReplay"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "SessionReplay" + JOIN "Tenancy" ON "Tenancy"."id" = "SessionReplay"."tenancyId" + WHERE "SessionReplay"."tenancyId" = $1::uuid + AND "SessionReplay"."sequenceId" IS NOT NULL + AND "SessionReplay"."sequenceId" > $2::bigint + ORDER BY "SessionReplay"."sequenceId" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT + "SessionReplay"."id" AS "id", + "SessionReplay"."projectUserId" AS "user_id", + "SessionReplay"."refreshTokenId" AS "refresh_token_id", + "SessionReplay"."startedAt" AS "started_at", + "SessionReplay"."lastEventAt" AS "last_event_at", + "SessionReplay"."createdAt" AS "created_at", + "SessionReplay"."sequenceId" AS "sequence_id", + "SessionReplay"."tenancyId", + false AS "is_deleted" + FROM "SessionReplay" + WHERE "SessionReplay"."tenancyId" = $1::uuid + AND "SessionReplay"."sequenceId" IS NOT NULL + AND "SessionReplay"."sequenceId" > $2::bigint + ORDER BY "SessionReplay"."sequenceId" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "refresh_token_id", + $4::timestamp without time zone AS "started_at", + $5::timestamp without time zone AS "last_event_at", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "session_replays" sr + USING params p + WHERE p."is_deleted" = true AND sr."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "session_replays" ( + "id", + "user_id", + "refresh_token_id", + "started_at", + "last_event_at", + "created_at" + ) + SELECT + p."id", + p."user_id", + p."refresh_token_id", + p."started_at", + p."last_event_at", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "refresh_token_id" = EXCLUDED."refresh_token_id", + "started_at" = EXCLUDED."started_at", + "last_event_at" = EXCLUDED."last_event_at", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From 10b2f19de1e4370f40c407790f325c7a08c758d6 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 16 Mar 2026 14:20:44 -0700 Subject: [PATCH 04/14] fix analytics query test --- .../endpoints/api/v1/analytics-query.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index 8e847c7d82..0760e64935 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -522,7 +522,12 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON SQLite FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON URL FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.session_replays TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_members TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.teams TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.users TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.aggregate_function_combinators TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.collations TO limited_user" }, @@ -561,10 +566,30 @@ it("can see only some tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { + "database": "default", + "name": "contact_channels", + }, + { + "database": "default", + "name": "email_outboxes", + }, { "database": "default", "name": "events", }, + { + "database": "default", + "name": "session_replays", + }, + { + "database": "default", + "name": "team_members", + }, + { + "database": "default", + "name": "teams", + }, { "database": "default", "name": "users", @@ -586,7 +611,12 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { "name": "contact_channels" }, + { "name": "email_outboxes" }, { "name": "events" }, + { "name": "session_replays" }, + { "name": "team_members" }, + { "name": "teams" }, { "name": "users" }, ], }, @@ -1068,7 +1098,12 @@ it("shows grants", async ({ expect }) => { "status": 200, "body": { "result": [ + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.session_replays TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_members TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.teams TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.users TO limited_user" }, ], }, From ab778e1bce397f322244f731e0ee1a9438df742b Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 16 Mar 2026 15:26:22 -0700 Subject: [PATCH 05/14] add email outbox cols --- apps/backend/scripts/clickhouse-migrations.ts | 14 +++ apps/backend/src/lib/external-db-sync.ts | 1 + .../src/config/db-sync-mappings.ts | 100 ++++++++++++++---- 3 files changed, 93 insertions(+), 22 deletions(-) diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index f56870c00d..15cb720dd7 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -346,14 +346,19 @@ CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( created_with LowCardinality(String), email_draft_id Nullable(String), email_programmatic_call_template_id Nullable(String), + theme_id Nullable(String), is_high_priority UInt8, rendered_is_transactional Nullable(UInt8), rendered_subject Nullable(String), rendered_notification_category_id Nullable(String), + started_rendering_at Nullable(DateTime64(3, 'UTC')), + finished_rendering_at Nullable(DateTime64(3, 'UTC')), + render_error Nullable(String), scheduled_at DateTime64(3, 'UTC'), created_at DateTime64(3, 'UTC'), started_sending_at Nullable(DateTime64(3, 'UTC')), finished_sending_at Nullable(DateTime64(3, 'UTC')), + server_error Nullable(String), sent_at Nullable(DateTime64(3, 'UTC')), delivered_at Nullable(DateTime64(3, 'UTC')), opened_at Nullable(DateTime64(3, 'UTC')), @@ -361,8 +366,10 @@ CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( unsubscribed_at Nullable(DateTime64(3, 'UTC')), marked_as_spam_at Nullable(DateTime64(3, 'UTC')), bounced_at Nullable(DateTime64(3, 'UTC')), + delivery_delayed_at Nullable(DateTime64(3, 'UTC')), can_have_delivery_info Nullable(UInt8), skipped_reason LowCardinality(Nullable(String)), + skipped_details Nullable(String), send_retries Int32, is_paused UInt8, sync_sequence_id Int64, @@ -387,14 +394,19 @@ SELECT created_with, email_draft_id, email_programmatic_call_template_id, + theme_id, is_high_priority, rendered_is_transactional, rendered_subject, rendered_notification_category_id, + started_rendering_at, + finished_rendering_at, + render_error, scheduled_at, created_at, started_sending_at, finished_sending_at, + server_error, sent_at, delivered_at, opened_at, @@ -402,8 +414,10 @@ SELECT unsubscribed_at, marked_as_spam_at, bounced_at, + delivery_delayed_at, can_have_delivery_info, skipped_reason, + skipped_details, send_retries, is_paused FROM analytics_internal.email_outboxes diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 519930537e..336e1e8f07 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -566,6 +566,7 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record Date: Tue, 17 Mar 2026 11:09:50 -0700 Subject: [PATCH 06/14] team member profile clickhouse table --- apps/backend/scripts/clickhouse-migrations.ts | 20 ++-- .../external-db-sync/sequencer/route.ts | 18 ++++ apps/backend/src/lib/external-db-sync.ts | 2 +- .../endpoints/api/v1/analytics-query.test.ts | 21 ++++ .../api/v1/external-db-sync-basics.test.ts | 2 +- .../api/v1/external-db-sync-utils.ts | 4 +- .../src/config/db-sync-mappings.ts | 102 +++++++++++++++--- 7 files changed, 143 insertions(+), 26 deletions(-) diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index f0b533c034..55e1f86082 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -20,8 +20,8 @@ export async function runClickhouseMigrations() { await client.exec({ query: CONTACT_CHANNELS_VIEW_SQL }); await client.exec({ query: TEAMS_TABLE_BASE_SQL }); await client.exec({ query: TEAMS_VIEW_SQL }); - await client.exec({ query: TEAM_MEMBERS_TABLE_BASE_SQL }); - await client.exec({ query: TEAM_MEMBERS_VIEW_SQL }); + await client.exec({ query: TEAM_MEMBER_PROFILES_TABLE_BASE_SQL }); + await client.exec({ query: TEAM_MEMBER_PROFILES_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -35,7 +35,7 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.users TO limited_user;", "GRANT SELECT ON default.contact_channels TO limited_user;", "GRANT SELECT ON default.teams TO limited_user;", - "GRANT SELECT ON default.team_members TO limited_user;", + "GRANT SELECT ON default.team_member_profiles TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -50,7 +50,7 @@ export async function runClickhouseMigrations() { query: "CREATE ROW POLICY IF NOT EXISTS teams_project_isolation ON default.teams FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS team_members_project_isolation ON default.team_members FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + query: "CREATE ROW POLICY IF NOT EXISTS team_member_profiles_project_isolation ON default.team_member_profiles FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); for (const query of queries) { await client.exec({ query }); @@ -295,14 +295,15 @@ FINAL WHERE sync_is_deleted = 0; `; -const TEAM_MEMBERS_TABLE_BASE_SQL = ` -CREATE TABLE IF NOT EXISTS analytics_internal.team_members ( +const TEAM_MEMBER_PROFILES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_member_profiles ( project_id String, branch_id String, team_id UUID, user_id UUID, display_name Nullable(String), profile_image_url Nullable(String), + user JSON, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -313,8 +314,8 @@ PARTITION BY toYYYYMM(created_at) ORDER BY (project_id, branch_id, team_id, user_id); `; -const TEAM_MEMBERS_VIEW_SQL = ` -CREATE OR REPLACE VIEW default.team_members +const TEAM_MEMBER_PROFILES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_member_profiles SQL SECURITY DEFINER AS SELECT @@ -324,8 +325,9 @@ SELECT user_id, display_name, profile_image_url, + user, created_at -FROM analytics_internal.team_members +FROM analytics_internal.team_member_profiles FINAL WHERE sync_is_deleted = 0; `; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index 78bbb99ffd..c5b190f94d 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -1,5 +1,6 @@ import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; import { enqueueExternalDbSyncBatch } from "@/lib/external-db-sync-queue"; +import { Prisma } from "@/generated/prisma/client"; import { globalPrismaClient } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { traceSpan } from "@/utils/telemetry"; @@ -78,6 +79,23 @@ async function backfillSequenceIds(batchSize: number): Promise { if (projectUserTenants.length > 0) { await enqueueExternalDbSyncBatch(projectUserTenants.map(t => t.tenancyId)); didUpdate = true; + + // Cascade: when a user changes, mark their TeamMember rows for re-sync + // so the embedded user JSON in team_member_profiles stays fresh + await globalPrismaClient.$executeRaw` + UPDATE "TeamMember" + SET "shouldUpdateSequenceId" = TRUE + FROM ( + SELECT DISTINCT "tenancyId", "projectUserId" + FROM "ProjectUser" + WHERE "tenancyId" IN (${Prisma.join(projectUserTenants.map(t => t.tenancyId))}) + AND "shouldUpdateSequenceId" = FALSE + AND "sequenceId" IS NOT NULL + ) AS changed_users + WHERE "TeamMember"."tenancyId" = changed_users."tenancyId" + AND "TeamMember"."projectUserId" = changed_users."projectUserId" + AND "TeamMember"."shouldUpdateSequenceId" = FALSE + `; } const contactChannelTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index bd9b0b1cab..966f822aaf 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -552,7 +552,7 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON SQLite FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON URL FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.teams TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.users TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.aggregate_function_combinators TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.collations TO limited_user" }, @@ -561,10 +564,22 @@ it("can see only some tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { + "database": "default", + "name": "contact_channels", + }, { "database": "default", "name": "events", }, + { + "database": "default", + "name": "team_member_profiles", + }, + { + "database": "default", + "name": "teams", + }, { "database": "default", "name": "users", @@ -586,7 +601,10 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { "name": "contact_channels" }, { "name": "events" }, + { "name": "team_member_profiles" }, + { "name": "teams" }, { "name": "users" }, ], }, @@ -1068,7 +1086,10 @@ it("shows grants", async ({ expect }) => { "status": 200, "body": { "result": [ + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.teams TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.users TO limited_user" }, ], }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 4c7bceee8d..b912b64215 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -711,7 +711,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { await waitForSyncedTeamMember(client, teamId, user.userId); - const res1 = await client.query(`SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, user.userId]); + const res1 = await client.query(`SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, user.userId]); expect(res1.rows.length).toBe(1); // Remove member diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index ae830bbe67..59b855373f 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -287,14 +287,14 @@ export async function waitForSyncedTeamDeletion(client: Client, teamId: string) } export async function waitForSyncedTeamMember(client: Client, teamId: string, userId: string) { - await waitForExternalDbRow(client, `SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + await waitForExternalDbRow(client, `SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { shouldExist: true, description: `team member (team=${teamId}, user=${userId}) to appear in external DB`, }); } export async function waitForSyncedTeamMemberDeletion(client: Client, teamId: string, userId: string) { - await waitForExternalDbRow(client, `SELECT * FROM "team_members" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { + await waitForExternalDbRow(client, `SELECT * FROM "team_member_profiles" WHERE "team_id" = $1 AND "user_id" = $2`, [teamId, userId], { shouldExist: false, description: `team member (team=${teamId}, user=${userId}) to be removed from external DB`, }); diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 92d6399970..f00b4ed0ae 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -663,21 +663,22 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, - "team_members": { - sourceTables: { "TeamMember": "TeamMember" }, - targetTable: "team_members", + "team_member_profiles": { + sourceTables: { "TeamMember": "TeamMember", "ProjectUser": "ProjectUser" }, + targetTable: "team_member_profiles", targetTableSchemas: { postgres: ` - CREATE TABLE IF NOT EXISTS "team_members" ( + CREATE TABLE IF NOT EXISTS "team_member_profiles" ( "team_id" uuid NOT NULL, "user_id" uuid NOT NULL, "display_name" text, "profile_image_url" text, + "user" jsonb NOT NULL DEFAULT '{}'::jsonb, "created_at" timestamp without time zone NOT NULL, PRIMARY KEY ("team_id", "user_id") ); - REVOKE ALL ON "team_members" FROM PUBLIC; - GRANT SELECT ON "team_members" TO PUBLIC; + REVOKE ALL ON "team_member_profiles" FROM PUBLIC; + GRANT SELECT ON "team_member_profiles" TO PUBLIC; CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( "mapping_name" text PRIMARY KEY NOT NULL, @@ -686,13 +687,14 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ); `.trim(), clickhouse: ` - CREATE TABLE IF NOT EXISTS analytics_internal.team_members ( + CREATE TABLE IF NOT EXISTS analytics_internal.team_member_profiles ( project_id String, branch_id String, team_id UUID, user_id UUID, display_name Nullable(String), profile_image_url Nullable(String), + user JSON, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -714,12 +716,46 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "TeamMember"."projectUserId" AS "user_id", "TeamMember"."displayName" AS "display_name", "TeamMember"."profileImageUrl" AS "profile_image_url", + jsonb_build_object( + 'id', "ProjectUser"."projectUserId", + 'display_name', "ProjectUser"."displayName", + 'primary_email', ( + SELECT "ContactChannel"."value" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + 'primary_email_verified', COALESCE( + ( + SELECT "ContactChannel"."isVerified" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + false + ), + 'profile_image_url', "ProjectUser"."profileImageUrl", + 'signed_up_at_millis', EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000, + 'client_metadata', COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb), + 'client_read_only_metadata', COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb), + 'server_metadata', COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb), + 'is_anonymous', "ProjectUser"."isAnonymous", + 'last_active_at_millis', CASE WHEN "ProjectUser"."lastActiveAt" IS NOT NULL THEN EXTRACT(EPOCH FROM "ProjectUser"."lastActiveAt") * 1000 ELSE NULL END + ) AS "user", "TeamMember"."createdAt" AS "created_at", "TeamMember"."sequenceId" AS "sync_sequence_id", "TeamMember"."tenancyId" AS "tenancyId", false AS "sync_is_deleted" FROM "TeamMember" JOIN "Tenancy" ON "Tenancy"."id" = "TeamMember"."tenancyId" + JOIN "ProjectUser" ON "ProjectUser"."projectUserId" = "TeamMember"."projectUserId" + AND "ProjectUser"."tenancyId" = "TeamMember"."tenancyId" WHERE "TeamMember"."tenancyId" = $1::uuid UNION ALL @@ -731,6 +767,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", NULL::text AS "display_name", NULL::text AS "profile_image_url", + '{}'::jsonb AS "user", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sync_sequence_id", "DeletedRow"."tenancyId" AS "tenancyId", @@ -755,11 +792,45 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "TeamMember"."projectUserId" AS "user_id", "TeamMember"."displayName" AS "display_name", "TeamMember"."profileImageUrl" AS "profile_image_url", + jsonb_build_object( + 'id', "ProjectUser"."projectUserId", + 'display_name', "ProjectUser"."displayName", + 'primary_email', ( + SELECT "ContactChannel"."value" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + 'primary_email_verified', COALESCE( + ( + SELECT "ContactChannel"."isVerified" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + false + ), + 'profile_image_url', "ProjectUser"."profileImageUrl", + 'signed_up_at_millis', EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000, + 'client_metadata', COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb), + 'client_read_only_metadata', COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb), + 'server_metadata', COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb), + 'is_anonymous', "ProjectUser"."isAnonymous", + 'last_active_at_millis', CASE WHEN "ProjectUser"."lastActiveAt" IS NOT NULL THEN EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000 ELSE NULL END + ) AS "user", "TeamMember"."createdAt" AS "created_at", "TeamMember"."sequenceId" AS "sequence_id", "TeamMember"."tenancyId", false AS "is_deleted" FROM "TeamMember" + JOIN "ProjectUser" ON "ProjectUser"."projectUserId" = "TeamMember"."projectUserId" + AND "ProjectUser"."tenancyId" = "TeamMember"."tenancyId" WHERE "TeamMember"."tenancyId" = $1::uuid UNION ALL @@ -769,6 +840,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", NULL::text AS "display_name", NULL::text AS "profile_image_url", + '{}'::jsonb AS "user", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sequence_id", "DeletedRow"."tenancyId", @@ -791,23 +863,25 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { $2::uuid AS "user_id", $3::text AS "display_name", $4::text AS "profile_image_url", - $5::timestamp without time zone AS "created_at", - $6::bigint AS "sequence_id", - $7::boolean AS "is_deleted", - $8::text AS "mapping_name" + $5::jsonb AS "user", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" ), deleted AS ( - DELETE FROM "team_members" tm + DELETE FROM "team_member_profiles" tm USING params p WHERE p."is_deleted" = true AND tm."team_id" = p."team_id" AND tm."user_id" = p."user_id" RETURNING 1 ), upserted AS ( - INSERT INTO "team_members" ( + INSERT INTO "team_member_profiles" ( "team_id", "user_id", "display_name", "profile_image_url", + "user", "created_at" ) SELECT @@ -815,12 +889,14 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { p."user_id", p."display_name", p."profile_image_url", + p."user", p."created_at" FROM params p WHERE p."is_deleted" = false ON CONFLICT ("team_id", "user_id") DO UPDATE SET "display_name" = EXCLUDED."display_name", "profile_image_url" = EXCLUDED."profile_image_url", + "user" = EXCLUDED."user", "created_at" = EXCLUDED."created_at" RETURNING 1 ) From 60e9b8685a2bd87296eb059d309f13b96d8c5421 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Tue, 17 Mar 2026 13:41:53 -0700 Subject: [PATCH 07/14] team member and team invite clickhouse tables --- .../migration.sql | 22 ++ apps/backend/prisma/schema.prisma | 9 + apps/backend/scripts/clickhouse-migrations.ts | 82 +++++ .../external-db-sync/sequencer/route.ts | 81 +++- .../app/api/latest/team-memberships/crud.tsx | 8 +- .../backend/src/app/api/latest/teams/crud.tsx | 12 +- .../backend/src/app/api/latest/users/crud.tsx | 7 +- apps/backend/src/lib/external-db-sync.ts | 274 +++++++++++++- apps/backend/src/lib/permissions.tsx | 25 +- .../verification-code-handler.tsx | 18 +- .../endpoints/api/v1/analytics-query.test.ts | 14 + .../api/v1/external-db-sync-basics.test.ts | 257 +++++++++++++ .../api/v1/external-db-sync-utils.ts | 28 ++ .../src/config/db-sync-mappings.ts | 346 ++++++++++++++++++ 14 files changed, 1166 insertions(+), 17 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql diff --git a/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql new file mode 100644 index 0000000000..b13d10e6c3 --- /dev/null +++ b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql @@ -0,0 +1,22 @@ +-- AlterTable +ALTER TABLE "TeamMemberDirectPermission" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "TeamMemberDirectPermission_sequenceId_key" ON "TeamMemberDirectPermission"("sequenceId"); + +-- CreateIndex +CREATE INDEX "TeamMemberDirectPermission_shouldUpdateSequenceId_idx" ON "TeamMemberDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- CreateIndex +CREATE INDEX "TeamMemberDirectPermission_tenancyId_sequenceId_idx" ON "TeamMemberDirectPermission"("tenancyId", "sequenceId"); + +-- AlterTable +ALTER TABLE "VerificationCode" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "VerificationCode_sequenceId_key" ON "VerificationCode"("sequenceId"); + +-- CreateIndex +CREATE INDEX "VerificationCode_shouldUpdateSequenceId_type_idx" ON "VerificationCode"("shouldUpdateSequenceId", "type"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index d6077f3039..f842fe8ae6 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -251,7 +251,12 @@ model TeamMemberDirectPermission { teamMember TeamMember @relation(fields: [tenancyId, projectUserId, teamId], references: [tenancyId, projectUserId, teamId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@unique([tenancyId, projectUserId, teamId, permissionId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "TeamMemberDirectPermission_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "TeamMemberDirectPermission_tenancyId_sequenceId_idx") } model ProjectUser { @@ -647,9 +652,13 @@ model VerificationCode { data Json attemptCount Int @default(0) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([projectId, branchId, id]) @@unique([projectId, branchId, code]) @@index([data(ops: JsonbPathOps)], type: Gin) + @@index([shouldUpdateSequenceId, type], name: "VerificationCode_shouldUpdateSequenceId_type_idx") } enum VerificationCodeType { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index 55e1f86082..f95cf0bd1f 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -22,6 +22,10 @@ export async function runClickhouseMigrations() { await client.exec({ query: TEAMS_VIEW_SQL }); await client.exec({ query: TEAM_MEMBER_PROFILES_TABLE_BASE_SQL }); await client.exec({ query: TEAM_MEMBER_PROFILES_VIEW_SQL }); + await client.exec({ query: TEAM_PERMISSIONS_TABLE_BASE_SQL }); + await client.exec({ query: TEAM_PERMISSIONS_VIEW_SQL }); + await client.exec({ query: TEAM_INVITATIONS_TABLE_BASE_SQL }); + await client.exec({ query: TEAM_INVITATIONS_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -36,6 +40,8 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.contact_channels TO limited_user;", "GRANT SELECT ON default.teams TO limited_user;", "GRANT SELECT ON default.team_member_profiles TO limited_user;", + "GRANT SELECT ON default.team_permissions TO limited_user;", + "GRANT SELECT ON default.team_invitations TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -52,6 +58,12 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS team_member_profiles_project_isolation ON default.team_member_profiles FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS team_permissions_project_isolation ON default.team_permissions FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS team_invitations_project_isolation ON default.team_invitations FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -332,6 +344,76 @@ FINAL WHERE sync_is_deleted = 0; `; +const TEAM_PERMISSIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + permission_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, team_id, user_id, permission_id); +`; + +const TEAM_PERMISSIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_permissions +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + team_id, + user_id, + permission_id, + created_at +FROM analytics_internal.team_permissions +FINAL +WHERE sync_is_deleted = 0; +`; + +const TEAM_INVITATIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.team_invitations ( + project_id String, + branch_id String, + id UUID, + team_id UUID, + team_display_name String, + recipient_email String, + expires_at_millis Int64, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const TEAM_INVITATIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.team_invitations +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + team_id, + team_display_name, + recipient_email, + expires_at_millis, + created_at +FROM analytics_internal.team_invitations +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index c5b190f94d..1ce3b58971 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -153,6 +153,25 @@ async function backfillSequenceIds(batchSize: number): Promise { if (teamTenants.length > 0) { await enqueueExternalDbSyncBatch(teamTenants.map(t => t.tenancyId)); didUpdate = true; + + // Cascade: when a team changes, mark related TEAM_INVITATION verification codes for re-sync + // so the team_display_name in team_invitations stays fresh + await globalPrismaClient.$executeRaw` + UPDATE "VerificationCode" + SET "shouldUpdateSequenceId" = TRUE + FROM ( + SELECT DISTINCT "Tenancy"."projectId", "Tenancy"."branchId" + FROM "Team" + JOIN "Tenancy" ON "Tenancy"."id" = "Team"."tenancyId" + WHERE "Team"."tenancyId" IN (${Prisma.join(teamTenants.map(t => t.tenancyId))}) + AND "Team"."shouldUpdateSequenceId" = FALSE + AND "Team"."sequenceId" IS NOT NULL + ) AS changed_teams + WHERE "VerificationCode"."projectId" = changed_teams."projectId" + AND "VerificationCode"."branchId" = changed_teams."branchId" + AND "VerificationCode"."type" = 'TEAM_INVITATION' + AND "VerificationCode"."shouldUpdateSequenceId" = FALSE + `; } const teamMemberTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` @@ -184,6 +203,66 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const teamPermissionTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id" + FROM "TeamMemberDirectPermission" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "TeamMemberDirectPermission" tp + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE tp."id" = r."id" + RETURNING tp."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.team-permission-tenants", teamPermissionTenants.length); + + if (teamPermissionTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamPermissionTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const teamInvitationTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "projectId", "branchId", "id" + FROM "VerificationCode" + WHERE "shouldUpdateSequenceId" = TRUE + AND "type" = 'TEAM_INVITATION' + ORDER BY "projectId", "branchId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "VerificationCode" vc + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE vc."projectId" = r."projectId" + AND vc."branchId" = r."branchId" + AND vc."id" = r."id" + RETURNING vc."projectId", vc."branchId" + ) + SELECT DISTINCT "Tenancy"."id" AS "tenancyId" + FROM updated_rows + JOIN "Tenancy" ON "Tenancy"."projectId" = updated_rows."projectId" + AND "Tenancy"."branchId" = updated_rows."branchId" + `; + + span.setAttribute("stack.external-db-sync.team-invitation-tenants", teamInvitationTenants.length); + + if (teamInvitationTenants.length > 0) { + await enqueueExternalDbSyncBatch(teamInvitationTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -213,7 +292,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/team-memberships/crud.tsx b/apps/backend/src/app/api/latest/team-memberships/crud.tsx index ae5dbfd43a..ae3e6f117b 100644 --- a/apps/backend/src/app/api/latest/team-memberships/crud.tsx +++ b/apps/backend/src/app/api/latest/team-memberships/crud.tsx @@ -1,4 +1,4 @@ -import { recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamPermissionDeletionsForTeamMember, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultTeamPermissions } from "@/lib/permissions"; import { ensureTeamExists, ensureTeamMembershipDoesNotExist, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -139,6 +139,12 @@ export const teamMembershipsCrudHandlers = createLazyProxy(() => createCrudHandl userId: params.user_id, }); + await recordExternalDbSyncTeamPermissionDeletionsForTeamMember(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + teamId: params.team_id, + }); + await recordExternalDbSyncDeletion(tx, { tableName: "TeamMember", tenancyId: auth.tenancy.id, diff --git a/apps/backend/src/app/api/latest/teams/crud.tsx b/apps/backend/src/app/api/latest/teams/crud.tsx index aab670506e..6ae06487fd 100644 --- a/apps/backend/src/app/api/latest/teams/crud.tsx +++ b/apps/backend/src/app/api/latest/teams/crud.tsx @@ -1,4 +1,4 @@ -import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForTeam, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncDeletion, recordExternalDbSyncTeamInvitationDeletionsForTeam, recordExternalDbSyncTeamMemberDeletionsForTeam, recordExternalDbSyncTeamPermissionDeletionsForTeam, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureTeamExists, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { sendTeamCreatedWebhook, sendTeamDeletedWebhook, sendTeamUpdatedWebhook } from "@/lib/webhooks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -195,6 +195,16 @@ export const teamsCrudHandlers = createLazyProxy(() => createCrudHandlers(teamsC } await ensureTeamExists(tx, { tenancyId: auth.tenancy.id, teamId: params.team_id }); + await recordExternalDbSyncTeamPermissionDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + + await recordExternalDbSyncTeamInvitationDeletionsForTeam(tx, { + tenancyId: auth.tenancy.id, + teamId: params.team_id, + }); + await recordExternalDbSyncTeamMemberDeletionsForTeam(tx, { tenancyId: auth.tenancy.id, teamId: params.team_id, diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index fc7e56006c..61c694e573 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,7 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; -import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -1207,6 +1207,11 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }); + await recordExternalDbSyncTeamPermissionDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 966f822aaf..e34770ebee 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -52,6 +52,18 @@ type ExternalDbSyncTarget = tenancyId: string, projectUserId: string, teamId: string, + } + | { + tableName: "TeamMemberDirectPermission", + tenancyId: string, + permissionDbId: string, + } + | { + tableName: "VerificationCode_TEAM_INVITATION", + tenancyId: string, + verificationCodeProjectId: string, + verificationCodeBranchId: string, + verificationCodeId: string, }; type ExternalDbType = NonNullable["type"]>; @@ -203,8 +215,7 @@ export async function recordExternalDbSyncDeletion( return; } - { - const _teamMemberTarget: { tableName: "TeamMember" } = target; + if (target.tableName === "TeamMember") { assertUuid(target.projectUserId, "projectUserId"); assertUuid(target.teamId, "teamId"); const insertedCount = await tx.$executeRaw(Prisma.sql` @@ -239,6 +250,85 @@ export async function recordExternalDbSyncDeletion( } return; } + + if (target.tableName === "TeamMemberDirectPermission") { + assertUuid(target.permissionDbId, "permissionDbId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "id" = ${target.permissionDbId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for TeamMemberDirectPermission, got ${insertedCount}.` + ); + } + return; + } + + { + const _verificationCodeTarget: { tableName: "VerificationCode_TEAM_INVITATION" } = target; + assertNonEmptyString(target.verificationCodeProjectId, "verificationCodeProjectId"); + assertNonEmptyString(target.verificationCodeBranchId, "verificationCodeBranchId"); + assertUuid(target.verificationCodeId, "verificationCodeId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "Tenancy"."id", + 'VerificationCode_TEAM_INVITATION', + jsonb_build_object('id', "VerificationCode"."id"), + to_jsonb("VerificationCode".*), + NOW(), + TRUE + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + WHERE "VerificationCode"."projectId" = ${target.verificationCodeProjectId} + AND "VerificationCode"."branchId" = ${target.verificationCodeBranchId} + AND "VerificationCode"."id" = ${target.verificationCodeId}::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + FOR UPDATE OF "VerificationCode" + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for VerificationCode_TEAM_INVITATION, got ${insertedCount}.` + ); + } + return; + } } export async function recordExternalDbSyncContactChannelDeletionsForUser( @@ -318,6 +408,167 @@ export async function recordExternalDbSyncTeamMemberDeletionsForTeam( `); } +export async function recordExternalDbSyncTeamPermissionDeletionsForTeamMember( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamPermissionDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "teamId" = ${options.teamId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamPermissionDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'TeamMemberDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'teamId', "teamId", + 'permissionId', "permissionId" + ), + to_jsonb("TeamMemberDirectPermission".*), + NOW(), + TRUE + FROM "TeamMemberDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncTeamInvitationDeletionsForTeam( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + teamId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.teamId, "teamId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "Tenancy"."id", + 'VerificationCode_TEAM_INVITATION', + jsonb_build_object('id', "VerificationCode"."id"), + to_jsonb("VerificationCode".*), + NOW(), + TRUE + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + WHERE "Tenancy"."id" = ${options.tenancyId}::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + AND "VerificationCode"."data"->>'team_id' = ${options.teamId} + FOR UPDATE OF "VerificationCode" + `); +} + export async function recordExternalDbSyncTeamMemberDeletionsForUser( tx: ExternalDbSyncClient, options: { @@ -481,7 +732,7 @@ async function pushRowsToExternalDb( } } -function getInternalDbFetchQuery(mapping: DbSyncMapping, dbType: ExternalDbType) { +function getInternalDbFetchQuery(mapping: DbSyncMapping) { return mapping.internalDbFetchQuery; } @@ -503,7 +754,7 @@ function parseSequenceId(value: unknown, mappingId: string): number | null { if (value == null) { return null; } - const seqNum = typeof value === "bigint" ? Number(value) : Number(value); + const seqNum = Number(value); if (!Number.isFinite(seqNum)) { throw new StackAssertionError( `Invalid sequence_id for mapping ${mappingId}: ${JSON.stringify(value)}` @@ -529,8 +780,8 @@ async function ensureClickhouseSchema( } // Map of target table name -> column normalizers for ClickHouse -// 'json' columns get JSON.stringify, 'boolean' columns get normalizeClickhouseBoolean -const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { +// 'json' columns get JSON.stringify, 'boolean' columns get normalizeClickhouseBoolean, 'bigint' columns get Number() +const CLICKHOUSE_COLUMN_NORMALIZERS: Record> = { users: { client_metadata: 'json', client_read_only_metadata: 'json', @@ -555,6 +806,13 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record createCodeObjectFromPrismaCode(code)); }, - async revokeCode(options) { - const { project, branchId } = parseProjectBranchCombo(options); + async revokeCode(revokeOptions) { + const { project, branchId } = parseProjectBranchCombo(revokeOptions); const tenancy = await getSoleTenancyFromProjectBranch(project.id, branchId); + // Record deletion for external DB sync if this is a TEAM_INVITATION code + if (options.type === 'TEAM_INVITATION') { + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "VerificationCode_TEAM_INVITATION", + tenancyId: tenancy.id, + verificationCodeProjectId: project.id, + verificationCodeBranchId: branchId, + verificationCodeId: revokeOptions.id, + }); + } + await globalPrismaClient.verificationCode.delete({ where: { projectId_branchId_id: { projectId: project.id, branchId, - id: options.id, + id: revokeOptions.id, }, }, }); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index f30ce66b6a..dbd757e0da 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -524,7 +524,9 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_permissions TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.teams TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.users TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SELECT ON system.aggregate_function_combinators TO limited_user" }, @@ -572,10 +574,18 @@ it("can see only some tables", async ({ expect }) => { "database": "default", "name": "events", }, + { + "database": "default", + "name": "team_invitations", + }, { "database": "default", "name": "team_member_profiles", }, + { + "database": "default", + "name": "team_permissions", + }, { "database": "default", "name": "teams", @@ -603,7 +613,9 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "result": [ { "name": "contact_channels" }, { "name": "events" }, + { "name": "team_invitations" }, { "name": "team_member_profiles" }, + { "name": "team_permissions" }, { "name": "teams" }, { "name": "users" }, ], @@ -1088,7 +1100,9 @@ it("shows grants", async ({ expect }) => { "result": [ { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_permissions TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.teams TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.users TO limited_user" }, ], diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index b912b64215..dddc88cc00 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -15,8 +15,12 @@ import { waitForSyncedDeletion, waitForSyncedTeam, waitForSyncedTeamDeletion, + waitForSyncedTeamInvitation, + waitForSyncedTeamInvitationDeletion, waitForSyncedTeamMember, waitForSyncedTeamMemberDeletion, + waitForSyncedTeamPermission, + waitForSyncedTeamPermissionDeletion, waitForTable } from './external-db-sync-utils'; @@ -837,6 +841,259 @@ describe.sequential('External DB Sync - Basic Tests', () => { await waitForSyncedTeamMemberDeletion(client, teamId, user.userId); }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a team, adds a member, grants a permission, verifies in external DB, + * revokes the permission, and verifies removal. + */ + test('TeamPermission CRUD sync (Postgres)', async () => { + const dbName = 'team_permission_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'tp-crud@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TP CRUD Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Add user as team member + const addMemberResponse = await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(addMemberResponse.status).toBe(201); + + // Grant a permission + const grantResponse = await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(grantResponse.status).toBe(201); + + await waitForSyncedTeamPermission(client, teamId, user.userId, '$read_members'); + + const res1 = await client.query(`SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, user.userId, '$read_members']); + expect(res1.rows.length).toBe(1); + + // Revoke the permission + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamPermissionDeletion(client, teamId, user.userId, '$read_members'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team + member + permission, queries ClickHouse analytics API to verify. + */ + test('TeamPermission sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const user = await User.create({ primary_email: 'tp-ch@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'TP CH Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT team_id, user_id, permission_id FROM team_permissions WHERE permission_id = {perm:String}", + params: { perm: '$read_members' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].permission_id).toBe('$read_members'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends a team invitation, verifies in external DB, revokes it, verifies removal. + */ + test('TeamInvitation sync (Postgres)', async () => { + const dbName = 'team_invitation_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { display_name: 'Invitation Test Project' }); + + const client = dbManager.getClient(dbName); + + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Invitation Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + // Send a team invitation + const inviteResponse = await niceBackendFetch('/api/v1/team-invitations/send-code', { + accessType: 'admin', + method: 'POST', + body: { team_id: teamId, email: 'invited@example.com', callback_url: 'http://localhost:12345/callback' }, + }); + expect(inviteResponse.status).toBe(200); + + await waitForSyncedTeamInvitation(client, 'invited@example.com'); + + const res1 = await client.query(`SELECT * FROM "team_invitations" WHERE "recipient_email" = $1`, ['invited@example.com']); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].team_display_name).toBe('Invitation Team'); + const invitationId = res1.rows[0].id; + + // Revoke the invitation + await niceBackendFetch(`/api/v1/team-invitations/${invitationId}?team_id=${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamInvitationDeletion(client, invitationId); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Sends a team invitation, queries ClickHouse analytics API to verify. + */ + test('TeamInvitation sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'CH Invitation Team' }, + }); + expect(createTeamResponse.status).toBe(201); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch('/api/v1/team-invitations/send-code', { + accessType: 'admin', + method: 'POST', + body: { team_id: teamId, email: 'ch-invited@example.com', callback_url: 'http://localhost:12345/callback' }, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT recipient_email, team_display_name FROM team_invitations WHERE recipient_email = {email:String}", + params: { email: 'ch-invited@example.com' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].recipient_email).toBe('ch-invited@example.com'); + expect(response!.body.result[0].team_display_name).toBe('CH Invitation Team'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a team with a member and permission, deletes the team, + * verifies team, member, and permissions are all gone. + */ + test('Cascade: Team delete removes permissions and invitations from external DB', async () => { + const dbName = 'cascade_team_perm_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'cascade-perm@example.com' }); + const createTeamResponse = await niceBackendFetch('/api/v1/teams', { + accessType: 'admin', + method: 'POST', + body: { display_name: 'Cascade Perm Team' }, + }); + const teamId = createTeamResponse.body.id; + + await niceBackendFetch(`/api/v1/team-memberships/${teamId}/${user.userId}`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await niceBackendFetch(`/api/v1/team-permissions/${teamId}/${user.userId}/$read_members`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await waitForSyncedTeamPermission(client, teamId, user.userId, '$read_members'); + await waitForSyncedTeam(client, 'Cascade Perm Team'); + + // Delete the team — should cascade-delete permissions too + await niceBackendFetch(`/api/v1/teams/${teamId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedTeamDeletion(client, teamId); + await waitForSyncedTeamPermissionDeletion(client, teamId, user.userId, '$read_members'); + }, TEST_TIMEOUT); + /** * What it does: * - Reads the external DB sync fusebox settings. diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 59b855373f..28cd9ad8b5 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -314,6 +314,34 @@ export async function waitForSyncedContactChannelDeletion(client: Client, value: }); } +export async function waitForSyncedTeamPermission(client: Client, teamId: string, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, userId, permissionId], { + shouldExist: true, + description: `team permission (team=${teamId}, user=${userId}, perm=${permissionId}) to appear in external DB`, + }); +} + +export async function waitForSyncedTeamPermissionDeletion(client: Client, teamId: string, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_permissions" WHERE "team_id" = $1 AND "user_id" = $2 AND "permission_id" = $3`, [teamId, userId, permissionId], { + shouldExist: false, + description: `team permission (team=${teamId}, user=${userId}, perm=${permissionId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedTeamInvitation(client: Client, recipientEmail: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_invitations" WHERE "recipient_email" = $1`, [recipientEmail], { + shouldExist: true, + description: `team invitation for "${recipientEmail}" to appear in external DB`, + }); +} + +export async function waitForSyncedTeamInvitationDeletion(client: Client, invitationId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "team_invitations" WHERE "id" = $1`, [invitationId], { + shouldExist: false, + description: `team invitation ${invitationId} to be removed from external DB`, + }); +} + /** * Helper to create a project and update its config with external DB settings. * Tracks the project for cleanup later. diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index f00b4ed0ae..bcf853dd4a 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -908,4 +908,350 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "team_permissions": { + sourceTables: { "TeamMemberDirectPermission": "TeamMemberDirectPermission" }, + targetTable: "team_permissions", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_permissions" ( + "team_id" uuid NOT NULL, + "user_id" uuid NOT NULL, + "permission_id" text NOT NULL, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("team_id", "user_id", "permission_id") + ); + REVOKE ALL ON "team_permissions" FROM PUBLIC; + GRANT SELECT ON "team_permissions" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( + project_id String, + branch_id String, + team_id UUID, + user_id UUID, + permission_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, team_id, user_id, permission_id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "TeamMemberDirectPermission"."teamId" AS "team_id", + "TeamMemberDirectPermission"."projectUserId" AS "user_id", + "TeamMemberDirectPermission"."permissionId" AS "permission_id", + "TeamMemberDirectPermission"."createdAt" AS "created_at", + "TeamMemberDirectPermission"."sequenceId" AS "sync_sequence_id", + "TeamMemberDirectPermission"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "TeamMemberDirectPermission" + JOIN "Tenancy" ON "Tenancy"."id" = "TeamMemberDirectPermission"."tenancyId" + WHERE "TeamMemberDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMemberDirectPermission' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "TeamMemberDirectPermission"."teamId" AS "team_id", + "TeamMemberDirectPermission"."projectUserId" AS "user_id", + "TeamMemberDirectPermission"."permissionId" AS "permission_id", + "TeamMemberDirectPermission"."createdAt" AS "created_at", + "TeamMemberDirectPermission"."sequenceId" AS "sequence_id", + "TeamMemberDirectPermission"."tenancyId", + false AS "is_deleted" + FROM "TeamMemberDirectPermission" + WHERE "TeamMemberDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'TeamMemberDirectPermission' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "team_id", + $2::uuid AS "user_id", + $3::text AS "permission_id", + $4::timestamp without time zone AS "created_at", + $5::bigint AS "sequence_id", + $6::boolean AS "is_deleted", + $7::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_permissions" tp + USING params p + WHERE p."is_deleted" = true AND tp."team_id" = p."team_id" AND tp."user_id" = p."user_id" AND tp."permission_id" = p."permission_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_permissions" ( + "team_id", + "user_id", + "permission_id", + "created_at" + ) + SELECT + p."team_id", + p."user_id", + p."permission_id", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("team_id", "user_id", "permission_id") DO UPDATE SET + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "team_invitations": { + sourceTables: { "VerificationCode": "VerificationCode" }, + targetTable: "team_invitations", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "team_invitations" ( + "id" uuid PRIMARY KEY NOT NULL, + "team_id" uuid NOT NULL, + "team_display_name" text NOT NULL, + "recipient_email" text NOT NULL, + "expires_at_millis" bigint NOT NULL, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "team_invitations" FROM PUBLIC; + GRANT SELECT ON "team_invitations" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.team_invitations ( + project_id String, + branch_id String, + id UUID, + team_id UUID, + team_display_name String, + recipient_email String, + expires_at_millis Int64, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "VerificationCode"."id"::uuid AS "id", + ("VerificationCode"."data"->>'team_id')::uuid AS "team_id", + "Team"."displayName" AS "team_display_name", + "VerificationCode"."method"->>'email' AS "recipient_email", + FLOOR(EXTRACT(EPOCH FROM "VerificationCode"."expiresAt") * 1000)::bigint AS "expires_at_millis", + "VerificationCode"."createdAt" AS "created_at", + "VerificationCode"."sequenceId" AS "sync_sequence_id", + "Tenancy"."id" AS "tenancyId", + false AS "sync_is_deleted" + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + LEFT JOIN "Team" ON "Team"."teamId" = ("VerificationCode"."data"->>'team_id')::uuid + AND "Team"."tenancyId" = "Tenancy"."id" + WHERE "Tenancy"."id" = $1::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + '00000000-0000-0000-0000-000000000000'::uuid AS "team_id", + ''::text AS "team_display_name", + ''::text AS "recipient_email", + 0::bigint AS "expires_at_millis", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'VerificationCode_TEAM_INVITATION' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "VerificationCode"."id"::uuid AS "id", + ("VerificationCode"."data"->>'team_id')::uuid AS "team_id", + "Team"."displayName" AS "team_display_name", + "VerificationCode"."method"->>'email' AS "recipient_email", + FLOOR(EXTRACT(EPOCH FROM "VerificationCode"."expiresAt") * 1000)::bigint AS "expires_at_millis", + "VerificationCode"."createdAt" AS "created_at", + "VerificationCode"."sequenceId" AS "sequence_id", + "Tenancy"."id" AS "tenancyId", + false AS "is_deleted" + FROM "VerificationCode" + JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" + AND "Tenancy"."branchId" = "VerificationCode"."branchId" + LEFT JOIN "Team" ON "Team"."teamId" = ("VerificationCode"."data"->>'team_id')::uuid + AND "Team"."tenancyId" = "Tenancy"."id" + WHERE "Tenancy"."id" = $1::uuid + AND "VerificationCode"."type" = 'TEAM_INVITATION' + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + '00000000-0000-0000-0000-000000000000'::uuid AS "team_id", + ''::text AS "team_display_name", + ''::text AS "recipient_email", + 0::bigint AS "expires_at_millis", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'VerificationCode_TEAM_INVITATION' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "team_id", + $3::text AS "team_display_name", + $4::text AS "recipient_email", + $5::bigint AS "expires_at_millis", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "team_invitations" ti + USING params p + WHERE p."is_deleted" = true AND ti."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "team_invitations" ( + "id", + "team_id", + "team_display_name", + "recipient_email", + "expires_at_millis", + "created_at" + ) + SELECT + p."id", + p."team_id", + p."team_display_name", + p."recipient_email", + p."expires_at_millis", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "team_id" = EXCLUDED."team_id", + "team_display_name" = EXCLUDED."team_display_name", + "recipient_email" = EXCLUDED."recipient_email", + "expires_at_millis" = EXCLUDED."expires_at_millis", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From 3896f629cc740f7e04bf2045567d8416bf97f5b9 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Tue, 17 Mar 2026 14:03:21 -0700 Subject: [PATCH 08/14] analytics query update test --- .../tests/backend/endpoints/api/v1/analytics-query.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index dbd757e0da..e954ae3878 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -523,6 +523,7 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON URL FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, @@ -570,6 +571,10 @@ it("can see only some tables", async ({ expect }) => { "database": "default", "name": "contact_channels", }, + { + "database": "default", + "name": "email_outboxes", + }, { "database": "default", "name": "events", @@ -612,6 +617,7 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "body": { "result": [ { "name": "contact_channels" }, + { "name": "email_outboxes" }, { "name": "events" }, { "name": "team_invitations" }, { "name": "team_member_profiles" }, @@ -1099,6 +1105,7 @@ it("shows grants", async ({ expect }) => { "body": { "result": [ { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, From 57176abafa9a049be5e8daa847f77470f3dc00b3 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Tue, 17 Mar 2026 15:12:57 -0700 Subject: [PATCH 09/14] update session replays table --- apps/backend/scripts/clickhouse-migrations.ts | 3 +- apps/backend/src/lib/external-db-sync.ts | 1 + .../endpoints/api/v1/analytics-query.test.ts | 4 +- .../api/v1/external-db-sync-basics.test.ts | 62 ++++++++++++++++--- .../api/v1/external-db-sync-utils.ts | 3 +- .../src/config/db-sync-mappings.ts | 32 +++++++--- 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index 21fcd47b75..fe88250cf3 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -525,6 +525,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.session_replays ( started_at DateTime64(3, 'UTC'), last_event_at DateTime64(3, 'UTC'), created_at DateTime64(3, 'UTC'), + chunk_count UInt64, sync_sequence_id Int64, sync_is_deleted UInt8, sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) @@ -539,7 +540,7 @@ CREATE OR REPLACE VIEW default.session_replays SQL SECURITY DEFINER AS SELECT project_id, branch_id, id, user_id, refresh_token_id, - started_at, last_event_at, created_at + started_at, last_event_at, created_at, chunk_count FROM analytics_internal.session_replays FINAL WHERE sync_is_deleted = 0; diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index fea9952e52..61377e2fe6 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -829,6 +829,7 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.session_replays TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_permissions TO limited_user" }, @@ -585,7 +586,6 @@ it("can see only some tables", async ({ expect }) => { }, { "database": "default", - "name": "team_members", "name": "team_invitations", }, { @@ -624,6 +624,7 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { { "name": "contact_channels" }, { "name": "email_outboxes" }, { "name": "events" }, + { "name": "session_replays" }, { "name": "team_invitations" }, { "name": "team_member_profiles" }, { "name": "team_permissions" }, @@ -1112,6 +1113,7 @@ it("shows grants", async ({ expect }) => { { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.session_replays TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_permissions TO limited_user" }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 1a31af5333..17799fa088 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -1355,6 +1355,24 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); expect(uploadRes.status).toBe(200); expect(uploadRes.body.deduped).toBe(false); + const replayId = uploadRes.body.session_replay_id; + + const secondUploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { + method: "POST", + accessType: "client", + body: { + browser_session_id: browserSessionId, + session_replay_segment_id: randomUUID(), + batch_id: randomUUID(), + started_at_ms: now + 1_000, + sent_at_ms: now + 1_500, + events: [ + { timestamp: now + 1_100, type: 2 }, + ], + }, + }); + expect(secondUploadRes.status).toBe(200); + expect(secondUploadRes.body.session_replay_id).toBe(replayId); await InternalApiKey.createAndSetProjectKeys(); @@ -1366,21 +1384,27 @@ describe.sequential('External DB Sync - Basic Tests', () => { let response; while (performance.now() - start < timeoutMs) { response = await runQueryForCurrentProject({ - query: "SELECT id, user_id, refresh_token_id, started_at, last_event_at FROM session_replays LIMIT 10", + query: "SELECT id, user_id, refresh_token_id, started_at, last_event_at, chunk_count FROM session_replays LIMIT 10", }); expect(response.status).toBe(200); - if (response.body.result.length >= 1) { + const syncedRow = response.body.result.find((resultRow: Record) => resultRow.id === replayId); + if (syncedRow && Number(syncedRow.chunk_count) === 2) { break; } await wait(intervalMs); } - expect(response!.body.result.length).toBeGreaterThanOrEqual(1); - const row = response!.body.result[0]; + const row = response!.body.result.find((resultRow: Record) => resultRow.id === replayId); + expect(row).toBeDefined(); + if (!row) { + throw new Error("Expected synced ClickHouse session replay row to be present."); + } + expect(row.id).toBe(replayId); expect(row.user_id).toBeDefined(); expect(row.refresh_token_id).toBeDefined(); expect(row.started_at).toBeDefined(); expect(row.last_event_at).toBeDefined(); + expect(Number(row.chunk_count)).toBe(2); }, TEST_TIMEOUT); /** @@ -1431,7 +1455,24 @@ describe.sequential('External DB Sync - Basic Tests', () => { const client = dbManager.getClient(dbName); // Wait for the session replay row to appear in external DB - await waitForSyncedSessionReplay(client, replayId); + const secondUploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { + method: "POST", + accessType: "client", + body: { + browser_session_id: browserSessionId, + session_replay_segment_id: randomUUID(), + batch_id: randomUUID(), + started_at_ms: now + 1_000, + sent_at_ms: now + 1_500, + events: [ + { timestamp: now + 1_100, type: 2 }, + ], + }, + }); + expect(secondUploadRes.status).toBe(200); + expect(secondUploadRes.body.session_replay_id).toBe(replayId); + + await waitForSyncedSessionReplay(client, replayId, 2); // Verify the synced row has expected columns const res = await client.query(`SELECT * FROM "session_replays" WHERE "id" = $1`, [replayId]); @@ -1439,8 +1480,13 @@ describe.sequential('External DB Sync - Basic Tests', () => { const row = res.rows[0]; expect(row.user_id).toBeDefined(); expect(row.refresh_token_id).toBeDefined(); - expect(row.started_at).toBeDefined(); - expect(row.last_event_at).toBeDefined(); + expect(row.created_at).toBeInstanceOf(Date); + expect(row.started_at).toBeInstanceOf(Date); + expect(row.last_event_at).toBeInstanceOf(Date); + expect(row).toMatchObject({ + id: replayId, + chunk_count: "2", + }); }, TEST_TIMEOUT); /** @@ -1481,5 +1527,3 @@ describe.sequential('External DB Sync - Basic Tests', () => { }, TEST_TIMEOUT); }); - - diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 55dd3e993d..bc6066408b 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -355,7 +355,7 @@ export async function waitForSyncedEmailOutbox(client: Client, emailId: string, ); } -export async function waitForSyncedSessionReplay(client: Client, replayId: string) { +export async function waitForSyncedSessionReplay(client: Client, replayId: string, expectedChunkCount?: number) { await waitForExternalDbRow( client, `SELECT * FROM "session_replays" WHERE "id" = $1`, @@ -363,6 +363,7 @@ export async function waitForSyncedSessionReplay(client: Client, replayId: strin { shouldExist: true, description: `session replay "${replayId}" to appear in external DB`, + checkRow: expectedChunkCount == null ? undefined : (row) => Number(row.chunk_count) === expectedChunkCount, }, ); } diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index b1f85a8555..e556c23f6b 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -1609,7 +1609,8 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "refresh_token_id" text NOT NULL, "started_at" timestamp without time zone NOT NULL, "last_event_at" timestamp without time zone NOT NULL, - "created_at" timestamp without time zone NOT NULL + "created_at" timestamp without time zone NOT NULL, + "chunk_count" bigint NOT NULL DEFAULT 0 ); REVOKE ALL ON "session_replays" FROM PUBLIC; GRANT SELECT ON "session_replays" TO PUBLIC; @@ -1630,6 +1631,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { started_at DateTime64(3, 'UTC'), last_event_at DateTime64(3, 'UTC'), created_at DateTime64(3, 'UTC'), + chunk_count UInt64, sync_sequence_id Int64, sync_is_deleted UInt8, sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) @@ -1650,6 +1652,12 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "SessionReplay"."startedAt" AS "started_at", "SessionReplay"."lastEventAt" AS "last_event_at", "SessionReplay"."createdAt" AS "created_at", + ( + SELECT COUNT(*) + FROM "SessionReplayChunk" + WHERE "SessionReplayChunk"."tenancyId" = "SessionReplay"."tenancyId" + AND "SessionReplayChunk"."sessionReplayId" = "SessionReplay"."id" + ) AS "chunk_count", "SessionReplay"."sequenceId" AS "sync_sequence_id", "SessionReplay"."tenancyId" AS "tenancyId", false AS "sync_is_deleted" @@ -1670,6 +1678,12 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "SessionReplay"."startedAt" AS "started_at", "SessionReplay"."lastEventAt" AS "last_event_at", "SessionReplay"."createdAt" AS "created_at", + ( + SELECT COUNT(*) + FROM "SessionReplayChunk" + WHERE "SessionReplayChunk"."tenancyId" = "SessionReplay"."tenancyId" + AND "SessionReplayChunk"."sessionReplayId" = "SessionReplay"."id" + ) AS "chunk_count", "SessionReplay"."sequenceId" AS "sequence_id", "SessionReplay"."tenancyId", false AS "is_deleted" @@ -1690,9 +1704,10 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { $4::timestamp without time zone AS "started_at", $5::timestamp without time zone AS "last_event_at", $6::timestamp without time zone AS "created_at", - $7::bigint AS "sequence_id", - $8::boolean AS "is_deleted", - $9::text AS "mapping_name" + $7::bigint AS "chunk_count", + $8::bigint AS "sequence_id", + $9::boolean AS "is_deleted", + $10::text AS "mapping_name" ), deleted AS ( DELETE FROM "session_replays" sr @@ -1707,7 +1722,8 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "refresh_token_id", "started_at", "last_event_at", - "created_at" + "created_at", + "chunk_count" ) SELECT p."id", @@ -1715,7 +1731,8 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { p."refresh_token_id", p."started_at", p."last_event_at", - p."created_at" + p."created_at", + p."chunk_count" FROM params p WHERE p."is_deleted" = false ON CONFLICT ("id") DO UPDATE SET @@ -1723,7 +1740,8 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "refresh_token_id" = EXCLUDED."refresh_token_id", "started_at" = EXCLUDED."started_at", "last_event_at" = EXCLUDED."last_event_at", - "created_at" = EXCLUDED."created_at" + "created_at" = EXCLUDED."created_at", + "chunk_count" = EXCLUDED."chunk_count" RETURNING 1 ) INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") From 66cff4d125bb97a588784101872954b93108bd39 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Wed, 18 Mar 2026 09:42:15 -0700 Subject: [PATCH 10/14] clickhouse sync project permissions and notifications --- .../migration.sql | 25 ++ apps/backend/prisma/schema.prisma | 10 + apps/backend/scripts/clickhouse-migrations.ts | 75 +++++ apps/backend/scripts/run-cron-jobs.ts | 3 +- .../emails/notification-preference/crud.tsx | 9 +- .../external-db-sync/sequencer/route.ts | 57 +++- .../backend/src/app/api/latest/users/crud.tsx | 12 +- apps/backend/src/lib/external-db-sync.ts | 161 +++++++++ apps/backend/src/lib/permissions.tsx | 37 ++- .../endpoints/api/v1/analytics-query.test.ts | 14 + .../api/v1/external-db-sync-basics.test.ts | 197 ++++++++++- .../api/v1/external-db-sync-utils.ts | 28 ++ .../src/config/db-sync-mappings.ts | 307 ++++++++++++++++++ 13 files changed, 907 insertions(+), 28 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql diff --git a/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql new file mode 100644 index 0000000000..5c40d540cf --- /dev/null +++ b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql @@ -0,0 +1,25 @@ +-- AlterTable +ALTER TABLE "ProjectUserDirectPermission" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "ProjectUserDirectPermission_sequenceId_key" ON "ProjectUserDirectPermission"("sequenceId"); + +-- CreateIndex +CREATE INDEX "ProjectUserDirectPermission_shouldUpdateSequenceId_idx" ON "ProjectUserDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- CreateIndex +CREATE INDEX "ProjectUserDirectPermission_tenancyId_sequenceId_idx" ON "ProjectUserDirectPermission"("tenancyId", "sequenceId"); + +-- AlterTable +ALTER TABLE "UserNotificationPreference" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "UserNotificationPreference_sequenceId_key" ON "UserNotificationPreference"("sequenceId"); + +-- CreateIndex +CREATE INDEX "UserNotificationPreference_shouldUpdateSequenceId_idx" ON "UserNotificationPreference"("shouldUpdateSequenceId", "tenancyId"); + +-- CreateIndex +CREATE INDEX "UserNotificationPreference_tenancyId_sequenceId_idx" ON "UserNotificationPreference"("tenancyId", "sequenceId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 455c0e4761..e87ebf21de 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -236,7 +236,12 @@ model ProjectUserDirectPermission { projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@unique([tenancyId, projectUserId, permissionId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserDirectPermission_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "ProjectUserDirectPermission_tenancyId_sequenceId_idx") } model TeamMemberDirectPermission { @@ -1089,8 +1094,13 @@ model UserNotificationPreference { enabled Boolean projectUser ProjectUser @relation(fields: [tenancyId, projectUserId], references: [tenancyId, projectUserId], onDelete: Cascade) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) @@unique([tenancyId, projectUserId, notificationCategoryId]) + @@index([shouldUpdateSequenceId, tenancyId], name: "UserNotificationPreference_shouldUpdateSequenceId_idx") + @@index([tenancyId, sequenceId], name: "UserNotificationPreference_tenancyId_sequenceId_idx") } model ThreadMessage { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index fe88250cf3..0f821b0902 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -30,6 +30,10 @@ export async function runClickhouseMigrations() { await client.exec({ query: EMAIL_OUTBOXES_VIEW_SQL }); await client.exec({ query: SESSION_REPLAYS_TABLE_BASE_SQL }); await client.exec({ query: SESSION_REPLAYS_VIEW_SQL }); + await client.exec({ query: PROJECT_PERMISSIONS_TABLE_BASE_SQL }); + await client.exec({ query: PROJECT_PERMISSIONS_VIEW_SQL }); + await client.exec({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }); + await client.exec({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -48,6 +52,8 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.team_invitations TO limited_user;", "GRANT SELECT ON default.email_outboxes TO limited_user;", "GRANT SELECT ON default.session_replays TO limited_user;", + "GRANT SELECT ON default.project_permissions TO limited_user;", + "GRANT SELECT ON default.notification_preferences TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -76,6 +82,12 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS session_replays_project_isolation ON default.session_replays FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS project_permissions_project_isolation ON default.project_permissions FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS notification_preferences_project_isolation ON default.notification_preferences FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -546,6 +558,69 @@ FINAL WHERE sync_is_deleted = 0; `; +const PROJECT_PERMISSIONS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( + project_id String, + branch_id String, + user_id UUID, + permission_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, user_id, permission_id); +`; + +const PROJECT_PERMISSIONS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.project_permissions +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + user_id, + permission_id, + created_at +FROM analytics_internal.project_permissions +FINAL +WHERE sync_is_deleted = 0; +`; + +const NOTIFICATION_PREFERENCES_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + notification_category_id String, + enabled UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +ORDER BY (project_id, branch_id, id); +`; + +const NOTIFICATION_PREFERENCES_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.notification_preferences +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + notification_category_id, + enabled +FROM analytics_internal.notification_preferences +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts index 98b9680ce5..3353d93813 100644 --- a/apps/backend/scripts/run-cron-jobs.ts +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -30,8 +30,7 @@ async function main() { if (runResult.status === "error") { captureError("run-cron-jobs", runResult.error); } - // Vercel only guarantees minute-granularity for cron jobs, so we randomize the interval - await wait(Math.random() * 120_000); + await wait(1000) } }); } diff --git a/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx b/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx index dd9deed63a..e9b9c8583e 100644 --- a/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx +++ b/apps/backend/src/app/api/latest/emails/notification-preference/crud.tsx @@ -1,3 +1,4 @@ +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { listNotificationCategories } from "@/lib/notification-categories"; import { ensureUserExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy } from "@/prisma-client"; @@ -40,15 +41,15 @@ export const notificationPreferencesCrudHandlers = createLazyProxy(() => createC notificationCategoryId: params.notification_category_id, }, }, - update: { + update: withExternalDbSyncUpdate({ enabled: data.enabled, - }, - create: { + }), + create: withExternalDbSyncUpdate({ tenancyId: auth.tenancy.id, projectUserId: userId, notificationCategoryId: params.notification_category_id, enabled: data.enabled, - }, + }), }); return { diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index ae5a63845e..9c506db171 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -319,6 +319,61 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const projectPermissionTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id" + FROM "ProjectUserDirectPermission" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserDirectPermission" pp + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE pp."id" = r."id" + RETURNING pp."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.project-permission-tenants", projectPermissionTenants.length); + + if (projectPermissionTenants.length > 0) { + await enqueueExternalDbSyncBatch(projectPermissionTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const notificationPreferenceTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "UserNotificationPreference" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "UserNotificationPreference" np + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE np."tenancyId" = r."tenancyId" + AND np."id" = r."id" + RETURNING np."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.notification-preference-tenants", notificationPreferenceTenants.length); + + if (notificationPreferenceTenants.length > 0) { + await enqueueExternalDbSyncBatch(notificationPreferenceTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -348,7 +403,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, EO=${emailOutboxTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, EO=${emailOutboxTenants.length}, SR=${sessionReplayTenants.length}, PP=${projectPermissionTenants.length}, NP=${notificationPreferenceTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index 61c694e573..7f0b1249f0 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,7 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; -import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncNotificationPreferenceDeletionsForUser, recordExternalDbSyncProjectPermissionDeletionsForUser, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -1212,6 +1212,16 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }); + await recordExternalDbSyncProjectPermissionDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncNotificationPreferenceDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 61377e2fe6..2c4f6e7f3a 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -58,6 +58,16 @@ type ExternalDbSyncTarget = tenancyId: string, permissionDbId: string, } + | { + tableName: "ProjectUserDirectPermission", + tenancyId: string, + permissionDbId: string, + } + | { + tableName: "UserNotificationPreference", + tenancyId: string, + notificationPreferenceId: string, + } | { tableName: "VerificationCode_TEAM_INVITATION", tenancyId: string, @@ -289,6 +299,80 @@ export async function recordExternalDbSyncDeletion( return; } + if (target.tableName === "ProjectUserDirectPermission") { + assertUuid(target.permissionDbId, "permissionDbId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'permissionId', "permissionId" + ), + to_jsonb("ProjectUserDirectPermission".*), + NOW(), + TRUE + FROM "ProjectUserDirectPermission" + WHERE "id" = ${target.permissionDbId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserDirectPermission, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "UserNotificationPreference") { + assertUuid(target.notificationPreferenceId, "notificationPreferenceId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'UserNotificationPreference', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'id', "id" + ), + to_jsonb("UserNotificationPreference".*), + NOW(), + TRUE + FROM "UserNotificationPreference" + WHERE "id" = ${target.notificationPreferenceId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for UserNotificationPreference, got ${insertedCount}.` + ); + } + return; + } + { const _verificationCodeTarget: { tableName: "VerificationCode_TEAM_INVITATION" } = target; assertNonEmptyString(target.verificationCodeProjectId, "verificationCodeProjectId"); @@ -604,6 +688,83 @@ export async function recordExternalDbSyncTeamMemberDeletionsForUser( `); } +export async function recordExternalDbSyncProjectPermissionDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserDirectPermission', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'projectUserId', "projectUserId", + 'permissionId', "permissionId" + ), + to_jsonb("ProjectUserDirectPermission".*), + NOW(), + TRUE + FROM "ProjectUserDirectPermission" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + +export async function recordExternalDbSyncNotificationPreferenceDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'UserNotificationPreference', + jsonb_build_object( + 'tenancyId', "tenancyId", + 'id', "id" + ), + to_jsonb("UserNotificationPreference".*), + NOW(), + TRUE + FROM "UserNotificationPreference" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + type PgErrorLike = { code?: string, constraint?: string, diff --git a/apps/backend/src/lib/permissions.tsx b/apps/backend/src/lib/permissions.tsx index 179b2bd84d..2bec329374 100644 --- a/apps/backend/src/lib/permissions.tsx +++ b/apps/backend/src/lib/permissions.tsx @@ -438,6 +438,20 @@ export async function deletePermissionDefinition( }, }); } else { + const projectPermissions = await sourceOfTruthTx.projectUserDirectPermission.findMany({ + where: { + tenancyId: options.tenancy.id, + permissionId: options.permissionId, + }, + select: { id: true }, + }); + for (const perm of projectPermissions) { + await recordExternalDbSyncDeletion(sourceOfTruthTx, { + tableName: "ProjectUserDirectPermission", + tenancyId: options.tenancy.id, + permissionDbId: perm.id, + }); + } await sourceOfTruthTx.projectUserDirectPermission.deleteMany({ where: { tenancyId: options.tenancy.id, @@ -471,12 +485,12 @@ export async function grantProjectPermission( permissionId: options.permissionId, }, }, - create: { + create: withExternalDbSyncUpdate({ permissionId: options.permissionId, projectUserId: options.userId, tenancyId: options.tenancy.id, - }, - update: {}, + }), + update: withExternalDbSyncUpdate({}), }); return { @@ -493,6 +507,23 @@ export async function revokeProjectPermission( permissionId: string, } ) { + const permissionRecord = await tx.projectUserDirectPermission.findUniqueOrThrow({ + where: { + tenancyId_projectUserId_permissionId: { + tenancyId: options.tenancy.id, + projectUserId: options.userId, + permissionId: options.permissionId, + }, + }, + select: { id: true }, + }); + + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUserDirectPermission", + tenancyId: options.tenancy.id, + permissionDbId: permissionRecord.id, + }); + await tx.projectUserDirectPermission.delete({ where: { tenancyId_projectUserId_permissionId: { diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index 1564a6e662..b16130758c 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -525,6 +525,8 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.notification_preferences TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.project_permissions TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.session_replays TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, @@ -580,6 +582,14 @@ it("can see only some tables", async ({ expect }) => { "database": "default", "name": "events", }, + { + "database": "default", + "name": "notification_preferences", + }, + { + "database": "default", + "name": "project_permissions", + }, { "database": "default", "name": "session_replays", @@ -624,6 +634,8 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { { "name": "contact_channels" }, { "name": "email_outboxes" }, { "name": "events" }, + { "name": "notification_preferences" }, + { "name": "project_permissions" }, { "name": "session_replays" }, { "name": "team_invitations" }, { "name": "team_member_profiles" }, @@ -1113,6 +1125,8 @@ it("shows grants", async ({ expect }) => { { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.notification_preferences TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.project_permissions TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.session_replays TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 17799fa088..7519648a81 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -26,6 +26,10 @@ import { waitForSyncedTeamMemberDeletion, waitForSyncedTeamPermission, waitForSyncedTeamPermissionDeletion, + waitForSyncedProjectPermission, + waitForSyncedProjectPermissionDeletion, + waitForCondition, + waitForSyncedNotificationPreference, waitForTable } from './external-db-sync-utils'; @@ -954,6 +958,168 @@ describe.sequential('External DB Sync - Basic Tests', () => { expect(response!.body.result[0].permission_id).toBe('$read_members'); }, TEST_TIMEOUT); + /** + * What it does: + * - Creates a user, grants a project permission, verifies in external DB, + * revokes the permission, and verifies removal. + */ + test('ProjectPermission CRUD sync (Postgres)', async () => { + const dbName = 'project_permission_crud_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + // Create a project permission definition via config + await Project.updateConfig({ + "rbac.permissions": { "test_perm": { scope: "project" } }, + }); + + const user = await User.create({ primary_email: 'pp-crud@example.com' }); + + // Grant a project permission + const grantResponse = await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/test_perm`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + expect(grantResponse.status).toBe(201); + + await waitForSyncedProjectPermission(client, user.userId, 'test_perm'); + + const res1 = await client.query(`SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [user.userId, 'test_perm']); + expect(res1.rows.length).toBe(1); + + // Revoke the permission + await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/test_perm`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedProjectPermissionDeletion(client, user.userId, 'test_perm'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user + project permission, queries ClickHouse analytics API to verify. + */ + test('ProjectPermission sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + // Create a project permission definition via config + await Project.updateConfig({ + "rbac.permissions": { "ch_test_perm": { scope: "project" } }, + }); + + const user = await User.create({ primary_email: 'pp-ch@example.com' }); + + await niceBackendFetch(`/api/v1/project-permissions/${user.userId}/ch_test_perm`, { + accessType: 'admin', + method: 'POST', + body: {}, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT user_id, permission_id FROM project_permissions WHERE permission_id = {perm:String}", + params: { perm: 'ch_test_perm' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].permission_id).toBe('ch_test_perm'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user, updates a notification preference, verifies in external DB. + */ + test('NotificationPreference sync (Postgres)', async () => { + const dbName = 'notification_pref_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'np-crud@example.com' }); + + // Update a notification preference + const updateResponse = await niceBackendFetch(`/api/v1/emails/notification-preference/${user.userId}/4f6f8873-3d04-46bd-8bef-18338b1a1b4c`, { + accessType: 'admin', + method: 'PATCH', + body: { enabled: false }, + }); + expect(updateResponse.status).toBe(200); + + await waitForSyncedNotificationPreference(client, user.userId, '4f6f8873-3d04-46bd-8bef-18338b1a1b4c'); + + const res1 = await client.query(`SELECT * FROM "notification_preferences" WHERE "user_id" = $1 AND "notification_category_id" = $2`, [user.userId, '4f6f8873-3d04-46bd-8bef-18338b1a1b4c']); + expect(res1.rows.length).toBe(1); + expect(res1.rows[0].enabled).toBe(false); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user + notification preference, queries ClickHouse analytics API to verify. + */ + test('NotificationPreference sync (ClickHouse)', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + + const user = await User.create({ primary_email: 'np-ch@example.com' }); + + await niceBackendFetch(`/api/v1/emails/notification-preference/${user.userId}/4f6f8873-3d04-46bd-8bef-18338b1a1b4c`, { + accessType: 'admin', + method: 'PATCH', + body: { enabled: false }, + }); + + await InternalApiKey.createAndSetProjectKeys(); + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT user_id, notification_category_id, enabled FROM notification_preferences WHERE notification_category_id = {cat:String}", + params: { cat: '4f6f8873-3d04-46bd-8bef-18338b1a1b4c' }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + break; + } + await wait(intervalMs); + } + + expect(response!.body.result.length).toBe(1); + expect(response!.body.result[0].notification_category_id).toBe('4f6f8873-3d04-46bd-8bef-18338b1a1b4c'); + }, TEST_TIMEOUT); + /** * What it does: * - Sends a team invitation, verifies in external DB, revokes it, verifies removal. @@ -1153,17 +1319,20 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); expect(sendResponse.status).toBe(200); - // Wait for the email to be processed (rendered + sent) - await wait(8_000); - - // Get the email ID from the outbox API - const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { - method: "GET", - accessType: "server", - }); - expect(listResponse.status).toBe(200); - expect(listResponse.body.items.length).toBeGreaterThanOrEqual(1); - const emailId = listResponse.body.items[0].id; + // Poll the outbox API until the email appears + let emailId!: string; + await waitForCondition( + async () => { + const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { + method: "GET", + accessType: "server", + }); + if (listResponse.status !== 200 || listResponse.body.items.length === 0) return false; + emailId = listResponse.body.items[0].id; + return true; + }, + { timeoutMs: 30_000, intervalMs: 500, description: 'email to appear in outbox' } + ); const client = dbManager.getClient(dbName); @@ -1225,9 +1394,6 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); expect(sendResponse.status).toBe(200); - // Wait for the email to be processed - await wait(8_000); - await InternalApiKey.createAndSetProjectKeys(); // Poll ClickHouse until the email_outboxes row appears @@ -1303,9 +1469,6 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); expect(sendResponse.status).toBe(200); - // Wait for the email to finish sending - await wait(8_000); - const client = dbManager.getClient(dbName); // The email should eventually reach SENT status in the external DB diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index bc6066408b..b51baf7c6c 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -368,6 +368,34 @@ export async function waitForSyncedSessionReplay(client: Client, replayId: strin ); } +export async function waitForSyncedProjectPermission(client: Client, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [userId, permissionId], { + shouldExist: true, + description: `project permission (user=${userId}, perm=${permissionId}) to appear in external DB`, + }); +} + +export async function waitForSyncedProjectPermissionDeletion(client: Client, userId: string, permissionId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "project_permissions" WHERE "user_id" = $1 AND "permission_id" = $2`, [userId, permissionId], { + shouldExist: false, + description: `project permission (user=${userId}, perm=${permissionId}) to be removed from external DB`, + }); +} + +export async function waitForSyncedNotificationPreference(client: Client, userId: string, notificationCategoryId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "notification_preferences" WHERE "user_id" = $1 AND "notification_category_id" = $2`, [userId, notificationCategoryId], { + shouldExist: true, + description: `notification preference (user=${userId}, category=${notificationCategoryId}) to appear in external DB`, + }); +} + +export async function waitForSyncedNotificationPreferenceDeletion(client: Client, notificationPreferenceId: string) { + await waitForExternalDbRow(client, `SELECT * FROM "notification_preferences" WHERE "id" = $1`, [notificationPreferenceId], { + shouldExist: false, + description: `notification preference ${notificationPreferenceId} to be removed from external DB`, + }); +} + export async function waitForSyncedEmailOutboxByStatus(client: Client, status: string) { await waitForExternalDbRow( client, diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index e556c23f6b..76566d2459 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -1752,4 +1752,311 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "project_permissions": { + sourceTables: { "ProjectUserDirectPermission": "ProjectUserDirectPermission" }, + targetTable: "project_permissions", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "project_permissions" ( + "user_id" uuid NOT NULL, + "permission_id" text NOT NULL, + "created_at" timestamp without time zone NOT NULL, + PRIMARY KEY ("user_id", "permission_id") + ); + REVOKE ALL ON "project_permissions" FROM PUBLIC; + GRANT SELECT ON "project_permissions" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( + project_id String, + branch_id String, + user_id UUID, + permission_id String, + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, user_id, permission_id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserDirectPermission"."projectUserId" AS "user_id", + "ProjectUserDirectPermission"."permissionId" AS "permission_id", + "ProjectUserDirectPermission"."createdAt" AS "created_at", + "ProjectUserDirectPermission"."sequenceId" AS "sync_sequence_id", + "ProjectUserDirectPermission"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserDirectPermission" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserDirectPermission"."tenancyId" + WHERE "ProjectUserDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserDirectPermission' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserDirectPermission"."projectUserId" AS "user_id", + "ProjectUserDirectPermission"."permissionId" AS "permission_id", + "ProjectUserDirectPermission"."createdAt" AS "created_at", + "ProjectUserDirectPermission"."sequenceId" AS "sequence_id", + "ProjectUserDirectPermission"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserDirectPermission" + WHERE "ProjectUserDirectPermission"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserDirectPermission' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "user_id", + $2::text AS "permission_id", + $3::timestamp without time zone AS "created_at", + $4::bigint AS "sequence_id", + $5::boolean AS "is_deleted", + $6::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "project_permissions" pp + USING params p + WHERE p."is_deleted" = true AND pp."user_id" = p."user_id" AND pp."permission_id" = p."permission_id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "project_permissions" ( + "user_id", + "permission_id", + "created_at" + ) + SELECT + p."user_id", + p."permission_id", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("user_id", "permission_id") DO UPDATE SET + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "notification_preferences": { + sourceTables: { "UserNotificationPreference": "UserNotificationPreference" }, + targetTable: "notification_preferences", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "notification_preferences" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "notification_category_id" text NOT NULL, + "enabled" boolean NOT NULL DEFAULT true + ); + REVOKE ALL ON "notification_preferences" FROM PUBLIC; + GRANT SELECT ON "notification_preferences" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + notification_category_id String, + enabled UInt8, + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + ORDER BY (project_id, branch_id, id); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "UserNotificationPreference"."id" AS "id", + "UserNotificationPreference"."projectUserId" AS "user_id", + "UserNotificationPreference"."notificationCategoryId" AS "notification_category_id", + "UserNotificationPreference"."enabled" AS "enabled", + "UserNotificationPreference"."sequenceId" AS "sync_sequence_id", + "UserNotificationPreference"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "UserNotificationPreference" + JOIN "Tenancy" ON "Tenancy"."id" = "UserNotificationPreference"."tenancyId" + WHERE "UserNotificationPreference"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + ("DeletedRow"."data"->>'notificationCategoryId')::uuid AS "notification_category_id", + ("DeletedRow"."data"->>'enabled')::boolean AS "enabled", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'UserNotificationPreference' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "UserNotificationPreference"."id" AS "id", + "UserNotificationPreference"."projectUserId" AS "user_id", + "UserNotificationPreference"."notificationCategoryId" AS "notification_category_id", + "UserNotificationPreference"."enabled" AS "enabled", + "UserNotificationPreference"."sequenceId" AS "sequence_id", + "UserNotificationPreference"."tenancyId", + false AS "is_deleted" + FROM "UserNotificationPreference" + WHERE "UserNotificationPreference"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + ("DeletedRow"."data"->>'notificationCategoryId')::uuid AS "notification_category_id", + ("DeletedRow"."data"->>'enabled')::boolean AS "enabled", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'UserNotificationPreference' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "notification_category_id", + $4::boolean AS "enabled", + $5::bigint AS "sequence_id", + $6::boolean AS "is_deleted", + $7::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "notification_preferences" np + USING params p + WHERE p."is_deleted" = true AND np."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "notification_preferences" ( + "id", + "user_id", + "notification_category_id", + "enabled" + ) + SELECT + p."id", + p."user_id", + p."notification_category_id", + p."enabled" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "notification_category_id" = EXCLUDED."notification_category_id", + "enabled" = EXCLUDED."enabled" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From bf008e21b2e0de8c294bb5b894c2bfa74860b765 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Wed, 18 Mar 2026 12:20:23 -0700 Subject: [PATCH 11/14] clickhouse sync session and connected accounts --- .../migration.sql | 25 ++ apps/backend/prisma/schema.prisma | 10 + apps/backend/scripts/clickhouse-migrations.ts | 86 ++++ apps/backend/scripts/run-cron-jobs.ts | 2 +- .../api/latest/auth/password/update/route.tsx | 7 + .../src/app/api/latest/auth/sessions/crud.tsx | 7 + .../latest/auth/sessions/current/route.tsx | 8 + .../external-db-sync/sequencer/route.ts | 58 ++- .../app/api/latest/oauth-providers/crud.tsx | 7 + .../backend/src/app/api/latest/users/crud.tsx | 17 +- apps/backend/src/lib/external-db-sync.ts | 168 ++++++++ .../endpoints/api/v1/analytics-query.test.ts | 14 + .../api/v1/external-db-sync-basics.test.ts | 204 ++++++++++ .../api/v1/external-db-sync-utils.ts | 48 +++ .../src/config/db-sync-mappings.ts | 378 ++++++++++++++++++ 15 files changed, 1036 insertions(+), 3 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql diff --git a/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql new file mode 100644 index 0000000000..003b0df2f9 --- /dev/null +++ b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql @@ -0,0 +1,25 @@ +-- AlterTable +ALTER TABLE "ProjectUserRefreshToken" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "ProjectUserRefreshToken_sequenceId_key" ON "ProjectUserRefreshToken"("sequenceId"); + +-- CreateIndex +CREATE INDEX "ProjectUserRefreshToken_shouldUpdateSequenceId_idx" ON "ProjectUserRefreshToken"("shouldUpdateSequenceId", "tenancyId"); + +-- CreateIndex +CREATE INDEX "ProjectUserRefreshToken_tenancyId_sequenceId_idx" ON "ProjectUserRefreshToken"("tenancyId", "sequenceId"); + +-- AlterTable +ALTER TABLE "ProjectUserOAuthAccount" ADD COLUMN "sequenceId" BIGINT, +ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; + +-- CreateIndex +CREATE UNIQUE INDEX "ProjectUserOAuthAccount_sequenceId_key" ON "ProjectUserOAuthAccount"("sequenceId"); + +-- CreateIndex +CREATE INDEX "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx" ON "ProjectUserOAuthAccount"("shouldUpdateSequenceId", "tenancyId"); + +-- CreateIndex +CREATE INDEX "ProjectUserOAuthAccount_tenancyId_sequenceId_idx" ON "ProjectUserOAuthAccount"("tenancyId", "sequenceId"); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index e87ebf21de..d655132adf 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -348,9 +348,14 @@ model ProjectUserOAuthAccount { allowConnectedAccounts Boolean @default(true) allowSignIn Boolean @default(true) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) @@unique([tenancyId, configOAuthProviderId, projectUserId, providerAccountId]) @@index([tenancyId, projectUserId]) + @@index([tenancyId, sequenceId], name: "ProjectUserOAuthAccount_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx") } model SessionReplay { @@ -622,7 +627,12 @@ model ProjectUserRefreshToken { expiresAt DateTime? isImpersonation Boolean @default(false) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + @@id([tenancyId, id]) + @@index([tenancyId, sequenceId], name: "ProjectUserRefreshToken_tenancyId_sequenceId_idx") + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUserRefreshToken_shouldUpdateSequenceId_idx") } model ProjectUserAuthorizationCode { diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index 0f821b0902..e16bd86d6d 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -34,6 +34,10 @@ export async function runClickhouseMigrations() { await client.exec({ query: PROJECT_PERMISSIONS_VIEW_SQL }); await client.exec({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }); await client.exec({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }); + await client.exec({ query: REFRESH_TOKENS_TABLE_BASE_SQL }); + await client.exec({ query: REFRESH_TOKENS_VIEW_SQL }); + await client.exec({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }); + await client.exec({ query: CONNECTED_ACCOUNTS_VIEW_SQL }); await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); @@ -54,6 +58,8 @@ export async function runClickhouseMigrations() { "GRANT SELECT ON default.session_replays TO limited_user;", "GRANT SELECT ON default.project_permissions TO limited_user;", "GRANT SELECT ON default.notification_preferences TO limited_user;", + "GRANT SELECT ON default.refresh_tokens TO limited_user;", + "GRANT SELECT ON default.connected_accounts TO limited_user;", ]; await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", @@ -88,6 +94,12 @@ export async function runClickhouseMigrations() { await client.exec({ query: "CREATE ROW POLICY IF NOT EXISTS notification_preferences_project_isolation ON default.notification_preferences FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS refresh_tokens_project_isolation ON default.refresh_tokens FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); + await client.exec({ + query: "CREATE ROW POLICY IF NOT EXISTS connected_accounts_project_isolation ON default.connected_accounts FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", + }); for (const query of queries) { await client.exec({ query }); } @@ -621,6 +633,80 @@ FINAL WHERE sync_is_deleted = 0; `; +const REFRESH_TOKENS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.refresh_tokens ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + created_at DateTime64(3, 'UTC'), + last_used_at DateTime64(3, 'UTC'), + is_impersonation UInt8, + expires_at Nullable(DateTime64(3, 'UTC')), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const REFRESH_TOKENS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.refresh_tokens +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + created_at, + last_used_at, + is_impersonation, + expires_at +FROM analytics_internal.refresh_tokens +FINAL +WHERE sync_is_deleted = 0; +`; + +const CONNECTED_ACCOUNTS_TABLE_BASE_SQL = ` +CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + provider String, + provider_account_id String, + email Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) +) +ENGINE ReplacingMergeTree(sync_sequence_id) +PARTITION BY toYYYYMM(created_at) +ORDER BY (project_id, branch_id, id); +`; + +const CONNECTED_ACCOUNTS_VIEW_SQL = ` +CREATE OR REPLACE VIEW default.connected_accounts +SQL SECURITY DEFINER +AS +SELECT + project_id, + branch_id, + id, + user_id, + provider, + provider_account_id, + email, + created_at +FROM analytics_internal.connected_accounts +FINAL +WHERE sync_is_deleted = 0; +`; + const EXTERNAL_ANALYTICS_DB_SQL = ` CREATE DATABASE IF NOT EXISTS analytics_internal; `; diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts index 3353d93813..abd2214958 100644 --- a/apps/backend/scripts/run-cron-jobs.ts +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -30,7 +30,7 @@ async function main() { if (runResult.status === "error") { captureError("run-cron-jobs", runResult.error); } - await wait(1000) + await wait(1000); } }); } diff --git a/apps/backend/src/app/api/latest/auth/password/update/route.tsx b/apps/backend/src/app/api/latest/auth/password/update/route.tsx index db6d43f244..75756e4783 100644 --- a/apps/backend/src/app/api/latest/auth/password/update/route.tsx +++ b/apps/backend/src/app/api/latest/auth/password/update/route.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncRefreshTokenDeletionsForUser } from "@/lib/external-db-sync"; import { getPrismaClientForTenancy, globalPrismaClient, retryTransaction } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { KnownErrors } from "@stackframe/stack-shared"; @@ -78,6 +79,12 @@ export const POST = createSmartRouteHandler({ }); // reset all other refresh tokens + await recordExternalDbSyncRefreshTokenDeletionsForUser(globalPrismaClient, { + tenancyId: tenancy.id, + projectUserId: user.id, + excludeRefreshToken: refreshToken?.[0], + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: tenancy.id, diff --git a/apps/backend/src/app/api/latest/auth/sessions/crud.tsx b/apps/backend/src/app/api/latest/auth/sessions/crud.tsx index 5f7d9d126a..759a2f81ee 100644 --- a/apps/backend/src/app/api/latest/auth/sessions/crud.tsx +++ b/apps/backend/src/app/api/latest/auth/sessions/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { globalPrismaClient } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; import { SmartRequestAuth } from "@/route-handlers/smart-request"; @@ -71,6 +72,12 @@ export const sessionsCrudHandlers = createLazyProxy(() => createCrudHandlers(ses throw new KnownErrors.CannotDeleteCurrentSession(); } + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "ProjectUserRefreshToken", + tenancyId: auth.tenancy.id, + refreshTokenId: params.id, + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: auth.tenancy.id, diff --git a/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx b/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx index 6df9ab3a5f..9ab3716eec 100644 --- a/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx +++ b/apps/backend/src/app/api/latest/auth/sessions/current/route.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { Prisma } from "@/generated/prisma/client"; @@ -32,6 +33,13 @@ export const DELETE = createSmartRouteHandler({ try { const prisma = await getPrismaClientForTenancy(tenancy); + + await recordExternalDbSyncDeletion(globalPrismaClient, { + tableName: "ProjectUserRefreshToken", + tenancyId: tenancy.id, + refreshTokenId, + }); + const result = await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: tenancy.id, diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index 9c506db171..470e89a55d 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -374,6 +374,62 @@ async function backfillSequenceIds(batchSize: number): Promise { didUpdate = true; } + const refreshTokenTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "ProjectUserRefreshToken" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserRefreshToken" rt + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE rt."tenancyId" = r."tenancyId" + AND rt."id" = r."id" + RETURNING rt."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.refresh-token-tenants", refreshTokenTenants.length); + + if (refreshTokenTenants.length > 0) { + await enqueueExternalDbSyncBatch(refreshTokenTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const oauthAccountTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "id" + FROM "ProjectUserOAuthAccount" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUserOAuthAccount" oa + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE oa."tenancyId" = r."tenancyId" + AND oa."id" = r."id" + RETURNING oa."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.oauth-account-tenants", oauthAccountTenants.length); + + if (oauthAccountTenants.length > 0) { + await enqueueExternalDbSyncBatch(oauthAccountTenants.map(t => t.tenancyId)); + didUpdate = true; + } + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` WITH rows_to_update AS ( SELECT "id", "tenancyId" @@ -403,7 +459,7 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); if (didUpdate) { - console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, EO=${emailOutboxTenants.length}, SR=${sessionReplayTenants.length}, PP=${projectPermissionTenants.length}, NP=${notificationPreferenceTenants.length}, DR=${deletedRowTenants.length}`); + console.log(`[Sequencer] Backfilled sequence IDs: USR=${projectUserTenants.length}, CC=${contactChannelTenants.length}, TM=${teamTenants.length}, TMB=${teamMemberTenants.length}, TP=${teamPermissionTenants.length}, TI=${teamInvitationTenants.length}, EO=${emailOutboxTenants.length}, SR=${sessionReplayTenants.length}, PP=${projectPermissionTenants.length}, NP=${notificationPreferenceTenants.length}, RT=${refreshTokenTenants.length}, CA=${oauthAccountTenants.length}, DR=${deletedRowTenants.length}`); } return didUpdate; diff --git a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx index 08eb0d6816..e636e6b2c2 100644 --- a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx +++ b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx @@ -1,3 +1,4 @@ +import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; import { ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -356,6 +357,12 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler }); } + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUserOAuthAccount", + tenancyId: auth.tenancy.id, + oauthAccountId: params.provider_id, + }); + await tx.projectUserOAuthAccount.delete({ where: { tenancyId_id: { diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index 7f0b1249f0..36b9d275c8 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,7 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; -import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncNotificationPreferenceDeletionsForUser, recordExternalDbSyncProjectPermissionDeletionsForUser, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, recordExternalDbSyncNotificationPreferenceDeletionsForUser, recordExternalDbSyncOAuthAccountDeletionsForUser, recordExternalDbSyncProjectPermissionDeletionsForUser, recordExternalDbSyncRefreshTokenDeletionsForUser, recordExternalDbSyncTeamMemberDeletionsForUser, recordExternalDbSyncTeamPermissionDeletionsForUser, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -1156,6 +1156,11 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC // if user password changed, reset all refresh tokens if (passwordHash !== undefined) { + await recordExternalDbSyncRefreshTokenDeletionsForUser(globalPrismaClient, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await globalPrismaClient.projectUserRefreshToken.deleteMany({ where: { tenancyId: auth.tenancy.id, @@ -1222,6 +1227,16 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }); + await recordExternalDbSyncRefreshTokenDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncOAuthAccountDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 2c4f6e7f3a..885952530f 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -74,6 +74,16 @@ type ExternalDbSyncTarget = verificationCodeProjectId: string, verificationCodeBranchId: string, verificationCodeId: string, + } + | { + tableName: "ProjectUserRefreshToken", + tenancyId: string, + refreshTokenId: string, + } + | { + tableName: "ProjectUserOAuthAccount", + tenancyId: string, + oauthAccountId: string, }; type ExternalDbType = NonNullable["type"]>; @@ -373,6 +383,74 @@ export async function recordExternalDbSyncDeletion( return; } + if (target.tableName === "ProjectUserRefreshToken") { + assertUuid(target.refreshTokenId, "refreshTokenId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserRefreshToken', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserRefreshToken".*), + NOW(), + TRUE + FROM "ProjectUserRefreshToken" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "id" = ${target.refreshTokenId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserRefreshToken, got ${insertedCount}.` + ); + } + return; + } + + if (target.tableName === "ProjectUserOAuthAccount") { + assertUuid(target.oauthAccountId, "oauthAccountId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserOAuthAccount', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserOAuthAccount".*), + NOW(), + TRUE + FROM "ProjectUserOAuthAccount" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "id" = ${target.oauthAccountId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUserOAuthAccount, got ${insertedCount}.` + ); + } + return; + } + { const _verificationCodeTarget: { tableName: "VerificationCode_TEAM_INVITATION" } = target; assertNonEmptyString(target.verificationCodeProjectId, "verificationCodeProjectId"); @@ -765,6 +843,82 @@ export async function recordExternalDbSyncNotificationPreferenceDeletionsForUser `); } +export async function recordExternalDbSyncRefreshTokenDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + excludeRefreshToken?: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + const excludeCondition = options.excludeRefreshToken + ? Prisma.sql`AND "refreshToken" != ${options.excludeRefreshToken}` + : Prisma.sql``; + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserRefreshToken', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserRefreshToken".*), + NOW(), + TRUE + FROM "ProjectUserRefreshToken" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + ${excludeCondition} + FOR UPDATE + `); +} + +export async function recordExternalDbSyncOAuthAccountDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUserOAuthAccount', + jsonb_build_object('tenancyId', "tenancyId", 'id', "id"), + to_jsonb("ProjectUserOAuthAccount".*), + NOW(), + TRUE + FROM "ProjectUserOAuthAccount" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + type PgErrorLike = { code?: string, constraint?: string, @@ -993,6 +1147,20 @@ const CLICKHOUSE_COLUMN_NORMALIZERS: Record { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON SQLite FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "REVOKE TABLE ENGINE ON URL FROM limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW DATABASES ON default.* TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.connected_accounts TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.contact_channels TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.events TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.notification_preferences TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.project_permissions TO limited_user" }, + { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.refresh_tokens TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.session_replays TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, @@ -570,6 +572,10 @@ it("can see only some tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { + "database": "default", + "name": "connected_accounts", + }, { "database": "default", "name": "contact_channels", @@ -590,6 +596,10 @@ it("can see only some tables", async ({ expect }) => { "database": "default", "name": "project_permissions", }, + { + "database": "default", + "name": "refresh_tokens", + }, { "database": "default", "name": "session_replays", @@ -631,11 +641,13 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { "status": 200, "body": { "result": [ + { "name": "connected_accounts" }, { "name": "contact_channels" }, { "name": "email_outboxes" }, { "name": "events" }, { "name": "notification_preferences" }, { "name": "project_permissions" }, + { "name": "refresh_tokens" }, { "name": "session_replays" }, { "name": "team_invitations" }, { "name": "team_member_profiles" }, @@ -1122,11 +1134,13 @@ it("shows grants", async ({ expect }) => { "status": 200, "body": { "result": [ + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.connected_accounts TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.contact_channels TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.email_outboxes TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.events TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.notification_preferences TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.project_permissions TO limited_user" }, + { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.refresh_tokens TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.session_replays TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 7519648a81..752b685630 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -13,10 +13,14 @@ import { verifyNotInExternalDb, waitForSyncedContactChannel, waitForSyncedContactChannelDeletion, + waitForSyncedConnectedAccount, + waitForSyncedConnectedAccountDeletion, waitForSyncedData, waitForSyncedDeletion, waitForSyncedEmailOutbox, waitForSyncedEmailOutboxByStatus, + waitForSyncedRefreshToken, + waitForSyncedRefreshTokenDeletion, waitForSyncedSessionReplay, waitForSyncedTeam, waitForSyncedTeamDeletion, @@ -1689,4 +1693,204 @@ describe.sequential('External DB Sync - Basic Tests', () => { }); }, TEST_TIMEOUT); + /** + * What it does: + * - Signs up a user (which creates a refresh token), waits for it to sync to the external DB. + * + * Why it matters: + * - Validates that refresh tokens are synced to external databases. + */ + test('Refresh token sync to external DB', async ({ expect }) => { + const dbName = 'refresh_token_sync'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: "postgres", + connectionString, + }, + }, { config: { magic_link_enabled: true } }); + + const signUpRes = await Auth.Otp.signIn(); + + // List sessions to get the session (refresh token) ID + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + expect(listRes.body.items.length).toBeGreaterThanOrEqual(1); + const sessionId = listRes.body.items[0].id; + + const client = dbManager.getClient(dbName); + await waitForSyncedRefreshToken(client, sessionId); + + const res = await client.query(`SELECT * FROM "refresh_tokens" WHERE "id" = $1`, [sessionId]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].user_id).toBe(signUpRes.userId); + expect(res.rows[0].is_impersonation).toBe(false); + expect(res.rows[0].created_at).toBeInstanceOf(Date); + expect(res.rows[0].last_used_at).toBeInstanceOf(Date); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, revokes the session, and waits for the deletion to sync. + * + * Why it matters: + * - Validates that refresh token deletions are synced to external databases. + */ + test('Refresh token deletion sync to external DB', async ({ expect }) => { + const dbName = 'refresh_token_delete_sync'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: "postgres", + connectionString, + }, + }, { config: { magic_link_enabled: true } }); + + const signUpRes = await Auth.Otp.signIn(); + + // Create a second session so we can revoke one + const newSession = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "server", + method: "POST", + body: { user_id: signUpRes.userId }, + }); + expect(newSession.status).toBe(200); + + // List sessions to find the second session ID + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + const nonCurrentSession = listRes.body.items.find((s: any) => !s.is_current_session); + expect(nonCurrentSession).toBeDefined(); + + const client = dbManager.getClient(dbName); + await waitForSyncedRefreshToken(client, nonCurrentSession.id); + + // Revoke the non-current session + const deleteRes = await niceBackendFetch(`/api/v1/auth/sessions/${nonCurrentSession.id}`, { + accessType: "client", + method: "DELETE", + query: { user_id: signUpRes.userId }, + }); + expect(deleteRes.status).toBe(200); + + await waitForSyncedRefreshTokenDeletion(client, nonCurrentSession.id); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, verifies refresh token appears in ClickHouse. + * + * Why it matters: + * - Validates ClickHouse refresh_tokens table sync. + */ + test('Refresh token sync to ClickHouse', async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + await InternalApiKey.createAndSetProjectKeys(); + + const signUpRes = await Auth.Otp.signIn(); + + const listRes = await niceBackendFetch("/api/v1/auth/sessions", { + accessType: "client", + method: "GET", + query: { user_id: signUpRes.userId }, + }); + expect(listRes.status).toBe(200); + const sessionId = listRes.body.items[0].id; + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, user_id, is_impersonation FROM refresh_tokens WHERE id = {session_id:UUID}", + params: { session_id: sessionId }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + expect(response.body.result[0]).toMatchObject({ + id: sessionId, + user_id: signUpRes.userId, + is_impersonation: 0, + }); + return; + } + await wait(intervalMs); + } + throw new StackAssertionError(`Timed out waiting for ClickHouse refresh token to sync.`, { response }); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Signs up a user, verifies connected account appears in ClickHouse. + * + * Why it matters: + * - Validates ClickHouse connected_accounts table sync. + */ + test('Connected account sync to ClickHouse', async ({ expect }) => { + // Use default project (has spotify configured) with analytics keys + await Auth.OAuth.signIn(); + await InternalApiKey.createAndSetProjectKeys(); + + // Get the user ID + const userRes = await niceBackendFetch("/api/v1/users/me", { + accessType: "client", + method: "GET", + }); + expect(userRes.status).toBe(200); + const userId = userRes.body.id; + + // Create an additional connected account via the oauth-providers API so we have a known ID + const createRes = await niceBackendFetch("/api/v1/oauth-providers", { + accessType: "server", + method: "POST", + body: { + user_id: userId, + provider_config_id: "spotify", + account_id: "ch-test-account-12345", + email: "chuser@example.com", + allow_sign_in: false, + allow_connected_accounts: true, + }, + }); + expect(createRes.status).toBe(201); + const accountId = createRes.body.id; + + const timeoutMs = 180_000; + const intervalMs = 2_000; + const start = performance.now(); + + let response; + while (performance.now() - start < timeoutMs) { + response = await runQueryForCurrentProject({ + query: "SELECT id, user_id, provider, provider_account_id, email FROM connected_accounts WHERE id = {account_id:UUID}", + params: { account_id: accountId }, + }); + expect(response.status).toBe(200); + if (response.body.result.length === 1) { + expect(response.body.result[0]).toMatchObject({ + id: accountId, + user_id: userId, + provider: "spotify", + provider_account_id: "ch-test-account-12345", + email: "chuser@example.com", + }); + return; + } + await wait(intervalMs); + } + throw new StackAssertionError(`Timed out waiting for ClickHouse connected account to sync.`, { response }); + }, TEST_TIMEOUT); + }); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index b51baf7c6c..4c91ae93a1 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -396,6 +396,54 @@ export async function waitForSyncedNotificationPreferenceDeletion(client: Client }); } +export async function waitForSyncedRefreshToken(client: Client, refreshTokenId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "refresh_tokens" WHERE "id" = $1`, + [refreshTokenId], + { + shouldExist: true, + description: `refresh token "${refreshTokenId}" to appear in external DB`, + }, + ); +} + +export async function waitForSyncedRefreshTokenDeletion(client: Client, refreshTokenId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "refresh_tokens" WHERE "id" = $1`, + [refreshTokenId], + { + shouldExist: false, + description: `refresh token "${refreshTokenId}" to be removed from external DB`, + }, + ); +} + +export async function waitForSyncedConnectedAccount(client: Client, accountId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "connected_accounts" WHERE "id" = $1`, + [accountId], + { + shouldExist: true, + description: `connected account "${accountId}" to appear in external DB`, + }, + ); +} + +export async function waitForSyncedConnectedAccountDeletion(client: Client, accountId: string) { + await waitForExternalDbRow( + client, + `SELECT * FROM "connected_accounts" WHERE "id" = $1`, + [accountId], + { + shouldExist: false, + description: `connected account "${accountId}" to be removed from external DB`, + }, + ); +} + export async function waitForSyncedEmailOutboxByStatus(client: Client, status: string) { await waitForExternalDbRow( client, diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 76566d2459..0f2ca5fc3b 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -2059,4 +2059,382 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, + "refresh_tokens": { + sourceTables: { "ProjectUserRefreshToken": "ProjectUserRefreshToken" }, + targetTable: "refresh_tokens", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "refresh_tokens" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "created_at" timestamp without time zone NOT NULL, + "last_used_at" timestamp without time zone NOT NULL, + "is_impersonation" boolean NOT NULL DEFAULT false, + "expires_at" timestamp without time zone + ); + REVOKE ALL ON "refresh_tokens" FROM PUBLIC; + GRANT SELECT ON "refresh_tokens" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.refresh_tokens ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + created_at DateTime64(3, 'UTC'), + last_used_at DateTime64(3, 'UTC'), + is_impersonation UInt8, + expires_at Nullable(DateTime64(3, 'UTC')), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + + CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata ( + tenancy_id UUID, + mapping_name String, + last_synced_sequence_id Int64, + updated_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(updated_at) + ORDER BY (tenancy_id, mapping_name); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserRefreshToken"."id" AS "id", + "ProjectUserRefreshToken"."projectUserId" AS "user_id", + "ProjectUserRefreshToken"."createdAt" AS "created_at", + "ProjectUserRefreshToken"."lastActiveAt" AS "last_used_at", + "ProjectUserRefreshToken"."isImpersonation" AS "is_impersonation", + "ProjectUserRefreshToken"."expiresAt" AS "expires_at", + "ProjectUserRefreshToken"."sequenceId" AS "sync_sequence_id", + "ProjectUserRefreshToken"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserRefreshToken" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserRefreshToken"."tenancyId" + WHERE "ProjectUserRefreshToken"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "last_used_at", + false AS "is_impersonation", + NULL::timestamp without time zone AS "expires_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserRefreshToken' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserRefreshToken"."id" AS "id", + "ProjectUserRefreshToken"."projectUserId" AS "user_id", + "ProjectUserRefreshToken"."createdAt" AS "created_at", + "ProjectUserRefreshToken"."lastActiveAt" AS "last_used_at", + "ProjectUserRefreshToken"."isImpersonation" AS "is_impersonation", + "ProjectUserRefreshToken"."expiresAt" AS "expires_at", + "ProjectUserRefreshToken"."sequenceId" AS "sequence_id", + "ProjectUserRefreshToken"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserRefreshToken" + WHERE "ProjectUserRefreshToken"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."deletedAt"::timestamp without time zone AS "last_used_at", + false AS "is_impersonation", + NULL::timestamp without time zone AS "expires_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserRefreshToken' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::timestamp without time zone AS "created_at", + $4::timestamp without time zone AS "last_used_at", + $5::boolean AS "is_impersonation", + $6::timestamp without time zone AS "expires_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "refresh_tokens" rt + USING params p + WHERE p."is_deleted" = true AND rt."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "refresh_tokens" ( + "id", + "user_id", + "created_at", + "last_used_at", + "is_impersonation", + "expires_at" + ) + SELECT + p."id", + p."user_id", + p."created_at", + p."last_used_at", + p."is_impersonation", + p."expires_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "created_at" = EXCLUDED."created_at", + "last_used_at" = EXCLUDED."last_used_at", + "is_impersonation" = EXCLUDED."is_impersonation", + "expires_at" = EXCLUDED."expires_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, + "connected_accounts": { + sourceTables: { "ProjectUserOAuthAccount": "ProjectUserOAuthAccount" }, + targetTable: "connected_accounts", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "connected_accounts" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" uuid NOT NULL, + "provider" text NOT NULL, + "provider_account_id" text NOT NULL, + "email" text, + "created_at" timestamp without time zone NOT NULL + ); + REVOKE ALL ON "connected_accounts" FROM PUBLIC; + GRANT SELECT ON "connected_accounts" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + clickhouse: ` + CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( + project_id String, + branch_id String, + id UUID, + user_id UUID, + provider String, + provider_account_id String, + email Nullable(String), + created_at DateTime64(3, 'UTC'), + sync_sequence_id Int64, + sync_is_deleted UInt8, + sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(sync_sequence_id) + PARTITION BY toYYYYMM(created_at) + ORDER BY (project_id, branch_id, id); + + CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata ( + tenancy_id UUID, + mapping_name String, + last_synced_sequence_id Int64, + updated_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE ReplacingMergeTree(updated_at) + ORDER BY (tenancy_id, mapping_name); + `.trim(), + }, + internalDbFetchQueries: { + clickhouse: ` + SELECT * + FROM ( + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + "ProjectUserOAuthAccount"."id" AS "id", + "ProjectUserOAuthAccount"."projectUserId" AS "user_id", + "ProjectUserOAuthAccount"."configOAuthProviderId" AS "provider", + "ProjectUserOAuthAccount"."providerAccountId" AS "provider_account_id", + "ProjectUserOAuthAccount"."email" AS "email", + "ProjectUserOAuthAccount"."createdAt" AS "created_at", + "ProjectUserOAuthAccount"."sequenceId" AS "sync_sequence_id", + "ProjectUserOAuthAccount"."tenancyId" AS "tenancyId", + false AS "sync_is_deleted" + FROM "ProjectUserOAuthAccount" + JOIN "Tenancy" ON "Tenancy"."id" = "ProjectUserOAuthAccount"."tenancyId" + WHERE "ProjectUserOAuthAccount"."tenancyId" = $1::uuid + AND "ProjectUserOAuthAccount"."projectUserId" IS NOT NULL + + UNION ALL + + SELECT + "Tenancy"."projectId" AS "project_id", + "Tenancy"."branchId" AS "branch_id", + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "provider", + NULL::text AS "provider_account_id", + NULL::text AS "email", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sync_sequence_id", + "DeletedRow"."tenancyId" AS "tenancyId", + true AS "sync_is_deleted" + FROM "DeletedRow" + JOIN "Tenancy" ON "Tenancy"."id" = "DeletedRow"."tenancyId" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserOAuthAccount' + ) AS "_src" + WHERE "sync_sequence_id" IS NOT NULL + AND "sync_sequence_id" > $2::bigint + ORDER BY "sync_sequence_id" ASC + LIMIT 1000 + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUserOAuthAccount"."id" AS "id", + "ProjectUserOAuthAccount"."projectUserId" AS "user_id", + "ProjectUserOAuthAccount"."configOAuthProviderId" AS "provider", + "ProjectUserOAuthAccount"."providerAccountId" AS "provider_account_id", + "ProjectUserOAuthAccount"."email" AS "email", + "ProjectUserOAuthAccount"."createdAt" AS "created_at", + "ProjectUserOAuthAccount"."sequenceId" AS "sequence_id", + "ProjectUserOAuthAccount"."tenancyId", + false AS "is_deleted" + FROM "ProjectUserOAuthAccount" + WHERE "ProjectUserOAuthAccount"."tenancyId" = $1::uuid + AND "ProjectUserOAuthAccount"."projectUserId" IS NOT NULL + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", + ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", + NULL::text AS "provider", + NULL::text AS "provider_account_id", + NULL::text AS "email", + "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUserOAuthAccount' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::uuid AS "user_id", + $3::text AS "provider", + $4::text AS "provider_account_id", + $5::text AS "email", + $6::timestamp without time zone AS "created_at", + $7::bigint AS "sequence_id", + $8::boolean AS "is_deleted", + $9::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "connected_accounts" ca + USING params p + WHERE p."is_deleted" = true AND ca."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "connected_accounts" ( + "id", + "user_id", + "provider", + "provider_account_id", + "email", + "created_at" + ) + SELECT + p."id", + p."user_id", + p."provider", + p."provider_account_id", + p."email", + p."created_at" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "user_id" = EXCLUDED."user_id", + "provider" = EXCLUDED."provider", + "provider_account_id" = EXCLUDED."provider_account_id", + "email" = EXCLUDED."email", + "created_at" = EXCLUDED."created_at" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, } as const; From 2bd11cb7ad362c6c2bb3df954e5995de94964bc3 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Wed, 18 Mar 2026 14:07:26 -0700 Subject: [PATCH 12/14] clickhouse improve sync speed --- apps/backend/src/lib/external-db-sync.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index 885952530f..d2f25202bf 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -1433,7 +1433,9 @@ async function syncClickhouseMapping( } const clickhouseTableName = `analytics_internal.${mapping.targetTable}`; - await ensureClickhouseSchema(client, tableSchema, clickhouseTableName); + // Skip ensureClickhouseSchema — we only sync to our own internal ClickHouse + // where tables are already created. CREATE TABLE IF NOT EXISTS is always a + // no-op but costs a slow DDL round trip per mapping per sync invocation. let lastSequenceId = await getClickhouseLastSyncedSequenceId(client, tenancyId, mappingId); From d9503fb0a24c1b7d973cd569b9f3044d4104a3e7 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 23 Mar 2026 09:23:20 -0700 Subject: [PATCH 13/14] remove extra comments --- apps/backend/src/lib/external-db-sync.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index d2f25202bf..d276e79dc8 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -1433,10 +1433,6 @@ async function syncClickhouseMapping( } const clickhouseTableName = `analytics_internal.${mapping.targetTable}`; - // Skip ensureClickhouseSchema — we only sync to our own internal ClickHouse - // where tables are already created. CREATE TABLE IF NOT EXISTS is always a - // no-op but costs a slow DDL round trip per mapping per sync invocation. - let lastSequenceId = await getClickhouseLastSyncedSequenceId(client, tenancyId, mappingId); const BATCH_LIMIT = 1000; From 98ba6786d1be73272c8d2be769b138712615ebd8 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Mon, 6 Apr 2026 14:43:44 -0700 Subject: [PATCH 14/14] pr comment fixes --- .../migration.sql | 18 - .../migration.sql | 9 - .../migration.sql | 9 - .../migration.sql | 15 - .../migration.sql | 18 - .../migration.sql | 18 - .../migration.sql | 154 +++++++++ apps/backend/scripts/clickhouse-migrations.ts | 250 ++++++-------- .../latest/emails/unsubscribe-link/route.tsx | 9 +- .../external-db-sync/sequencer/route.ts | 17 - .../internal/external-db-sync/status/route.ts | 138 ++++++++ .../app/api/latest/oauth-providers/crud.tsx | 6 +- .../api/latest/team-member-profiles/crud.tsx | 5 +- apps/backend/src/lib/email-queue-step.tsx | 6 +- apps/backend/src/lib/external-db-sync.ts | 5 +- apps/backend/src/lib/permissions.tsx | 8 +- .../external-db-sync/page-client.tsx | 76 +++-- .../endpoints/api/v1/analytics-query.test.ts | 7 - .../api/v1/external-db-sync-basics.test.ts | 187 +---------- .../src/config/db-sync-mappings.ts | 307 ++---------------- 20 files changed, 501 insertions(+), 761 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql diff --git a/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql index 358cf0514c..fc5562113d 100644 --- a/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql +++ b/apps/backend/prisma/migrations/20260316000000_add_team_team_member_sequence_columns/migration.sql @@ -2,24 +2,6 @@ ALTER TABLE "Team" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; --- CreateIndex -CREATE UNIQUE INDEX "Team_sequenceId_key" ON "Team"("sequenceId"); - --- CreateIndex -CREATE INDEX "Team_tenancyId_sequenceId_idx" ON "Team"("tenancyId", "sequenceId"); - --- CreateIndex -CREATE INDEX "Team_shouldUpdateSequenceId_idx" ON "Team"("shouldUpdateSequenceId", "tenancyId"); - -- AlterTable ALTER TABLE "TeamMember" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "TeamMember_sequenceId_key" ON "TeamMember"("sequenceId"); - --- CreateIndex -CREATE INDEX "TeamMember_tenancyId_sequenceId_idx" ON "TeamMember"("tenancyId", "sequenceId"); - --- CreateIndex -CREATE INDEX "TeamMember_shouldUpdateSequenceId_idx" ON "TeamMember"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql index 74b5681c01..c296a66581 100644 --- a/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql +++ b/apps/backend/prisma/migrations/20260316000001_add_email_outbox_sequence_columns/migration.sql @@ -1,12 +1,3 @@ -- AlterTable ALTER TABLE "EmailOutbox" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "EmailOutbox_sequenceId_key" ON "EmailOutbox"("sequenceId"); - --- CreateIndex -CREATE INDEX "EmailOutbox_tenancyId_sequenceId_idx" ON "EmailOutbox"("tenancyId", "sequenceId"); - --- CreateIndex -CREATE INDEX "EmailOutbox_shouldUpdateSequenceId_idx" ON "EmailOutbox"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql index 88e0751fd3..50a85170e2 100644 --- a/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql +++ b/apps/backend/prisma/migrations/20260316000002_add_session_replay_sequence_columns/migration.sql @@ -1,12 +1,3 @@ -- AlterTable ALTER TABLE "SessionReplay" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "SessionReplay_sequenceId_key" ON "SessionReplay"("sequenceId"); - --- CreateIndex -CREATE INDEX "SessionReplay_tenancyId_sequenceId_idx" ON "SessionReplay"("tenancyId", "sequenceId"); - --- CreateIndex -CREATE INDEX "SessionReplay_shouldUpdateSequenceId_idx" ON "SessionReplay"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql index b13d10e6c3..d3158401a8 100644 --- a/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql +++ b/apps/backend/prisma/migrations/20260317000000_add_team_permission_invitation_sequence_columns/migration.sql @@ -2,21 +2,6 @@ ALTER TABLE "TeamMemberDirectPermission" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; --- CreateIndex -CREATE UNIQUE INDEX "TeamMemberDirectPermission_sequenceId_key" ON "TeamMemberDirectPermission"("sequenceId"); - --- CreateIndex -CREATE INDEX "TeamMemberDirectPermission_shouldUpdateSequenceId_idx" ON "TeamMemberDirectPermission"("shouldUpdateSequenceId", "tenancyId"); - --- CreateIndex -CREATE INDEX "TeamMemberDirectPermission_tenancyId_sequenceId_idx" ON "TeamMemberDirectPermission"("tenancyId", "sequenceId"); - -- AlterTable ALTER TABLE "VerificationCode" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "VerificationCode_sequenceId_key" ON "VerificationCode"("sequenceId"); - --- CreateIndex -CREATE INDEX "VerificationCode_shouldUpdateSequenceId_type_idx" ON "VerificationCode"("shouldUpdateSequenceId", "type"); diff --git a/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql index 5c40d540cf..d39d03dd21 100644 --- a/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql +++ b/apps/backend/prisma/migrations/20260317000001_add_project_permission_notification_preference_sequence_columns/migration.sql @@ -2,24 +2,6 @@ ALTER TABLE "ProjectUserDirectPermission" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; --- CreateIndex -CREATE UNIQUE INDEX "ProjectUserDirectPermission_sequenceId_key" ON "ProjectUserDirectPermission"("sequenceId"); - --- CreateIndex -CREATE INDEX "ProjectUserDirectPermission_shouldUpdateSequenceId_idx" ON "ProjectUserDirectPermission"("shouldUpdateSequenceId", "tenancyId"); - --- CreateIndex -CREATE INDEX "ProjectUserDirectPermission_tenancyId_sequenceId_idx" ON "ProjectUserDirectPermission"("tenancyId", "sequenceId"); - -- AlterTable ALTER TABLE "UserNotificationPreference" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "UserNotificationPreference_sequenceId_key" ON "UserNotificationPreference"("sequenceId"); - --- CreateIndex -CREATE INDEX "UserNotificationPreference_shouldUpdateSequenceId_idx" ON "UserNotificationPreference"("shouldUpdateSequenceId", "tenancyId"); - --- CreateIndex -CREATE INDEX "UserNotificationPreference_tenancyId_sequenceId_idx" ON "UserNotificationPreference"("tenancyId", "sequenceId"); diff --git a/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql index 003b0df2f9..61906b1ba2 100644 --- a/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql +++ b/apps/backend/prisma/migrations/20260318000000_add_sequence_id_to_refresh_tokens_and_oauth_accounts/migration.sql @@ -2,24 +2,6 @@ ALTER TABLE "ProjectUserRefreshToken" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; --- CreateIndex -CREATE UNIQUE INDEX "ProjectUserRefreshToken_sequenceId_key" ON "ProjectUserRefreshToken"("sequenceId"); - --- CreateIndex -CREATE INDEX "ProjectUserRefreshToken_shouldUpdateSequenceId_idx" ON "ProjectUserRefreshToken"("shouldUpdateSequenceId", "tenancyId"); - --- CreateIndex -CREATE INDEX "ProjectUserRefreshToken_tenancyId_sequenceId_idx" ON "ProjectUserRefreshToken"("tenancyId", "sequenceId"); - -- AlterTable ALTER TABLE "ProjectUserOAuthAccount" ADD COLUMN "sequenceId" BIGINT, ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT true; - --- CreateIndex -CREATE UNIQUE INDEX "ProjectUserOAuthAccount_sequenceId_key" ON "ProjectUserOAuthAccount"("sequenceId"); - --- CreateIndex -CREATE INDEX "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx" ON "ProjectUserOAuthAccount"("shouldUpdateSequenceId", "tenancyId"); - --- CreateIndex -CREATE INDEX "ProjectUserOAuthAccount_tenancyId_sequenceId_idx" ON "ProjectUserOAuthAccount"("tenancyId", "sequenceId"); diff --git a/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql b/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql new file mode 100644 index 0000000000..c96f0f5648 --- /dev/null +++ b/apps/backend/prisma/migrations/20260318000001_add_sequence_indexes_concurrently/migration.sql @@ -0,0 +1,154 @@ +-- Team indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "Team_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."Team"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "Team_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."Team"("shouldUpdateSequenceId", "tenancyId"); + +-- TeamMember indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMember_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMember"("shouldUpdateSequenceId", "tenancyId"); + +-- EmailOutbox indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "EmailOutbox_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."EmailOutbox"("shouldUpdateSequenceId", "tenancyId"); + +-- SessionReplay indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "SessionReplay_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."SessionReplay"("shouldUpdateSequenceId", "tenancyId"); + +-- TeamMemberDirectPermission indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TeamMemberDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."TeamMemberDirectPermission"("tenancyId", "sequenceId"); + +-- VerificationCode indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "VerificationCode_shouldUpdateSequenceId_type_idx" ON /* SCHEMA_NAME_SENTINEL */."VerificationCode"("shouldUpdateSequenceId", "type"); + +-- ProjectUserDirectPermission indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserDirectPermission_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserDirectPermission"("tenancyId", "sequenceId"); + +-- UserNotificationPreference indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "UserNotificationPreference_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."UserNotificationPreference"("tenancyId", "sequenceId"); + +-- ProjectUserRefreshToken indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserRefreshToken_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserRefreshToken"("tenancyId", "sequenceId"); + +-- ProjectUserOAuthAccount indexes +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUserOAuthAccount_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUserOAuthAccount"("tenancyId", "sequenceId"); diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index e16bd86d6d..8db2a40777 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -2,108 +2,86 @@ import { getClickhouseAdminClient } from "@/lib/clickhouse"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export async function runClickhouseMigrations() { + const start = performance.now(); console.log("[Clickhouse] Running Clickhouse migrations..."); const client = getClickhouseAdminClient(); const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD"); - await client.exec({ - query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}", - query_params: { clickhouseExternalPassword }, - }); - // todo: create migration files - await client.exec({ query: EXTERNAL_ANALYTICS_DB_SQL }); - await client.exec({ query: SYNC_METADATA_TABLE_SQL }); - await client.exec({ query: EVENTS_TABLE_BASE_SQL }); - await client.exec({ query: EVENTS_VIEW_SQL }); - await client.exec({ query: USERS_TABLE_BASE_SQL }); - await client.exec({ query: USERS_VIEW_SQL }); - await client.exec({ query: CONTACT_CHANNELS_TABLE_BASE_SQL }); - await client.exec({ query: CONTACT_CHANNELS_VIEW_SQL }); - await client.exec({ query: TEAMS_TABLE_BASE_SQL }); - await client.exec({ query: TEAMS_VIEW_SQL }); - await client.exec({ query: TEAM_MEMBER_PROFILES_TABLE_BASE_SQL }); - await client.exec({ query: TEAM_MEMBER_PROFILES_VIEW_SQL }); - await client.exec({ query: TEAM_PERMISSIONS_TABLE_BASE_SQL }); - await client.exec({ query: TEAM_PERMISSIONS_VIEW_SQL }); - await client.exec({ query: TEAM_INVITATIONS_TABLE_BASE_SQL }); - await client.exec({ query: TEAM_INVITATIONS_VIEW_SQL }); - await client.exec({ query: EMAIL_OUTBOXES_TABLE_BASE_SQL }); - await client.exec({ query: EMAIL_OUTBOXES_VIEW_SQL }); - await client.exec({ query: SESSION_REPLAYS_TABLE_BASE_SQL }); - await client.exec({ query: SESSION_REPLAYS_VIEW_SQL }); - await client.exec({ query: PROJECT_PERMISSIONS_TABLE_BASE_SQL }); - await client.exec({ query: PROJECT_PERMISSIONS_VIEW_SQL }); - await client.exec({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }); - await client.exec({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }); - await client.exec({ query: REFRESH_TOKENS_TABLE_BASE_SQL }); - await client.exec({ query: REFRESH_TOKENS_VIEW_SQL }); - await client.exec({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }); - await client.exec({ query: CONNECTED_ACCOUNTS_VIEW_SQL }); - await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); - await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); - await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); - await client.exec({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL }); - // Recreate the events view so SELECT * picks up columns added by EVENTS_ADD_REPLAY_COLUMNS_SQL - await client.exec({ query: EVENTS_VIEW_SQL }); - const queries = [ - "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;", - "REVOKE ALL FROM limited_user;", - "GRANT SELECT ON default.events TO limited_user;", - "GRANT SELECT ON default.users TO limited_user;", - "GRANT SELECT ON default.contact_channels TO limited_user;", - "GRANT SELECT ON default.teams TO limited_user;", - "GRANT SELECT ON default.team_member_profiles TO limited_user;", - "GRANT SELECT ON default.team_permissions TO limited_user;", - "GRANT SELECT ON default.team_invitations TO limited_user;", - "GRANT SELECT ON default.email_outboxes TO limited_user;", - "GRANT SELECT ON default.session_replays TO limited_user;", - "GRANT SELECT ON default.project_permissions TO limited_user;", - "GRANT SELECT ON default.notification_preferences TO limited_user;", - "GRANT SELECT ON default.refresh_tokens TO limited_user;", - "GRANT SELECT ON default.connected_accounts TO limited_user;", + + // Setup — database, user, sync metadata + await client.command({ query: EXTERNAL_ANALYTICS_DB_SQL }); + await Promise.all([ + client.command({ + query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}", + query_params: { clickhouseExternalPassword }, + }), + client.command({ query: SYNC_METADATA_TABLE_SQL }), + ]); + + // Create all tables in parallel + await Promise.all([ + client.command({ query: EVENTS_TABLE_BASE_SQL }), + client.command({ query: USERS_TABLE_BASE_SQL }), + client.command({ query: CONTACT_CHANNELS_TABLE_BASE_SQL }), + client.command({ query: TEAMS_TABLE_BASE_SQL }), + client.command({ query: TEAM_MEMBER_PROFILES_TABLE_BASE_SQL }), + client.command({ query: TEAM_PERMISSIONS_TABLE_BASE_SQL }), + client.command({ query: TEAM_INVITATIONS_TABLE_BASE_SQL }), + client.command({ query: EMAIL_OUTBOXES_TABLE_BASE_SQL }), + + client.command({ query: PROJECT_PERMISSIONS_TABLE_BASE_SQL }), + client.command({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }), + client.command({ query: REFRESH_TOKENS_TABLE_BASE_SQL }), + client.command({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }), + ]); + + // Alter events table (must come before views that reference new columns) + await client.command({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL }); + + // Create all views in parallel + await Promise.all([ + client.command({ query: EVENTS_VIEW_SQL }), + client.command({ query: USERS_VIEW_SQL }), + client.command({ query: CONTACT_CHANNELS_VIEW_SQL }), + client.command({ query: TEAMS_VIEW_SQL }), + client.command({ query: TEAM_MEMBER_PROFILES_VIEW_SQL }), + client.command({ query: TEAM_PERMISSIONS_VIEW_SQL }), + client.command({ query: TEAM_INVITATIONS_VIEW_SQL }), + client.command({ query: EMAIL_OUTBOXES_VIEW_SQL }), + + client.command({ query: PROJECT_PERMISSIONS_VIEW_SQL }), + client.command({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }), + client.command({ query: REFRESH_TOKENS_VIEW_SQL }), + client.command({ query: CONNECTED_ACCOUNTS_VIEW_SQL }), + ]); + + // Data migrations (mutations) + await Promise.all([ + client.command({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }), + client.command({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }), + client.command({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL }), + ]); + + // Row policies in parallel + const tables = [ + "events", "users", "contact_channels", "teams", "team_member_profiles", + "team_permissions", "team_invitations", "email_outboxes", + "project_permissions", "notification_preferences", "refresh_tokens", "connected_accounts", ]; - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS users_project_isolation ON default.users FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS contact_channels_project_isolation ON default.contact_channels FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS teams_project_isolation ON default.teams FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS team_member_profiles_project_isolation ON default.team_member_profiles FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS team_permissions_project_isolation ON default.team_permissions FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS team_invitations_project_isolation ON default.team_invitations FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS email_outboxes_project_isolation ON default.email_outboxes FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS session_replays_project_isolation ON default.session_replays FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS project_permissions_project_isolation ON default.project_permissions FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS notification_preferences_project_isolation ON default.notification_preferences FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS refresh_tokens_project_isolation ON default.refresh_tokens FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - await client.exec({ - query: "CREATE ROW POLICY IF NOT EXISTS connected_accounts_project_isolation ON default.connected_accounts FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user", - }); - for (const query of queries) { - await client.exec({ query }); - } - console.log("[Clickhouse] Clickhouse migrations complete"); + await Promise.all(tables.map(table => + client.command({ + query: `CREATE ROW POLICY IF NOT EXISTS ${table}_project_isolation ON default.${table} FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user`, + }) + )); + + // Grants + await client.command({ query: "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;" }); + await client.command({ query: "REVOKE ALL FROM limited_user;" }); + await Promise.all(tables.map(table => + client.command({ query: `GRANT SELECT ON default.${table} TO limited_user;` }) + )); + + const elapsed = ((performance.now() - start) / 1000).toFixed(1); + console.log(`[Clickhouse] Clickhouse migrations complete (${elapsed}s)`); await client.close(); } @@ -351,7 +329,6 @@ CREATE TABLE IF NOT EXISTS analytics_internal.team_member_profiles ( user_id UUID, display_name Nullable(String), profile_image_url Nullable(String), - user JSON, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -373,7 +350,6 @@ SELECT user_id, display_name, profile_image_url, - user, created_at FROM analytics_internal.team_member_profiles FINAL @@ -386,7 +362,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( branch_id String, team_id UUID, user_id UUID, - permission_id String, + id String, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -394,7 +370,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.team_permissions ( ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) -ORDER BY (project_id, branch_id, team_id, user_id, permission_id); +ORDER BY (project_id, branch_id, team_id, user_id, id); `; const TEAM_PERMISSIONS_VIEW_SQL = ` @@ -406,7 +382,7 @@ SELECT branch_id, team_id, user_id, - permission_id, + id, created_at FROM analytics_internal.team_permissions FINAL @@ -462,18 +438,17 @@ CREATE TABLE IF NOT EXISTS analytics_internal.email_outboxes ( email_programmatic_call_template_id Nullable(String), theme_id Nullable(String), is_high_priority UInt8, - rendered_is_transactional Nullable(UInt8), - rendered_subject Nullable(String), - rendered_notification_category_id Nullable(String), + is_transactional Nullable(UInt8), + subject Nullable(String), + notification_category_id Nullable(String), started_rendering_at Nullable(DateTime64(3, 'UTC')), - finished_rendering_at Nullable(DateTime64(3, 'UTC')), + rendered_at Nullable(DateTime64(3, 'UTC')), render_error Nullable(String), scheduled_at DateTime64(3, 'UTC'), created_at DateTime64(3, 'UTC'), + updated_at DateTime64(3, 'UTC'), started_sending_at Nullable(DateTime64(3, 'UTC')), - finished_sending_at Nullable(DateTime64(3, 'UTC')), server_error Nullable(String), - sent_at Nullable(DateTime64(3, 'UTC')), delivered_at Nullable(DateTime64(3, 'UTC')), opened_at Nullable(DateTime64(3, 'UTC')), clicked_at Nullable(DateTime64(3, 'UTC')), @@ -510,18 +485,17 @@ SELECT email_programmatic_call_template_id, theme_id, is_high_priority, - rendered_is_transactional, - rendered_subject, - rendered_notification_category_id, + is_transactional, + subject, + notification_category_id, started_rendering_at, - finished_rendering_at, + rendered_at, render_error, scheduled_at, created_at, + updated_at, started_sending_at, - finished_sending_at, server_error, - sent_at, delivered_at, opened_at, clicked_at, @@ -539,43 +513,13 @@ FINAL WHERE sync_is_deleted = 0; `; -const SESSION_REPLAYS_TABLE_BASE_SQL = ` -CREATE TABLE IF NOT EXISTS analytics_internal.session_replays ( - project_id String, - branch_id String, - id UUID, - user_id UUID, - refresh_token_id String, - started_at DateTime64(3, 'UTC'), - last_event_at DateTime64(3, 'UTC'), - created_at DateTime64(3, 'UTC'), - chunk_count UInt64, - sync_sequence_id Int64, - sync_is_deleted UInt8, - sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) -) -ENGINE ReplacingMergeTree(sync_sequence_id) -PARTITION BY toYYYYMM(started_at) -ORDER BY (project_id, branch_id, id); -`; - -const SESSION_REPLAYS_VIEW_SQL = ` -CREATE OR REPLACE VIEW default.session_replays -SQL SECURITY DEFINER -AS -SELECT project_id, branch_id, id, user_id, refresh_token_id, - started_at, last_event_at, created_at, chunk_count -FROM analytics_internal.session_replays -FINAL -WHERE sync_is_deleted = 0; -`; const PROJECT_PERMISSIONS_TABLE_BASE_SQL = ` CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( project_id String, branch_id String, user_id UUID, - permission_id String, + id String, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -583,7 +527,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.project_permissions ( ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) -ORDER BY (project_id, branch_id, user_id, permission_id); +ORDER BY (project_id, branch_id, user_id, id); `; const PROJECT_PERMISSIONS_VIEW_SQL = ` @@ -594,7 +538,7 @@ SELECT project_id, branch_id, user_id, - permission_id, + id, created_at FROM analytics_internal.project_permissions FINAL @@ -605,7 +549,6 @@ const NOTIFICATION_PREFERENCES_TABLE_BASE_SQL = ` CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( project_id String, branch_id String, - id UUID, user_id UUID, notification_category_id String, enabled UInt8, @@ -614,7 +557,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) ) ENGINE ReplacingMergeTree(sync_sequence_id) -ORDER BY (project_id, branch_id, id); +ORDER BY (project_id, branch_id, user_id, notification_category_id); `; const NOTIFICATION_PREFERENCES_VIEW_SQL = ` @@ -624,7 +567,6 @@ AS SELECT project_id, branch_id, - id, user_id, notification_category_id, enabled @@ -674,11 +616,9 @@ const CONNECTED_ACCOUNTS_TABLE_BASE_SQL = ` CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( project_id String, branch_id String, - id UUID, user_id UUID, provider String, provider_account_id String, - email Nullable(String), created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -686,7 +626,7 @@ CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) -ORDER BY (project_id, branch_id, id); +ORDER BY (project_id, branch_id, user_id, provider, provider_account_id); `; const CONNECTED_ACCOUNTS_VIEW_SQL = ` @@ -696,11 +636,9 @@ AS SELECT project_id, branch_id, - id, user_id, provider, provider_account_id, - email, created_at FROM analytics_internal.connected_accounts FINAL diff --git a/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx b/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx index 9e04384779..19e5696a16 100644 --- a/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx +++ b/apps/backend/src/app/api/latest/emails/unsubscribe-link/route.tsx @@ -1,3 +1,4 @@ +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client"; import { VerificationCodeType } from "@/generated/prisma/client"; @@ -51,15 +52,15 @@ export async function GET(request: NextRequest) { notificationCategoryId: notification_category_id, }, }, - update: { + update: withExternalDbSyncUpdate({ enabled: false, - }, - create: { + }), + create: withExternalDbSyncUpdate({ tenancyId: tenancy.id, projectUserId: user_id, notificationCategoryId: notification_category_id, enabled: false, - }, + }), }); return new Response('

Successfully unsubscribed from notification group

', { diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index 470e89a55d..bae534572d 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -79,23 +79,6 @@ async function backfillSequenceIds(batchSize: number): Promise { if (projectUserTenants.length > 0) { await enqueueExternalDbSyncBatch(projectUserTenants.map(t => t.tenancyId)); didUpdate = true; - - // Cascade: when a user changes, mark their TeamMember rows for re-sync - // so the embedded user JSON in team_member_profiles stays fresh - await globalPrismaClient.$executeRaw` - UPDATE "TeamMember" - SET "shouldUpdateSequenceId" = TRUE - FROM ( - SELECT DISTINCT "tenancyId", "projectUserId" - FROM "ProjectUser" - WHERE "tenancyId" IN (${Prisma.join(projectUserTenants.map(t => t.tenancyId))}) - AND "shouldUpdateSequenceId" = FALSE - AND "sequenceId" IS NOT NULL - ) AS changed_users - WHERE "TeamMember"."tenancyId" = changed_users."tenancyId" - AND "TeamMember"."projectUserId" = changed_users."projectUserId" - AND "TeamMember"."shouldUpdateSequenceId" = FALSE - `; } const contactChannelTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts index f8de41b24f..48890173a4 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts @@ -87,7 +87,15 @@ const globalSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + teams: sequenceStatsSchema.defined(), + team_members: sequenceStatsSchema.defined(), + team_permissions: sequenceStatsSchema.defined(), + team_invitations: sequenceStatsSchema.defined(), email_outboxes: sequenceStatsSchema.defined(), + project_permissions: sequenceStatsSchema.defined(), + notification_preferences: sequenceStatsSchema.defined(), + refresh_tokens: sequenceStatsSchema.defined(), + connected_accounts: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -120,7 +128,15 @@ const responseSchema = yupObject({ sequencer: yupObject({ project_users: sequenceStatsSchema.defined(), contact_channels: sequenceStatsSchema.defined(), + teams: sequenceStatsSchema.defined(), + team_members: sequenceStatsSchema.defined(), + team_permissions: sequenceStatsSchema.defined(), + team_invitations: sequenceStatsSchema.defined(), email_outboxes: sequenceStatsSchema.defined(), + project_permissions: sequenceStatsSchema.defined(), + notification_preferences: sequenceStatsSchema.defined(), + refresh_tokens: sequenceStatsSchema.defined(), + connected_accounts: sequenceStatsSchema.defined(), deleted_rows: sequenceStatsSchema.shape({ by_table: yupArray(deletedRowByTableSchema).defined(), }).defined(), @@ -310,6 +326,52 @@ async function fetchInternalStats(tenancyId: string | null) { ${tenancyWhere} `).at(0) ?? throwErr("Contact channel stats query returned no rows."); + const teamStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "Team" + ${tenancyWhere} + `).at(0) ?? throwErr("Team stats query returned no rows."); + + const teamMemberStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "TeamMember" + ${tenancyWhere} + `).at(0) ?? throwErr("Team member stats query returned no rows."); + + const teamPermissionStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "TeamMemberDirectPermission" + ${tenancyWhere} + `).at(0) ?? throwErr("Team permission stats query returned no rows."); + + const teamInvitationStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "VerificationCode" + ${tenancyId + ? Prisma.sql`JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" AND "Tenancy"."branchId" = "VerificationCode"."branchId" WHERE "type" = 'TEAM_INVITATION' AND "Tenancy"."id" = ${tenancyId}::uuid` + : Prisma.sql`WHERE "type" = 'TEAM_INVITATION'`} + `).at(0) ?? throwErr("Team invitation stats query returned no rows."); + const emailOutboxStatsRow = (await globalPrismaClient.$queryRaw` SELECT COUNT(*)::bigint AS "total", @@ -321,6 +383,50 @@ async function fetchInternalStats(tenancyId: string | null) { ${tenancyWhere} `).at(0) ?? throwErr("Email outbox stats query returned no rows."); + const projectPermissionStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserDirectPermission" + ${tenancyWhere} + `).at(0) ?? throwErr("Project permission stats query returned no rows."); + + const notificationPreferenceStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "UserNotificationPreference" + ${tenancyWhere} + `).at(0) ?? throwErr("Notification preference stats query returned no rows."); + + const refreshTokenStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserRefreshToken" + ${tenancyWhere} + `).at(0) ?? throwErr("Refresh token stats query returned no rows."); + + const connectedAccountStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUserOAuthAccount" + ${tenancyWhere} + `).at(0) ?? throwErr("Connected account stats query returned no rows."); + const deletedRowStatsRow = (await globalPrismaClient.$queryRaw` SELECT COUNT(*)::bigint AS "total", @@ -367,7 +473,15 @@ async function fetchInternalStats(tenancyId: string | null) { const projectUsersStats = formatSequenceStats(projectUserStatsRow); const contactChannelStats = formatSequenceStats(contactChannelStatsRow); + const teamStats = formatSequenceStats(teamStatsRow); + const teamMemberStats = formatSequenceStats(teamMemberStatsRow); + const teamPermissionStats = formatSequenceStats(teamPermissionStatsRow); + const teamInvitationStats = formatSequenceStats(teamInvitationStatsRow); const emailOutboxStats = formatSequenceStats(emailOutboxStatsRow); + const projectPermissionStats = formatSequenceStats(projectPermissionStatsRow); + const notificationPreferenceStats = formatSequenceStats(notificationPreferenceStatsRow); + const refreshTokenStats = formatSequenceStats(refreshTokenStatsRow); + const connectedAccountStats = formatSequenceStats(connectedAccountStatsRow); const deletedRowStats = formatSequenceStats(deletedRowStatsRow); const deletedRowsByTable = deletedRowsByTableRows.map((row) => ({ @@ -380,7 +494,15 @@ async function fetchInternalStats(tenancyId: string | null) { return { projectUsersStats, contactChannelStats, + teamStats, + teamMemberStats, + teamPermissionStats, + teamInvitationStats, emailOutboxStats, + projectPermissionStats, + notificationPreferenceStats, + refreshTokenStats, + connectedAccountStats, deletedRowStats, deletedRowsByTable, outgoingStatsRow, @@ -1026,7 +1148,15 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: globalStats.projectUsersStats, contact_channels: globalStats.contactChannelStats, + teams: globalStats.teamStats, + team_members: globalStats.teamMemberStats, + team_permissions: globalStats.teamPermissionStats, + team_invitations: globalStats.teamInvitationStats, email_outboxes: globalStats.emailOutboxStats, + project_permissions: globalStats.projectPermissionStats, + notification_preferences: globalStats.notificationPreferenceStats, + refresh_tokens: globalStats.refreshTokenStats, + connected_accounts: globalStats.connectedAccountStats, deleted_rows: { ...globalStats.deletedRowStats, by_table: globalStats.deletedRowsByTable, @@ -1045,7 +1175,15 @@ export const GET = createSmartRouteHandler({ sequencer: { project_users: currentStats.projectUsersStats, contact_channels: currentStats.contactChannelStats, + teams: currentStats.teamStats, + team_members: currentStats.teamMemberStats, + team_permissions: currentStats.teamPermissionStats, + team_invitations: currentStats.teamInvitationStats, email_outboxes: currentStats.emailOutboxStats, + project_permissions: currentStats.projectPermissionStats, + notification_preferences: currentStats.notificationPreferenceStats, + refresh_tokens: currentStats.refreshTokenStats, + connected_accounts: currentStats.connectedAccountStats, deleted_rows: { ...currentStats.deletedRowStats, by_table: currentStats.deletedRowsByTable, diff --git a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx index e636e6b2c2..4a01639372 100644 --- a/apps/backend/src/app/api/latest/oauth-providers/crud.tsx +++ b/apps/backend/src/app/api/latest/oauth-providers/crud.tsx @@ -1,4 +1,4 @@ -import { recordExternalDbSyncDeletion } from "@/lib/external-db-sync"; +import { recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; @@ -310,10 +310,10 @@ export const oauthProviderCrudHandlers = createLazyProxy(() => createCrudHandler id: params.provider_id, }, }, - data: { + data: withExternalDbSyncUpdate({ email: data.email, providerAccountId: data.account_id, - }, + }), }); const providerType = resolveProviderType(auth.tenancy, existingOAuthAccount.configOAuthProviderId) diff --git a/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx b/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx index 1e39098801..552a49484d 100644 --- a/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx +++ b/apps/backend/src/app/api/latest/team-member-profiles/crud.tsx @@ -1,4 +1,5 @@ import { Prisma } from "@/generated/prisma/client"; +import { withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureTeamExists, ensureTeamMembershipExists, ensureUserExists, ensureUserTeamPermissionExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; @@ -146,10 +147,10 @@ export const teamMemberProfilesCrudHandlers = createLazyProxy(() => createCrudHa teamId: params.team_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name, profileImageUrl: await uploadAndGetUrl(data.profile_image_url, "team-member-profile-images") - }, + }), include: fullInclude, }); diff --git a/apps/backend/src/lib/email-queue-step.tsx b/apps/backend/src/lib/email-queue-step.tsx index 28002cb2ab..453c1f0bd9 100644 --- a/apps/backend/src/lib/email-queue-step.tsx +++ b/apps/backend/src/lib/email-queue-step.tsx @@ -242,7 +242,8 @@ async function claimEmailsForRendering(workerId: string): Promise UPDATE "EmailOutbox" AS e SET "renderedByWorkerId" = ${workerId}::uuid, - "startedRenderingAt" = NOW() + "startedRenderingAt" = NOW(), + "shouldUpdateSequenceId" = TRUE FROM selected WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" RETURNING e.*; @@ -531,7 +532,8 @@ async function claimEmailsForSending(tx: PrismaClientTransaction, tenancyId: str FOR UPDATE SKIP LOCKED ) UPDATE "EmailOutbox" AS e - SET "startedSendingAt" = NOW() + SET "startedSendingAt" = NOW(), + "shouldUpdateSequenceId" = TRUE FROM selected WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" RETURNING e.*; diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts index d276e79dc8..b3a3431d89 100644 --- a/apps/backend/src/lib/external-db-sync.ts +++ b/apps/backend/src/lib/external-db-sync.ts @@ -298,6 +298,7 @@ export async function recordExternalDbSyncDeletion( TRUE FROM "TeamMemberDirectPermission" WHERE "id" = ${target.permissionDbId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid FOR UPDATE `); @@ -335,6 +336,7 @@ export async function recordExternalDbSyncDeletion( TRUE FROM "ProjectUserDirectPermission" WHERE "id" = ${target.permissionDbId}::uuid + AND "tenancyId" = ${target.tenancyId}::uuid FOR UPDATE `); @@ -477,7 +479,8 @@ export async function recordExternalDbSyncDeletion( FROM "VerificationCode" JOIN "Tenancy" ON "Tenancy"."projectId" = "VerificationCode"."projectId" AND "Tenancy"."branchId" = "VerificationCode"."branchId" - WHERE "VerificationCode"."projectId" = ${target.verificationCodeProjectId} + WHERE "Tenancy"."id" = ${target.tenancyId}::uuid + AND "VerificationCode"."projectId" = ${target.verificationCodeProjectId} AND "VerificationCode"."branchId" = ${target.verificationCodeBranchId} AND "VerificationCode"."id" = ${target.verificationCodeId}::uuid AND "VerificationCode"."type" = 'TEAM_INVITATION' diff --git a/apps/backend/src/lib/permissions.tsx b/apps/backend/src/lib/permissions.tsx index 2bec329374..7429ead3a6 100644 --- a/apps/backend/src/lib/permissions.tsx +++ b/apps/backend/src/lib/permissions.tsx @@ -332,9 +332,9 @@ export async function updatePermissionDefinition( tenancyId: options.tenancy.id, permissionId: options.oldId, }, - data: { + data: withExternalDbSyncUpdate({ permissionId: newId, - }, + }), }); await sourceOfTruthTx.projectUserDirectPermission.updateMany({ @@ -342,9 +342,9 @@ export async function updatePermissionDefinition( tenancyId: options.tenancy.id, permissionId: options.oldId, }, - data: { + data: withExternalDbSyncUpdate({ permissionId: newId, - }, + }), }); return { diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx index c1e7d57951..02c2bd30a5 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx @@ -80,6 +80,15 @@ type ExternalDbSyncStatus = { sequencer: { project_users: SequenceStats, contact_channels: SequenceStats, + teams: SequenceStats, + team_members: SequenceStats, + team_permissions: SequenceStats, + team_invitations: SequenceStats, + email_outboxes: SequenceStats, + project_permissions: SequenceStats, + notification_preferences: SequenceStats, + refresh_tokens: SequenceStats, + connected_accounts: SequenceStats, deleted_rows: DeletedRowStats, }, poller: PollerStats, @@ -95,6 +104,15 @@ type ExternalDbSyncStatus = { sequencer: { project_users: SequenceStats, contact_channels: SequenceStats, + teams: SequenceStats, + team_members: SequenceStats, + team_permissions: SequenceStats, + team_invitations: SequenceStats, + email_outboxes: SequenceStats, + project_permissions: SequenceStats, + notification_preferences: SequenceStats, + refresh_tokens: SequenceStats, + connected_accounts: SequenceStats, deleted_rows: DeletedRowStats, }, poller: PollerStats, @@ -400,6 +418,15 @@ export default function PageClient() { const sequencerPending = sumBigIntStrings([ summarySource.sequencer.project_users.pending, summarySource.sequencer.contact_channels.pending, + summarySource.sequencer.teams.pending, + summarySource.sequencer.team_members.pending, + summarySource.sequencer.team_permissions.pending, + summarySource.sequencer.team_invitations.pending, + summarySource.sequencer.email_outboxes.pending, + summarySource.sequencer.project_permissions.pending, + summarySource.sequencer.notification_preferences.pending, + summarySource.sequencer.refresh_tokens.pending, + summarySource.sequencer.connected_accounts.pending, summarySource.sequencer.deleted_rows.pending, ]); const mappingPending = sumBigIntStrings( @@ -518,7 +545,7 @@ export default function PageClient() { -
ProjectUser + ContactChannel + DeletedRow rows waiting for sequence IDs.
+
All synced table rows waiting for sequence IDs.
Throughput {loadingState ? "—" : formatThroughput(throughputStats?.sequencer ?? null)} @@ -578,30 +605,29 @@ export default function PageClient() { - - ProjectUser - - - - - - - - ContactChannel - - - - - - - - DeletedRow - - - - - - + {([ + ["ProjectUser", status?.sequencer.project_users], + ["ContactChannel", status?.sequencer.contact_channels], + ["Team", status?.sequencer.teams], + ["TeamMember", status?.sequencer.team_members], + ["TeamPermission", status?.sequencer.team_permissions], + ["TeamInvitation", status?.sequencer.team_invitations], + ["EmailOutbox", status?.sequencer.email_outboxes], + ["ProjectPermission", status?.sequencer.project_permissions], + ["NotificationPref", status?.sequencer.notification_preferences], + ["RefreshToken", status?.sequencer.refresh_tokens], + ["ConnectedAccount", status?.sequencer.connected_accounts], + ["DeletedRow", status?.sequencer.deleted_rows], + ] as const).map(([name, stats]) => ( + + {name} + + + + + + + ))} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts index bfd0b16100..afcdc93243 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-query.test.ts @@ -529,7 +529,6 @@ it("has limited grants", async ({ expect }) => { { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.notification_preferences TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.project_permissions TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.refresh_tokens TO limited_user" }, - { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.session_replays TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" }, { "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_permissions TO limited_user" }, @@ -600,10 +599,6 @@ it("can see only some tables", async ({ expect }) => { "database": "default", "name": "refresh_tokens", }, - { - "database": "default", - "name": "session_replays", - }, { "database": "default", "name": "team_invitations", @@ -648,7 +643,6 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => { { "name": "notification_preferences" }, { "name": "project_permissions" }, { "name": "refresh_tokens" }, - { "name": "session_replays" }, { "name": "team_invitations" }, { "name": "team_member_profiles" }, { "name": "team_permissions" }, @@ -1141,7 +1135,6 @@ it("shows grants", async ({ expect }) => { { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.notification_preferences TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.project_permissions TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.refresh_tokens TO limited_user" }, - { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.session_replays TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" }, { "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_permissions TO limited_user" }, diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts index 752b685630..191c31068d 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -21,7 +21,7 @@ import { waitForSyncedEmailOutboxByStatus, waitForSyncedRefreshToken, waitForSyncedRefreshTokenDeletion, - waitForSyncedSessionReplay, + waitForSyncedTeam, waitForSyncedTeamDeletion, waitForSyncedTeamInvitation, @@ -948,7 +948,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { let response; while (performance.now() - start < timeoutMs) { response = await runQueryForCurrentProject({ - query: "SELECT team_id, user_id, permission_id FROM team_permissions WHERE permission_id = {perm:String}", + query: "SELECT team_id, user_id, id FROM team_permissions WHERE id = {perm:String}", params: { perm: '$read_members' }, }); expect(response.status).toBe(200); @@ -959,7 +959,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { } expect(response!.body.result.length).toBe(1); - expect(response!.body.result[0].permission_id).toBe('$read_members'); + expect(response!.body.result[0].id).toBe('$read_members'); }, TEST_TIMEOUT); /** @@ -1038,7 +1038,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { let response; while (performance.now() - start < timeoutMs) { response = await runQueryForCurrentProject({ - query: "SELECT user_id, permission_id FROM project_permissions WHERE permission_id = {perm:String}", + query: "SELECT user_id, id FROM project_permissions WHERE id = {perm:String}", params: { perm: 'ch_test_perm' }, }); expect(response.status).toBe(200); @@ -1049,7 +1049,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { } expect(response!.body.result.length).toBe(1); - expect(response!.body.result[0].permission_id).toBe('ch_test_perm'); + expect(response!.body.result[0].id).toBe('ch_test_perm'); }, TEST_TIMEOUT); /** @@ -1419,7 +1419,7 @@ describe.sequential('External DB Sync - Basic Tests', () => { expect(response!.body.result.length).toBeGreaterThanOrEqual(1); const row = response!.body.result[0]; - expect(row.created_with).toBe('PROGRAMMATIC_CALL'); + expect(row.created_with).toBe('programmatic-call'); }, TEST_TIMEOUT); /** @@ -1487,175 +1487,6 @@ describe.sequential('External DB Sync - Basic Tests', () => { expect(row.send_retries).toBe(0); }, TEST_TIMEOUT); - /** - * What it does: - * - Creates a project with analytics, signs in a user, uploads a session replay batch, - * and verifies the session replay row is synced to ClickHouse. - */ - test('SessionReplay sync (ClickHouse)', async ({ expect }) => { - await Project.createAndSwitch({ - config: { - magic_link_enabled: true, - }, - }); - await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } }); - await Auth.Otp.signIn(); - - const now = Date.now(); - const browserSessionId = randomUUID(); - const batchId = randomUUID(); - - const uploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { - method: "POST", - accessType: "client", - body: { - browser_session_id: browserSessionId, - session_replay_segment_id: randomUUID(), - batch_id: batchId, - started_at_ms: now, - sent_at_ms: now + 500, - events: [ - { timestamp: now + 100, type: 2 }, - { timestamp: now + 200, type: 3 }, - ], - }, - }); - expect(uploadRes.status).toBe(200); - expect(uploadRes.body.deduped).toBe(false); - const replayId = uploadRes.body.session_replay_id; - - const secondUploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { - method: "POST", - accessType: "client", - body: { - browser_session_id: browserSessionId, - session_replay_segment_id: randomUUID(), - batch_id: randomUUID(), - started_at_ms: now + 1_000, - sent_at_ms: now + 1_500, - events: [ - { timestamp: now + 1_100, type: 2 }, - ], - }, - }); - expect(secondUploadRes.status).toBe(200); - expect(secondUploadRes.body.session_replay_id).toBe(replayId); - - await InternalApiKey.createAndSetProjectKeys(); - - // Poll ClickHouse until the session_replays row appears - const timeoutMs = 180_000; - const intervalMs = 2_000; - const start = performance.now(); - - let response; - while (performance.now() - start < timeoutMs) { - response = await runQueryForCurrentProject({ - query: "SELECT id, user_id, refresh_token_id, started_at, last_event_at, chunk_count FROM session_replays LIMIT 10", - }); - expect(response.status).toBe(200); - const syncedRow = response.body.result.find((resultRow: Record) => resultRow.id === replayId); - if (syncedRow && Number(syncedRow.chunk_count) === 2) { - break; - } - await wait(intervalMs); - } - - const row = response!.body.result.find((resultRow: Record) => resultRow.id === replayId); - expect(row).toBeDefined(); - if (!row) { - throw new Error("Expected synced ClickHouse session replay row to be present."); - } - expect(row.id).toBe(replayId); - expect(row.user_id).toBeDefined(); - expect(row.refresh_token_id).toBeDefined(); - expect(row.started_at).toBeDefined(); - expect(row.last_event_at).toBeDefined(); - expect(Number(row.chunk_count)).toBe(2); - }, TEST_TIMEOUT); - - /** - * What it does: - * - Creates a project with an external Postgres DB, signs in a user, - * uploads a session replay batch, and verifies the row is synced to external Postgres. - */ - test('SessionReplay sync (Postgres)', async () => { - const dbName = 'session_replay_pg_test'; - const connectionString = await dbManager.createDatabase(dbName); - - await createProjectWithExternalDb({ - main: { - type: 'postgres', - connectionString, - } - }, { - display_name: 'Session Replay Sync Test', - config: { - magic_link_enabled: true, - }, - }); - await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } }); - await Auth.Otp.signIn(); - - const now = Date.now(); - const browserSessionId = randomUUID(); - const batchId = randomUUID(); - - const uploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { - method: "POST", - accessType: "client", - body: { - browser_session_id: browserSessionId, - session_replay_segment_id: randomUUID(), - batch_id: batchId, - started_at_ms: now, - sent_at_ms: now + 500, - events: [ - { timestamp: now + 100, type: 2 }, - { timestamp: now + 200, type: 3 }, - ], - }, - }); - expect(uploadRes.status).toBe(200); - const replayId = uploadRes.body.session_replay_id; - - const client = dbManager.getClient(dbName); - - // Wait for the session replay row to appear in external DB - const secondUploadRes = await niceBackendFetch("/api/v1/session-replays/batch", { - method: "POST", - accessType: "client", - body: { - browser_session_id: browserSessionId, - session_replay_segment_id: randomUUID(), - batch_id: randomUUID(), - started_at_ms: now + 1_000, - sent_at_ms: now + 1_500, - events: [ - { timestamp: now + 1_100, type: 2 }, - ], - }, - }); - expect(secondUploadRes.status).toBe(200); - expect(secondUploadRes.body.session_replay_id).toBe(replayId); - - await waitForSyncedSessionReplay(client, replayId, 2); - - // Verify the synced row has expected columns - const res = await client.query(`SELECT * FROM "session_replays" WHERE "id" = $1`, [replayId]); - expect(res.rows.length).toBe(1); - const row = res.rows[0]; - expect(row.user_id).toBeDefined(); - expect(row.refresh_token_id).toBeDefined(); - expect(row.created_at).toBeInstanceOf(Date); - expect(row.started_at).toBeInstanceOf(Date); - expect(row.last_event_at).toBeInstanceOf(Date); - expect(row).toMatchObject({ - id: replayId, - chunk_count: "2", - }); - }, TEST_TIMEOUT); - /** * What it does: * - Reads the external DB sync fusebox settings. @@ -1874,17 +1705,15 @@ describe.sequential('External DB Sync - Basic Tests', () => { let response; while (performance.now() - start < timeoutMs) { response = await runQueryForCurrentProject({ - query: "SELECT id, user_id, provider, provider_account_id, email FROM connected_accounts WHERE id = {account_id:UUID}", - params: { account_id: accountId }, + query: "SELECT user_id, provider, provider_account_id FROM connected_accounts WHERE provider_account_id = {account_id:String} AND user_id = {user_id:UUID}", + params: { account_id: "ch-test-account-12345", user_id: userId }, }); expect(response.status).toBe(200); if (response.body.result.length === 1) { expect(response.body.result[0]).toMatchObject({ - id: accountId, user_id: userId, provider: "spotify", provider_account_id: "ch-test-account-12345", - email: "chuser@example.com", }); return; } diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts index 88de1db9f1..bbca0df6b5 100644 --- a/packages/stack-shared/src/config/db-sync-mappings.ts +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -664,7 +664,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { }, }, "team_member_profiles": { - sourceTables: { "TeamMember": "TeamMember", "ProjectUser": "ProjectUser" }, + sourceTables: { "TeamMember": "TeamMember" }, targetTable: "team_member_profiles", targetTableSchemas: { postgres: ` @@ -673,7 +673,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "user_id" uuid NOT NULL, "display_name" text, "profile_image_url" text, - "user" jsonb NOT NULL DEFAULT '{}'::jsonb, "created_at" timestamp without time zone NOT NULL, PRIMARY KEY ("team_id", "user_id") ); @@ -694,7 +693,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { user_id UUID, display_name Nullable(String), profile_image_url Nullable(String), - user JSON, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -716,46 +714,12 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "TeamMember"."projectUserId" AS "user_id", "TeamMember"."displayName" AS "display_name", "TeamMember"."profileImageUrl" AS "profile_image_url", - jsonb_build_object( - 'id', "ProjectUser"."projectUserId", - 'display_name', "ProjectUser"."displayName", - 'primary_email', ( - SELECT "ContactChannel"."value" - FROM "ContactChannel" - WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" - AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" - AND "ContactChannel"."type" = 'EMAIL' - AND "ContactChannel"."isPrimary" = 'TRUE' - LIMIT 1 - ), - 'primary_email_verified', COALESCE( - ( - SELECT "ContactChannel"."isVerified" - FROM "ContactChannel" - WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" - AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" - AND "ContactChannel"."type" = 'EMAIL' - AND "ContactChannel"."isPrimary" = 'TRUE' - LIMIT 1 - ), - false - ), - 'profile_image_url', "ProjectUser"."profileImageUrl", - 'signed_up_at_millis', EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000, - 'client_metadata', COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb), - 'client_read_only_metadata', COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb), - 'server_metadata', COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb), - 'is_anonymous', "ProjectUser"."isAnonymous", - 'last_active_at_millis', CASE WHEN "ProjectUser"."lastActiveAt" IS NOT NULL THEN EXTRACT(EPOCH FROM "ProjectUser"."lastActiveAt") * 1000 ELSE NULL END - ) AS "user", "TeamMember"."createdAt" AS "created_at", "TeamMember"."sequenceId" AS "sync_sequence_id", "TeamMember"."tenancyId" AS "tenancyId", false AS "sync_is_deleted" FROM "TeamMember" JOIN "Tenancy" ON "Tenancy"."id" = "TeamMember"."tenancyId" - JOIN "ProjectUser" ON "ProjectUser"."projectUserId" = "TeamMember"."projectUserId" - AND "ProjectUser"."tenancyId" = "TeamMember"."tenancyId" WHERE "TeamMember"."tenancyId" = $1::uuid UNION ALL @@ -767,7 +731,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", NULL::text AS "display_name", NULL::text AS "profile_image_url", - '{}'::jsonb AS "user", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sync_sequence_id", "DeletedRow"."tenancyId" AS "tenancyId", @@ -792,45 +755,11 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "TeamMember"."projectUserId" AS "user_id", "TeamMember"."displayName" AS "display_name", "TeamMember"."profileImageUrl" AS "profile_image_url", - jsonb_build_object( - 'id', "ProjectUser"."projectUserId", - 'display_name', "ProjectUser"."displayName", - 'primary_email', ( - SELECT "ContactChannel"."value" - FROM "ContactChannel" - WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" - AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" - AND "ContactChannel"."type" = 'EMAIL' - AND "ContactChannel"."isPrimary" = 'TRUE' - LIMIT 1 - ), - 'primary_email_verified', COALESCE( - ( - SELECT "ContactChannel"."isVerified" - FROM "ContactChannel" - WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" - AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" - AND "ContactChannel"."type" = 'EMAIL' - AND "ContactChannel"."isPrimary" = 'TRUE' - LIMIT 1 - ), - false - ), - 'profile_image_url', "ProjectUser"."profileImageUrl", - 'signed_up_at_millis', EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000, - 'client_metadata', COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb), - 'client_read_only_metadata', COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb), - 'server_metadata', COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb), - 'is_anonymous', "ProjectUser"."isAnonymous", - 'last_active_at_millis', CASE WHEN "ProjectUser"."lastActiveAt" IS NOT NULL THEN EXTRACT(EPOCH FROM "ProjectUser"."createdAt") * 1000 ELSE NULL END - ) AS "user", "TeamMember"."createdAt" AS "created_at", "TeamMember"."sequenceId" AS "sequence_id", "TeamMember"."tenancyId", false AS "is_deleted" FROM "TeamMember" - JOIN "ProjectUser" ON "ProjectUser"."projectUserId" = "TeamMember"."projectUserId" - AND "ProjectUser"."tenancyId" = "TeamMember"."tenancyId" WHERE "TeamMember"."tenancyId" = $1::uuid UNION ALL @@ -840,7 +769,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", NULL::text AS "display_name", NULL::text AS "profile_image_url", - '{}'::jsonb AS "user", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sequence_id", "DeletedRow"."tenancyId", @@ -863,11 +791,10 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { $2::uuid AS "user_id", $3::text AS "display_name", $4::text AS "profile_image_url", - $5::jsonb AS "user", - $6::timestamp without time zone AS "created_at", - $7::bigint AS "sequence_id", - $8::boolean AS "is_deleted", - $9::text AS "mapping_name" + $5::timestamp without time zone AS "created_at", + $6::bigint AS "sequence_id", + $7::boolean AS "is_deleted", + $8::text AS "mapping_name" ), deleted AS ( DELETE FROM "team_member_profiles" tm @@ -881,7 +808,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "user_id", "display_name", "profile_image_url", - "user", "created_at" ) SELECT @@ -889,14 +815,12 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { p."user_id", p."display_name", p."profile_image_url", - p."user", p."created_at" FROM params p WHERE p."is_deleted" = false ON CONFLICT ("team_id", "user_id") DO UPDATE SET "display_name" = EXCLUDED."display_name", "profile_image_url" = EXCLUDED."profile_image_url", - "user" = EXCLUDED."user", "created_at" = EXCLUDED."created_at" RETURNING 1 ) @@ -935,7 +859,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { branch_id String, team_id UUID, user_id UUID, - permission_id String, + id String, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -943,7 +867,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) - ORDER BY (project_id, branch_id, team_id, user_id, permission_id); + ORDER BY (project_id, branch_id, team_id, user_id, id); `.trim(), }, internalDbFetchQueries: { @@ -955,7 +879,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "Tenancy"."branchId" AS "branch_id", "TeamMemberDirectPermission"."teamId" AS "team_id", "TeamMemberDirectPermission"."projectUserId" AS "user_id", - "TeamMemberDirectPermission"."permissionId" AS "permission_id", + "TeamMemberDirectPermission"."permissionId" AS "id", "TeamMemberDirectPermission"."createdAt" AS "created_at", "TeamMemberDirectPermission"."sequenceId" AS "sync_sequence_id", "TeamMemberDirectPermission"."tenancyId" AS "tenancyId", @@ -971,7 +895,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "Tenancy"."branchId" AS "branch_id", ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", - "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sync_sequence_id", "DeletedRow"."tenancyId" AS "tenancyId", @@ -1007,7 +931,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT ("DeletedRow"."primaryKey"->>'teamId')::uuid AS "team_id", ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", - "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sequence_id", "DeletedRow"."tenancyId", @@ -1314,18 +1238,17 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { email_programmatic_call_template_id Nullable(String), theme_id Nullable(String), is_high_priority UInt8, - rendered_is_transactional Nullable(UInt8), - rendered_subject Nullable(String), - rendered_notification_category_id Nullable(String), + is_transactional Nullable(UInt8), + subject Nullable(String), + notification_category_id Nullable(String), started_rendering_at Nullable(DateTime64(3, 'UTC')), - finished_rendering_at Nullable(DateTime64(3, 'UTC')), + rendered_at Nullable(DateTime64(3, 'UTC')), render_error Nullable(String), scheduled_at DateTime64(3, 'UTC'), created_at DateTime64(3, 'UTC'), + updated_at DateTime64(3, 'UTC'), started_sending_at Nullable(DateTime64(3, 'UTC')), - finished_sending_at Nullable(DateTime64(3, 'UTC')), server_error Nullable(String), - sent_at Nullable(DateTime64(3, 'UTC')), delivered_at Nullable(DateTime64(3, 'UTC')), opened_at Nullable(DateTime64(3, 'UTC')), clicked_at Nullable(DateTime64(3, 'UTC')), @@ -1353,25 +1276,24 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", "EmailOutbox"."id" AS "id", - "EmailOutbox"."status"::text AS "status", - "EmailOutbox"."simpleStatus"::text AS "simple_status", - "EmailOutbox"."createdWith"::text AS "created_with", + LOWER(REPLACE("EmailOutbox"."status"::text, '_', '-')) AS "status", + LOWER(REPLACE("EmailOutbox"."simpleStatus"::text, '_', '-')) AS "simple_status", + CASE WHEN "EmailOutbox"."createdWith"::text = 'DRAFT' THEN 'draft' ELSE 'programmatic-call' END AS "created_with", "EmailOutbox"."emailDraftId" AS "email_draft_id", "EmailOutbox"."emailProgrammaticCallTemplateId" AS "email_programmatic_call_template_id", "EmailOutbox"."themeId" AS "theme_id", "EmailOutbox"."isHighPriority" AS "is_high_priority", - "EmailOutbox"."renderedIsTransactional" AS "rendered_is_transactional", - "EmailOutbox"."renderedSubject" AS "rendered_subject", - "EmailOutbox"."renderedNotificationCategoryId" AS "rendered_notification_category_id", + "EmailOutbox"."renderedIsTransactional" AS "is_transactional", + "EmailOutbox"."renderedSubject" AS "subject", + "EmailOutbox"."renderedNotificationCategoryId" AS "notification_category_id", "EmailOutbox"."startedRenderingAt" AS "started_rendering_at", - "EmailOutbox"."finishedRenderingAt" AS "finished_rendering_at", + "EmailOutbox"."finishedRenderingAt" AS "rendered_at", "EmailOutbox"."renderErrorExternalMessage" AS "render_error", "EmailOutbox"."scheduledAt" AS "scheduled_at", "EmailOutbox"."createdAt" AS "created_at", + "EmailOutbox"."updatedAt" AS "updated_at", "EmailOutbox"."startedSendingAt" AS "started_sending_at", - "EmailOutbox"."finishedSendingAt" AS "finished_sending_at", "EmailOutbox"."sendServerErrorExternalMessage" AS "server_error", - "EmailOutbox"."sentAt" AS "sent_at", "EmailOutbox"."deliveredAt" AS "delivered_at", "EmailOutbox"."openedAt" AS "opened_at", "EmailOutbox"."clickedAt" AS "clicked_at", @@ -1380,7 +1302,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "EmailOutbox"."bouncedAt" AS "bounced_at", "EmailOutbox"."deliveryDelayedAt" AS "delivery_delayed_at", "EmailOutbox"."canHaveDeliveryInfo" AS "can_have_delivery_info", - "EmailOutbox"."skippedReason"::text AS "skipped_reason", + LOWER(REPLACE("EmailOutbox"."skippedReason"::text, '_', '-')) AS "skipped_reason", "EmailOutbox"."skippedDetails" AS "skipped_details", "EmailOutbox"."sendRetries" AS "send_retries", "EmailOutbox"."isPaused" AS "is_paused", @@ -1598,160 +1520,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { `.trim(), }, }, - "session_replays": { - sourceTables: { "SessionReplay": "SessionReplay" }, - targetTable: "session_replays", - targetTableSchemas: { - postgres: ` - CREATE TABLE IF NOT EXISTS "session_replays" ( - "id" uuid PRIMARY KEY NOT NULL, - "user_id" uuid NOT NULL, - "refresh_token_id" text NOT NULL, - "started_at" timestamp without time zone NOT NULL, - "last_event_at" timestamp without time zone NOT NULL, - "created_at" timestamp without time zone NOT NULL, - "chunk_count" bigint NOT NULL DEFAULT 0 - ); - REVOKE ALL ON "session_replays" FROM PUBLIC; - GRANT SELECT ON "session_replays" TO PUBLIC; - - CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( - "mapping_name" text PRIMARY KEY NOT NULL, - "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, - "updated_at" timestamp without time zone NOT NULL DEFAULT now() - ); - `.trim(), - clickhouse: ` - CREATE TABLE IF NOT EXISTS analytics_internal.session_replays ( - project_id String, - branch_id String, - id UUID, - user_id UUID, - refresh_token_id String, - started_at DateTime64(3, 'UTC'), - last_event_at DateTime64(3, 'UTC'), - created_at DateTime64(3, 'UTC'), - chunk_count UInt64, - sync_sequence_id Int64, - sync_is_deleted UInt8, - sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) - ) - ENGINE ReplacingMergeTree(sync_sequence_id) - PARTITION BY toYYYYMM(started_at) - ORDER BY (project_id, branch_id, id); - `.trim(), - }, - internalDbFetchQueries: { - clickhouse: ` - SELECT - "Tenancy"."projectId" AS "project_id", - "Tenancy"."branchId" AS "branch_id", - "SessionReplay"."id" AS "id", - "SessionReplay"."projectUserId" AS "user_id", - "SessionReplay"."refreshTokenId" AS "refresh_token_id", - "SessionReplay"."startedAt" AS "started_at", - "SessionReplay"."lastEventAt" AS "last_event_at", - "SessionReplay"."createdAt" AS "created_at", - ( - SELECT COUNT(*) - FROM "SessionReplayChunk" - WHERE "SessionReplayChunk"."tenancyId" = "SessionReplay"."tenancyId" - AND "SessionReplayChunk"."sessionReplayId" = "SessionReplay"."id" - ) AS "chunk_count", - "SessionReplay"."sequenceId" AS "sync_sequence_id", - "SessionReplay"."tenancyId" AS "tenancyId", - false AS "sync_is_deleted" - FROM "SessionReplay" - JOIN "Tenancy" ON "Tenancy"."id" = "SessionReplay"."tenancyId" - WHERE "SessionReplay"."tenancyId" = $1::uuid - AND "SessionReplay"."sequenceId" IS NOT NULL - AND "SessionReplay"."sequenceId" > $2::bigint - ORDER BY "SessionReplay"."sequenceId" ASC - LIMIT 1000 - `.trim(), - }, - internalDbFetchQuery: ` - SELECT - "SessionReplay"."id" AS "id", - "SessionReplay"."projectUserId" AS "user_id", - "SessionReplay"."refreshTokenId" AS "refresh_token_id", - "SessionReplay"."startedAt" AS "started_at", - "SessionReplay"."lastEventAt" AS "last_event_at", - "SessionReplay"."createdAt" AS "created_at", - ( - SELECT COUNT(*) - FROM "SessionReplayChunk" - WHERE "SessionReplayChunk"."tenancyId" = "SessionReplay"."tenancyId" - AND "SessionReplayChunk"."sessionReplayId" = "SessionReplay"."id" - ) AS "chunk_count", - "SessionReplay"."sequenceId" AS "sequence_id", - "SessionReplay"."tenancyId", - false AS "is_deleted" - FROM "SessionReplay" - WHERE "SessionReplay"."tenancyId" = $1::uuid - AND "SessionReplay"."sequenceId" IS NOT NULL - AND "SessionReplay"."sequenceId" > $2::bigint - ORDER BY "SessionReplay"."sequenceId" ASC - LIMIT 1000 - `.trim(), - externalDbUpdateQueries: { - postgres: ` - WITH params AS ( - SELECT - $1::uuid AS "id", - $2::uuid AS "user_id", - $3::text AS "refresh_token_id", - $4::timestamp without time zone AS "started_at", - $5::timestamp without time zone AS "last_event_at", - $6::timestamp without time zone AS "created_at", - $7::bigint AS "chunk_count", - $8::bigint AS "sequence_id", - $9::boolean AS "is_deleted", - $10::text AS "mapping_name" - ), - deleted AS ( - DELETE FROM "session_replays" sr - USING params p - WHERE p."is_deleted" = true AND sr."id" = p."id" - RETURNING 1 - ), - upserted AS ( - INSERT INTO "session_replays" ( - "id", - "user_id", - "refresh_token_id", - "started_at", - "last_event_at", - "created_at", - "chunk_count" - ) - SELECT - p."id", - p."user_id", - p."refresh_token_id", - p."started_at", - p."last_event_at", - p."created_at", - p."chunk_count" - FROM params p - WHERE p."is_deleted" = false - ON CONFLICT ("id") DO UPDATE SET - "user_id" = EXCLUDED."user_id", - "refresh_token_id" = EXCLUDED."refresh_token_id", - "started_at" = EXCLUDED."started_at", - "last_event_at" = EXCLUDED."last_event_at", - "created_at" = EXCLUDED."created_at", - "chunk_count" = EXCLUDED."chunk_count" - RETURNING 1 - ) - INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") - SELECT p."mapping_name", p."sequence_id", now() FROM params p - ON CONFLICT ("mapping_name") DO UPDATE SET - "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), - "updated_at" = now(); - `.trim(), - }, - }, "project_permissions": { sourceTables: { "ProjectUserDirectPermission": "ProjectUserDirectPermission" }, targetTable: "project_permissions", @@ -1777,7 +1545,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { project_id String, branch_id String, user_id UUID, - permission_id String, + id String, created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -1785,7 +1553,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) - ORDER BY (project_id, branch_id, user_id, permission_id); + ORDER BY (project_id, branch_id, user_id, id); `.trim(), }, internalDbFetchQueries: { @@ -1796,7 +1564,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", "ProjectUserDirectPermission"."projectUserId" AS "user_id", - "ProjectUserDirectPermission"."permissionId" AS "permission_id", + "ProjectUserDirectPermission"."permissionId" AS "id", "ProjectUserDirectPermission"."createdAt" AS "created_at", "ProjectUserDirectPermission"."sequenceId" AS "sync_sequence_id", "ProjectUserDirectPermission"."tenancyId" AS "tenancyId", @@ -1811,7 +1579,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", - "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sync_sequence_id", "DeletedRow"."tenancyId" AS "tenancyId", @@ -1845,7 +1613,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "user_id", - "DeletedRow"."primaryKey"->>'permissionId' AS "permission_id", + "DeletedRow"."primaryKey"->>'permissionId' AS "id", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sequence_id", "DeletedRow"."tenancyId", @@ -1925,7 +1693,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { CREATE TABLE IF NOT EXISTS analytics_internal.notification_preferences ( project_id String, branch_id String, - id UUID, user_id UUID, notification_category_id String, enabled UInt8, @@ -1934,7 +1701,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3) ) ENGINE ReplacingMergeTree(sync_sequence_id) - ORDER BY (project_id, branch_id, id); + ORDER BY (project_id, branch_id, user_id, notification_category_id); `.trim(), }, internalDbFetchQueries: { @@ -1944,7 +1711,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", - "UserNotificationPreference"."id" AS "id", "UserNotificationPreference"."projectUserId" AS "user_id", "UserNotificationPreference"."notificationCategoryId" AS "notification_category_id", "UserNotificationPreference"."enabled" AS "enabled", @@ -1960,7 +1726,6 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", - ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", ("DeletedRow"."data"->>'notificationCategoryId')::uuid AS "notification_category_id", ("DeletedRow"."data"->>'enabled')::boolean AS "enabled", @@ -2273,11 +2038,9 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { CREATE TABLE IF NOT EXISTS analytics_internal.connected_accounts ( project_id String, branch_id String, - id UUID, user_id UUID, provider String, provider_account_id String, - email Nullable(String), created_at DateTime64(3, 'UTC'), sync_sequence_id Int64, sync_is_deleted UInt8, @@ -2285,7 +2048,7 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { ) ENGINE ReplacingMergeTree(sync_sequence_id) PARTITION BY toYYYYMM(created_at) - ORDER BY (project_id, branch_id, id); + ORDER BY (project_id, branch_id, user_id, provider, provider_account_id); CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata ( tenancy_id UUID, @@ -2304,11 +2067,9 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", - "ProjectUserOAuthAccount"."id" AS "id", "ProjectUserOAuthAccount"."projectUserId" AS "user_id", "ProjectUserOAuthAccount"."configOAuthProviderId" AS "provider", "ProjectUserOAuthAccount"."providerAccountId" AS "provider_account_id", - "ProjectUserOAuthAccount"."email" AS "email", "ProjectUserOAuthAccount"."createdAt" AS "created_at", "ProjectUserOAuthAccount"."sequenceId" AS "sync_sequence_id", "ProjectUserOAuthAccount"."tenancyId" AS "tenancyId", @@ -2323,11 +2084,9 @@ export const DEFAULT_DB_SYNC_MAPPINGS = { SELECT "Tenancy"."projectId" AS "project_id", "Tenancy"."branchId" AS "branch_id", - ("DeletedRow"."primaryKey"->>'id')::uuid AS "id", ("DeletedRow"."data"->>'projectUserId')::uuid AS "user_id", - NULL::text AS "provider", - NULL::text AS "provider_account_id", - NULL::text AS "email", + "DeletedRow"."data"->>'configOAuthProviderId' AS "provider", + "DeletedRow"."data"->>'providerAccountId' AS "provider_account_id", "DeletedRow"."deletedAt"::timestamp without time zone AS "created_at", "DeletedRow"."sequenceId" AS "sync_sequence_id", "DeletedRow"."tenancyId" AS "tenancyId",