From b3e71b0fe98c4820aafb4a75fc909992fe771b85 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Fri, 6 Feb 2026 14:51:58 -0800 Subject: [PATCH 01/11] Reduce test flakeyness --- .github/workflows/e2e-api-tests.yaml | 3 +- .../e2e-custom-base-port-api-tests.yaml | 1 - .../e2e-source-of-truth-api-tests.yaml | 1 - ...rt-dev-and-test-with-custom-base-port.yaml | 1 - .github/workflows/restart-dev-and-test.yaml | 1 - .../setup-tests-with-custom-base-port.yaml | 1 - .github/workflows/setup-tests.yaml | 1 - .../internal/external-db-sync/poller/route.ts | 16 +-- .../external-db-sync/sequencer/route.ts | 19 +--- apps/e2e/tests/backend/backend-helpers.ts | 36 ++++++- .../endpoints/api/v1/emails/email-helpers.ts | 19 +--- .../api/v1/emails/email-queue.test.ts | 41 +------ .../api/v1/emails/outbox-api.test.ts | 102 ++++-------------- .../api/v1/external-db-sync-race.test.ts | 36 +++---- .../api/v1/external-db-sync-utils.ts | 53 --------- .../backend/endpoints/api/v1/users.test.ts | 2 +- 16 files changed, 85 insertions(+), 248 deletions(-) diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml index 6aa1e55e2d..3d33c11066 100644 --- a/.github/workflows/e2e-api-tests.yaml +++ b/.github/workflows/e2e-api-tests.yaml @@ -19,7 +19,6 @@ jobs: NODE_ENV: test STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" @@ -160,7 +159,7 @@ jobs: run: sleep 10 - name: Run tests - run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} ${{ matrix.freestyle-mode == 'prod' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && 'mail' || '' }} - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml index 1a1cf52efe..d458b0653d 100644 --- a/.github/workflows/e2e-custom-base-port-api-tests.yaml +++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml @@ -19,7 +19,6 @@ jobs: STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe" NEXT_PUBLIC_STACK_PORT_PREFIX: "67" - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" diff --git a/.github/workflows/e2e-source-of-truth-api-tests.yaml b/.github/workflows/e2e-source-of-truth-api-tests.yaml index 99b66e51b6..46f5cebfc5 100644 --- a/.github/workflows/e2e-source-of-truth-api-tests.yaml +++ b/.github/workflows/e2e-source-of-truth-api-tests.yaml @@ -21,7 +21,6 @@ jobs: STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}' STACK_TEST_SOURCE_OF_TRUTH: true STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" diff --git a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml index 65179046fb..75ccb222e3 100644 --- a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml +++ b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml @@ -19,7 +19,6 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" diff --git a/.github/workflows/restart-dev-and-test.yaml b/.github/workflows/restart-dev-and-test.yaml index 6d091edb3c..e9e133aa35 100644 --- a/.github/workflows/restart-dev-and-test.yaml +++ b/.github/workflows/restart-dev-and-test.yaml @@ -18,7 +18,6 @@ jobs: restart-dev-and-test: runs-on: ubicloud-standard-16 env: - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" diff --git a/.github/workflows/setup-tests-with-custom-base-port.yaml b/.github/workflows/setup-tests-with-custom-base-port.yaml index 0c9ac4a6ca..95a04f2896 100644 --- a/.github/workflows/setup-tests-with-custom-base-port.yaml +++ b/.github/workflows/setup-tests-with-custom-base-port.yaml @@ -19,7 +19,6 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" diff --git a/.github/workflows/setup-tests.yaml b/.github/workflows/setup-tests.yaml index 66374120c5..fe81c28ecc 100644 --- a/.github/workflows/setup-tests.yaml +++ b/.github/workflows/setup-tests.yaml @@ -18,7 +18,6 @@ jobs: setup-tests: runs-on: ubicloud-standard-16 env: - STACK_FORCE_EXTERNAL_DB_SYNC: "true" STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts index a12edab49f..06b3a09d78 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts @@ -30,13 +30,6 @@ function parseMaxDurationMs(value: string | undefined): number { return parsed; } -function parseStopWhenIdle(value: string | undefined): boolean { - if (!value) return false; - if (value === "true") return true; - if (value === "false") return false; - throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); -} - function directSyncEnabled(): boolean { return getEnvVariable(DIRECT_SYNC_ENV, "") === "true"; } @@ -69,7 +62,6 @@ export const GET = createSmartRouteHandler({ }).defined(), query: yupObject({ maxDurationMs: yupString().optional(), - stopWhenIdle: yupString().optional(), }).defined(), }), response: yupObject({ @@ -91,13 +83,11 @@ export const GET = createSmartRouteHandler({ return await traceSpan("external-db-sync.poller", async (span) => { const startTime = performance.now(); const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); - const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); const pollIntervalMs = 50; const staleClaimIntervalMinutes = 5; const pollerClaimLimit = getPollerClaimLimit(); span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); - span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); span.setAttribute("stack.external-db-sync.poller-claim-limit", pollerClaimLimit); span.setAttribute("stack.external-db-sync.direct-sync", directSyncEnabled()); @@ -235,7 +225,7 @@ export const GET = createSmartRouteHandler({ } type PollerIterationResult = { - stopReason: "disabled" | "idle" | null, + stopReason: "disabled" | null, processed: number, }; @@ -255,8 +245,8 @@ export const GET = createSmartRouteHandler({ const pendingRequests = await claimPendingRequests(); iterationSpan.setAttribute("stack.external-db-sync.pending-count", pendingRequests.length); - if (stopWhenIdle && pendingRequests.length === 0) { - return { stopReason: "idle", processed: 0 }; + if (pendingRequests.length > 0) { + console.log(`[Poller] Processing ${pendingRequests.length} pending requests`); } const processed = await processRequests(pendingRequests); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts index 7a51da1f27..a114299147 100644 --- a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -27,13 +27,6 @@ function parseMaxDurationMs(value: string | undefined): number { return parsed; } -function parseStopWhenIdle(value: string | undefined): boolean { - if (!value) return false; - if (value === "true") return true; - if (value === "false") return false; - throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); -} - function getSequencerBatchSize(): number { const rawValue = getEnvVariable(SEQUENCER_BATCH_SIZE_ENV, ""); if (!rawValue) return DEFAULT_BATCH_SIZE; @@ -145,6 +138,10 @@ async function backfillSequenceIds(batchSize: number): Promise { span.setAttribute("stack.external-db-sync.did-update", didUpdate); + if (didUpdate) { + console.log(`[Sequencer] Backfilled USR=${projectUserTenants.length} CC=${contactChannelTenants.length} DEL=${deletedRowTenants.length}`); + } + return didUpdate; }); } @@ -169,7 +166,6 @@ export const GET = createSmartRouteHandler({ }).defined(), query: yupObject({ maxDurationMs: yupString().optional(), - stopWhenIdle: yupString().optional(), }).defined(), }), response: yupObject({ @@ -190,19 +186,17 @@ export const GET = createSmartRouteHandler({ return await traceSpan("external-db-sync.sequencer", async (span) => { const startTime = performance.now(); const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); - const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); const pollIntervalMs = 50; const batchSize = getSequencerBatchSize(); span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); - span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); span.setAttribute("stack.external-db-sync.batch-size", batchSize); let iterations = 0; type SequencerIterationResult = { - stopReason: "disabled" | "idle" | null, + stopReason: "disabled" | null, }; while (performance.now() - startTime < maxDurationMs) { @@ -221,9 +215,6 @@ export const GET = createSmartRouteHandler({ try { const didUpdate = await backfillSequenceIds(batchSize); iterationSpan.setAttribute("stack.external-db-sync.did-update", didUpdate); - if (stopWhenIdle && !didUpdate) { - return { stopReason: "idle" }; - } } catch (error) { iterationSpan.setAttribute("stack.external-db-sync.iteration-error", true); captureError( diff --git a/apps/e2e/tests/backend/backend-helpers.ts b/apps/e2e/tests/backend/backend-helpers.ts index b6e21122ce..bed3b5af78 100644 --- a/apps/e2e/tests/backend/backend-helpers.ts +++ b/apps/e2e/tests/backend/backend-helpers.ts @@ -186,6 +186,36 @@ export async function bumpEmailAddress(options: { unindexed?: boolean } = {}) { return mailbox; } +// Helper to get emails from the outbox, filtered by subject if provided +export async function getOutboxEmails(options?: { subject?: string }) { + const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { + method: "GET", + accessType: "server", + }); + if (options?.subject) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return listResponse.body.items.filter((e: any) => e.subject === options.subject); + } + return listResponse.body.items; +} + +// Helper to poll the outbox until an email with the expected subject and status appears +export async function waitForOutboxEmailWithStatus(subject: string, status: string) { + const maxRetries = 24; + let emails: any[] = []; + for (let i = 0; i < maxRetries; i++) { + emails = await getOutboxEmails({ subject }); + if (emails.length > 0 && emails[0].status === status) { + return emails; + } + await wait(500); + } + throw new StackAssertionError( + `Timeout waiting for outbox email with subject "${subject}" and status "${status}"`, + { foundEmails: emails } + ); +} + export namespace Auth { export async function fastSignUp(body: any = {}) { const { userId } = await User.create(body); @@ -405,7 +435,11 @@ export namespace Auth { } await wait(100 + i * 20); if (i >= 30) { - throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, { response, messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })) }); + throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, { + response, + messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })), + outboxEmails: await getOutboxEmails(), + }); } } return { diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts index 9cbba4b924..4fd8b724d8 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts @@ -1,18 +1 @@ -import { niceBackendFetch } from "../../../../backend-helpers"; - -/** - * Helper to get emails from the outbox, filtered by subject if provided. - * Shared across email test files to avoid duplication. - */ -export async function getOutboxEmails(options?: { subject?: string }) { - const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { - method: "GET", - accessType: "server", - }); - if (options?.subject) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return listResponse.body.items.filter((e: any) => e.subject === options.subject); - } - return listResponse.body.items; -} - +export { getOutboxEmails, waitForOutboxEmailWithStatus } from "../../../../backend-helpers"; diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts index e343f06296..93d4326c8e 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts @@ -1,49 +1,10 @@ -import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; import { wait } from "@stackframe/stack-shared/dist/utils/promises"; import { deindent, nicify } from "@stackframe/stack-shared/dist/utils/strings"; import beautify from "js-beautify"; import { describe } from "vitest"; import { it, logIfTestFails } from "../../../../../helpers"; import { withPortPrefix } from "../../../../../helpers/ports"; -import { Auth, Project, User, backendContext, bumpEmailAddress, niceBackendFetch } from "../../../../backend-helpers"; - -type OutboxEmail = { - id: string, - subject?: string, - status: string, - skipped_reason?: string, - is_transactional?: boolean, - tsx_source?: string, -}; - -// Helper to get emails from the outbox, filtered by subject if provided -async function getOutboxEmails(options?: { subject?: string }): Promise { - const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { - method: "GET", - accessType: "server", - }); - if (options?.subject) { - return listResponse.body.items.filter((e: any) => e.subject === options.subject); - } - return listResponse.body.items; -} - -// Helper to poll the outbox until an email with the expected subject and status appears -async function waitForOutboxEmailWithStatus(subject: string, status: string): Promise { - const maxRetries = 20; - let emails: OutboxEmail[] = []; - for (let i = 0; i < maxRetries; i++) { - emails = await getOutboxEmails({ subject }); - if (emails.length > 0 && emails[0].status === status) { - return emails; - } - await wait(500); - } - throw new StackAssertionError( - `Timeout waiting for outbox email with subject "${subject}" and status "${status}"`, - { foundEmails: emails } - ); -} +import { Auth, Project, User, backendContext, bumpEmailAddress, getOutboxEmails, niceBackendFetch, waitForOutboxEmailWithStatus } from "../../../../backend-helpers"; const testEmailConfig = { type: "standard", diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts index c01855d3a5..8e76b055cf 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts @@ -3,7 +3,7 @@ import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; import { describe } from "vitest"; import { it } from "../../../../../helpers"; import { withPortPrefix } from "../../../../../helpers/ports"; -import { Project, backendContext, bumpEmailAddress, niceBackendFetch } from "../../../../backend-helpers"; +import { Project, backendContext, bumpEmailAddress, niceBackendFetch, waitForOutboxEmailWithStatus } from "../../../../backend-helpers"; const testEmailConfig = { type: "standard", @@ -52,18 +52,6 @@ const slowTemplate = deindent` } `; -// Helper to get emails from the outbox, filtered by subject if provided -async function getOutboxEmails(options?: { subject?: string }) { - const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { - method: "GET", - accessType: "server", - }); - if (options?.subject) { - return listResponse.body.items.filter((e: any) => e.subject === options.subject); - } - return listResponse.body.items; -} - describe("email outbox API", () => { describe("list endpoint", () => { it("should list emails in the outbox", async ({ expect }) => { @@ -250,12 +238,8 @@ describe("email outbox API", () => { }, }); - // Wait for email to be processed - await wait(7_000); - - // Get the email from the list endpoint - const emails = await getOutboxEmails({ subject: "Get Test Email" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach sent status + const emails = await waitForOutboxEmailWithStatus("Get Test Email", "sent"); const emailId = emails[0].id; // Get the email by ID @@ -317,11 +301,8 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - // Get the email ID from first project - const emails = await getOutboxEmails({ subject: "Cross Project Test Email" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach sent status + const emails = await waitForOutboxEmailWithStatus("Cross Project Test Email", "sent"); const emailId = emails[0].id; // Create second project @@ -374,12 +355,8 @@ describe("email outbox API", () => { }, }); - // Wait for email to be sent - await wait(7_000); - - // Get the email ID - const emails = await getOutboxEmails({ subject: "Not Editable Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach sent status + const emails = await waitForOutboxEmailWithStatus("Not Editable Test", "sent"); const emailId = emails[0].id; // Try to edit @@ -423,17 +400,10 @@ describe("email outbox API", () => { }, }); - // Wait for email to be processed and skipped - await wait(7_000); - - // Get the email - const emails = await getOutboxEmails({ subject: "Skipped Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach skipped status + const emails = await waitForOutboxEmailWithStatus("Skipped Test", "skipped"); const email = emails[0]; - // Verify it's skipped - expect(email.status).toBe("skipped"); - // Try to edit const editResponse = await niceBackendFetch(`/api/v1/emails/outbox/${email.id}`, { method: "PATCH", @@ -479,10 +449,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "Status Test Email" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("Status Test Email", "sent"); const email = emails[0]; // Check discriminated union fields @@ -526,10 +493,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "Skipped Status Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("Skipped Status Test", "skipped"); const email = emails[0]; expect(email.status).toBe("skipped"); @@ -574,11 +538,8 @@ describe("email outbox API", () => { }); expect(sendResponse.status).toBe(200); - // Wait for email to be sent - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "Edit TSX Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach sent status + const emails = await waitForOutboxEmailWithStatus("Edit TSX Test", "sent"); const emailId = emails[0].id; // For emails that are already SENT, we can't edit them @@ -1100,14 +1061,9 @@ describe("email outbox API", () => { }, }); - // Wait for email to be processed and skipped - await wait(7_000); - - // Get the email and verify it's skipped - const emails = await getOutboxEmails({ subject: "Cancel Already Skipped Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + // Wait for email to reach skipped status + const emails = await waitForOutboxEmailWithStatus("Cancel Already Skipped Test", "skipped"); const email = emails[0]; - expect(email.status).toBe("skipped"); expect(email.skipped_reason).toBe("USER_HAS_NO_PRIMARY_EMAIL"); // Try to cancel an already-skipped email - should fail with EMAIL_NOT_EDITABLE @@ -1155,10 +1111,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "Recipient Type Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("Recipient Type Test", "sent"); const email = emails[0]; expect(email.to.type).toBe("user-primary-email"); @@ -1247,10 +1200,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "Base Fields Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("Base Fields Test", "sent"); const email = emails[0]; // Check all base fields @@ -1317,15 +1267,11 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - // Verify we have both sent and skipped emails - const sentEmails = await getOutboxEmails({ subject: "Multi Status Test Sent" }); - expect(sentEmails.length).toBeGreaterThanOrEqual(1); + const sentEmails = await waitForOutboxEmailWithStatus("Multi Status Test Sent", "sent"); expect(sentEmails[0].status).toBe("sent"); - const skippedEmails = await getOutboxEmails({ subject: "Multi Status Test Skipped" }); - expect(skippedEmails.length).toBeGreaterThanOrEqual(1); + const skippedEmails = await waitForOutboxEmailWithStatus("Multi Status Test Skipped", "skipped"); expect(skippedEmails[0].status).toBe("skipped"); }); }); @@ -1361,10 +1307,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "SENT Snapshot Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("SENT Snapshot Test", "sent"); const email = emails[0]; // Verify the structure matches the expected discriminated union @@ -1410,10 +1353,7 @@ describe("email outbox API", () => { }, }); - await wait(7_000); - - const emails = await getOutboxEmails({ subject: "SKIPPED Snapshot Test" }); - expect(emails.length).toBeGreaterThanOrEqual(1); + const emails = await waitForOutboxEmailWithStatus("SKIPPED Snapshot Test", "skipped"); const email = emails[0]; // Verify the structure matches the expected discriminated union for skipped diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts index 2cb93a4912..e48e51bc3b 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts @@ -10,14 +10,11 @@ import { TEST_TIMEOUT, TestDbManager, createProjectWithExternalDb as createProjectWithExternalDbRaw, - forceExternalDbSync, waitForCondition, waitForSyncedDeletion, waitForTable } from './external-db-sync-utils'; -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - describe.sequential('External DB Sync - Race Condition Tests', () => { let dbManager: TestDbManager; const createProjectWithExternalDb = ( @@ -376,24 +373,25 @@ describe.sequential('External DB Sync - Race Condition Tests', () => { [user.userId], ); - const forced = await forceExternalDbSync(); - if (!forced) { - await sleep(70000); - } - - const during = await externalClient.query<{ - display_name: string | null, - }>( - ` - SELECT "display_name" - FROM "users" - WHERE "primary_email" = $1 - `, - [`${dbName}@example.com`], + let row: any; + await waitForCondition( + async () => { + const res = await externalClient.query<{ + display_name: string | null, + }>( + `SELECT "display_name" FROM "users" WHERE "primary_email" = $1`, + [`${dbName}@example.com`], + ); + if (res.rows.length !== 1) return false; + row = res.rows[0]; + return true; + }, + { + description: 'waiting for sync to complete', + timeoutMs: 10000, + }, ); - expect(during.rows.length).toBe(1); - const row = during.rows[0]; // Uncommitted transaction should not be visible expect(row.display_name).not.toBe('Transaction 1'); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts index 2e588e61f7..1b0521af4f 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -10,18 +10,6 @@ export const POSTGRES_USER = process.env.EXTERNAL_DB_TEST_USER || 'postgres'; export const POSTGRES_PASSWORD = process.env.EXTERNAL_DB_TEST_PASSWORD || 'PASSWORD-PLACEHOLDER--uqfEC1hmmv'; export const TEST_TIMEOUT = 240000; export const HIGH_VOLUME_TIMEOUT = 600000; // 10 minutes for 1500+ users -const SHOULD_FORCE_EXTERNAL_DB_SYNC = process.env.STACK_FORCE_EXTERNAL_DB_SYNC === 'true'; -const FORCE_SYNC_MAX_DURATION_MS = (() => { - const raw = process.env.STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS; - if (!raw) return 5000; - const parsed = Number.parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error('STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS must be a positive integer'); - } - return parsed; -})(); -const FORCE_SYNC_INTERVAL_MS = 2000; -let lastForcedSyncAt = -Infinity; // Connection settings to prevent connection leaks const CLIENT_CONFIG: Partial = { @@ -143,7 +131,6 @@ export async function waitForCondition( while (performance.now() - startTime < timeoutMs) { try { - await maybeForceExternalDbSync(); if (await checkFn()) { return; } @@ -162,46 +149,6 @@ export async function waitForCondition( throw new Error(`Timeout waiting for ${description} after ${timeoutMs}ms`); } -export async function forceExternalDbSync(): Promise { - if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return false; - - const cronSecret = process.env.CRON_SECRET; - if (!cronSecret) { - throw new Error('CRON_SECRET is required when STACK_FORCE_EXTERNAL_DB_SYNC=true'); - } - - lastForcedSyncAt = performance.now(); - - await niceFetch(new URL('/api/latest/internal/external-db-sync/sequencer', STACK_BACKEND_BASE_URL), { - query: { - maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), - stopWhenIdle: "true", - }, - headers: { - Authorization: `Bearer ${cronSecret}`, - }, - }); - await niceFetch(new URL('/api/latest/internal/external-db-sync/poller', STACK_BACKEND_BASE_URL), { - query: { - maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), - stopWhenIdle: "true", - }, - headers: { - Authorization: `Bearer ${cronSecret}`, - }, - }); - return true; -} - -async function maybeForceExternalDbSync() { - if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return; - - const now = performance.now(); - if (now - lastForcedSyncAt < FORCE_SYNC_INTERVAL_MS) return; - - await forceExternalDbSync(); -} - /** * Wait for data to appear in external DB (relies on automatic cron job) */ diff --git a/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts index f5e545640d..8d95ecdf97 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts @@ -13,7 +13,7 @@ describe("without project access", () => { await backendContext.with({ projectKeys: InternalProjectKeys, }, async () => { - await Auth.Otp.signIn(); + await Auth.fastSignUp(); }); const response = await niceBackendFetch("/api/v1/users/me"); expect(response).toMatchInlineSnapshot(` From 25996907ddaaa4ea2e0bd9c37dacf5116df923f4 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Fri, 6 Feb 2026 16:03:40 -0800 Subject: [PATCH 02/11] fix --- .../api/v1/emails/outbox-api.test.ts | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts index 8e76b055cf..da81f38760 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts @@ -1,9 +1,10 @@ +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; import { wait } from "@stackframe/stack-shared/dist/utils/promises"; import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; import { describe } from "vitest"; import { it } from "../../../../../helpers"; import { withPortPrefix } from "../../../../../helpers/ports"; -import { Project, backendContext, bumpEmailAddress, niceBackendFetch, waitForOutboxEmailWithStatus } from "../../../../backend-helpers"; +import { Project, backendContext, bumpEmailAddress, getOutboxEmails, niceBackendFetch, waitForOutboxEmailWithStatus } from "../../../../backend-helpers"; const testEmailConfig = { type: "standard", @@ -601,10 +602,8 @@ describe("email outbox API", () => { expect(sendResponse.status).toBe(200); // Poll until we find the email and can pause it (with timeout) - let emailId: string | null = null; - let pauseSucceeded = false; - - for (let i = 0; i < 20; i++) { + let emailId: string; + for (let i = 0;; i++) { const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { method: "GET", accessType: "server", @@ -622,19 +621,61 @@ describe("email outbox API", () => { }, }); - if (pauseResponse.status === 200 && pauseResponse.body.status === "paused") { - pauseSucceeded = true; - break; + expect(pauseResponse).toMatchInlineSnapshot(` + NiceResponse { + "status": 200, + "body": { + "created_at_millis": , + "has_delivered": false, + "has_rendered": false, + "id": "", + "is_paused": true, + "scheduled_at_millis": , + "simple_status": "in-progress", + "skip_deliverability_check": false, + "status": "paused", + "theme_id": null, + "to": { + "type": "user-primary-email", + "user_id": "", + }, + "tsx_source": deindent\` + import { Container } from "@react-email/components"; + import { Subject, NotificationCategory, Props } from "@stackframe/emails"; + + // Artificial delay to make the email slow to render + const startTime = performance.now(); + while (performance.now() - startTime < 500) { + // Busy wait - 500ms delay + } + + export function EmailTemplate({ user, project }) { + return ( + + + +
Slow email content
+ + ); + } + \`, + "updated_at_millis": , + "variables": {}, + }, + "headers": Headers {