From 2380972f38e4484ba2be6ca06caa2abdb0380382 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Tue, 10 Mar 2026 09:42:12 -0700 Subject: [PATCH 1/4] fix: tests bleeding across files, flaky neon test Because provisionProject used to be exported from provision.test.ts, when the test failed in provision.test.ts, CI registered it as coming from webhooks.test.ts. This is obviously bad so we extract it to a shared helper. The neon flaky test in provision.test.ts doesn't actually need to verify the email. Plus, there's a flag in the signUpWithEmail function that exists for precisely the purpose of skipping the check. This e2e test isn't related to sign up emails at all and they aren't part of the critical path. --- .../custom/projects/provision-helpers.ts | 13 +++++++++++++ .../custom/projects/provision.test.ts | 13 +------------ .../integrations/custom/projects/transfer.test.ts | 2 +- .../neon/projects/provision-helpers.ts | 13 +++++++++++++ .../integrations/neon/projects/provision.test.ts | 15 ++------------- .../integrations/neon/projects/transfer.test.ts | 2 +- .../api/v1/integrations/neon/webhooks.test.ts | 2 +- 7 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision-helpers.ts create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision-helpers.ts diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision-helpers.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision-helpers.ts new file mode 100644 index 0000000000..4c9172c4b9 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision-helpers.ts @@ -0,0 +1,13 @@ +import { niceBackendFetch } from "../../../../../../backend-helpers"; + +export async function provisionProject() { + return await niceBackendFetch("/api/v1/integrations/custom/projects/provision", { + method: "POST", + body: { + display_name: "Test project", + }, + headers: { + "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==", + }, + }); +} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision.test.ts index 174c3a86d7..d8479c4b3c 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision.test.ts @@ -1,17 +1,6 @@ import { it } from "../../../../../../../helpers"; import { backendContext, niceBackendFetch } from "../../../../../../backend-helpers"; - -export async function provisionProject() { - return await niceBackendFetch("/api/v1/integrations/custom/projects/provision", { - method: "POST", - body: { - display_name: "Test project", - }, - headers: { - "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==", - }, - }); -} +import { provisionProject } from "./provision-helpers"; it("should be able to provision a new project if client details are correct", async ({ expect }) => { const response = await provisionProject(); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/transfer.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/transfer.test.ts index 523c18cefc..e6635d015f 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/transfer.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/transfer.test.ts @@ -2,7 +2,7 @@ import { urlString } from "@stackframe/stack-shared/dist/utils/urls"; import { expect } from "vitest"; import { it } from "../../../../../../../helpers"; import { Auth, Project, niceBackendFetch } from "../../../../../../backend-helpers"; -import { provisionProject } from "./provision.test"; +import { provisionProject } from "./provision-helpers"; async function initiateTransfer(projectId: string) { const response = await niceBackendFetch("/api/v1/integrations/custom/projects/transfer/initiate", { diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision-helpers.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision-helpers.ts new file mode 100644 index 0000000000..a46de2ea56 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision-helpers.ts @@ -0,0 +1,13 @@ +import { niceBackendFetch } from "../../../../../../backend-helpers"; + +export async function provisionProject() { + return await niceBackendFetch("/api/v1/integrations/neon/projects/provision", { + method: "POST", + body: { + display_name: "Test project", + }, + headers: { + "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==", + }, + }); +} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision.test.ts index 5d14140c0b..fcc2141cd4 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision.test.ts @@ -1,18 +1,7 @@ import { decryptValue, hashKey } from "@stackframe/stack-shared/dist/helpers/vault/client-side"; import { it } from "../../../../../../../helpers"; import { Auth, InternalApiKey, InternalProjectKeys, backendContext, niceBackendFetch } from "../../../../../../backend-helpers"; - -export async function provisionProject() { - return await niceBackendFetch("/api/v1/integrations/neon/projects/provision", { - method: "POST", - body: { - display_name: "Test project", - }, - headers: { - "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==", - }, - }); -} +import { provisionProject } from "./provision-helpers"; it("should be able to provision a new project if neon client details are correct", async ({ expect }) => { const response = await provisionProject(); @@ -112,7 +101,7 @@ it("should be able to provision a new project if neon client details are correct `); // ensure we can create a user in the new project (make sure it's writable) - const signInResponse = await Auth.Password.signUpWithEmail({ password: "test1234" }); + const signInResponse = await Auth.Password.signUpWithEmail({ password: "test1234", noWaitForEmail: true }); expect(signInResponse).toMatchInlineSnapshot(` { "email": "default-mailbox--@stack-generated.example.com", diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/transfer.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/transfer.test.ts index a4899fce6e..ea78be2eb3 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/transfer.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/transfer.test.ts @@ -2,7 +2,7 @@ import { urlString } from "@stackframe/stack-shared/dist/utils/urls"; import { expect } from "vitest"; import { it } from "../../../../../../../helpers"; import { Auth, Project, niceBackendFetch } from "../../../../../../backend-helpers"; -import { provisionProject } from "./provision.test"; +import { provisionProject } from "./provision-helpers"; async function initiateTransfer(projectId: string) { const response = await niceBackendFetch("/api/v1/integrations/neon/projects/transfer/initiate", { diff --git a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/webhooks.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/webhooks.test.ts index 1121c7b842..5faeb809cc 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/webhooks.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/webhooks.test.ts @@ -1,6 +1,6 @@ import { it } from "../../../../../../helpers"; import { niceBackendFetch } from "../../../../../backend-helpers"; -import { provisionProject } from "./projects/provision.test"; +import { provisionProject } from "./projects/provision-helpers"; it("should be able to create a webhook", async ({ expect }) => { From 2ee86232247e771770b2f49e3aa85b5c572db1b2 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Tue, 10 Mar 2026 09:46:46 -0700 Subject: [PATCH 2/4] fix: flaky register tests Same issue with it checking if email has arrived when other tests in our test suite already do that. We also bump up number of retries to make any other tests that depend on this more robust --- .../backend/endpoints/api/v1/auth/passkey/register.test.ts | 4 ++-- apps/e2e/tests/helpers.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/e2e/tests/backend/endpoints/api/v1/auth/passkey/register.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/auth/passkey/register.test.ts index 9540b0b53e..97f1b29473 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/auth/passkey/register.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/auth/passkey/register.test.ts @@ -6,14 +6,14 @@ import { Auth, niceBackendFetch, Project } from "../../../../../backend-helpers" it("should allow initiating passkey registration", async ({ expect }) => { await Project.createAndSwitch({ config: { passkey_enabled: true } }); - const res = await Auth.Password.signUpWithEmail(); + await Auth.Password.signUpWithEmail({ noWaitForEmail: true }); await Auth.Passkey.initiateRegistration(); }); it("should successfully register a passkey", async ({ expect }) => { await Project.createAndSwitch({ config: { passkey_enabled: true } }); - const res = await Auth.Password.signUpWithEmail(); + await Auth.Password.signUpWithEmail({ noWaitForEmail: true }); await Auth.Passkey.register(); }); diff --git a/apps/e2e/tests/helpers.ts b/apps/e2e/tests/helpers.ts index bff855132d..32a2255f79 100644 --- a/apps/e2e/tests/helpers.ts +++ b/apps/e2e/tests/helpers.ts @@ -241,7 +241,7 @@ export class Mailbox { }; this.waitForMessagesWithSubjectCount = async (subject: string, minCount: number, options?: { noBody?: boolean }) => { - const maxRetries = 25; + const maxRetries = 30; let messages: MailboxMessage[] = []; for (let i = 0; i < maxRetries; i++) { messages = await this.fetchMessages(options); From 23b1215fc3f57b6db0667eb76bcc511f036c1279 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Tue, 10 Mar 2026 09:54:05 -0700 Subject: [PATCH 3/4] fix: flaky password test that depends on signincode Bumping up number of attempts. Since it's polling with early exit, if it finds it early it exits. So the average case shouldn't change. --- apps/e2e/tests/backend/backend-helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/e2e/tests/backend/backend-helpers.ts b/apps/e2e/tests/backend/backend-helpers.ts index ce4aeff1fd..f477cf3e04 100644 --- a/apps/e2e/tests/backend/backend-helpers.ts +++ b/apps/e2e/tests/backend/backend-helpers.ts @@ -455,7 +455,7 @@ export namespace Auth { break; } await wait(100 + i * 20); - if (i >= 30) { + if (i >= 40) { throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, { response, messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })), From 9230513726b2caadf3ad09e2689f12a7e2113b68 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Tue, 10 Mar 2026 10:44:11 -0700 Subject: [PATCH 4/4] fix: flaky email tests for email test switching to a polling approach is the right fix, like the todo says. for outbox api it's dicier due to a race condition. We --- .../api/v1/emails/outbox-api.test.ts | 43 +++++++++++-------- apps/e2e/tests/js/email.test.ts | 14 +++--- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts index 503086930a..81c46a04e4 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts @@ -38,8 +38,8 @@ const slowTemplate = deindent` // Artificial delay to make the email slow to render const startTime = performance.now(); - while (performance.now() - startTime < 500) { - // Busy wait - 500ms delay + while (performance.now() - startTime < 2000) { + // Busy wait - 2000ms delay } export function EmailTemplate({ user, project }) { @@ -648,8 +648,8 @@ describe("email outbox API", () => { // Artificial delay to make the email slow to render const startTime = performance.now(); - while (performance.now() - startTime < 500) { - // Busy wait - 500ms delay + while (performance.now() - startTime < 2000) { + // Busy wait - 2000ms delay } export function EmailTemplate({ user, project }) { @@ -670,12 +670,12 @@ describe("email outbox API", () => { `); break; } else { - if (i >= 20) { + if (i >= 50) { throw new StackAssertionError(`Timeout waiting for email in the outbox`, { outboxEmails: await getOutboxEmails(), }); } - await wait(25); + await wait(100); } } @@ -884,7 +884,7 @@ describe("email outbox API", () => { let emailId: string | null = null; let pauseSucceeded = false; - for (let i = 0; i < 20; i++) { + for (let i = 0; i < 50; i++) { const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { method: "GET", accessType: "server", @@ -908,7 +908,7 @@ describe("email outbox API", () => { } } - await wait(25); + await wait(100); } // These assertions must always run - test fails if we couldn't pause @@ -937,15 +937,20 @@ describe("email outbox API", () => { // After unpausing, the email should go back to processing (preparing/rendering/scheduled/etc) expect(unpauseResponse.body.status).not.toBe("paused"); - // Wait for the email to be sent (since we unpaused it) - await wait(7_000); - - // Verify the email was eventually sent - const finalGetResponse = await niceBackendFetch(`/api/v1/emails/outbox/${emailId}`, { - method: "GET", - accessType: "server", - }); - expect(finalGetResponse.body.status).toBe("sent"); + // Poll until the email is sent (since we unpaused it) + for (let i = 0; ; i++) { + const finalGetResponse = await niceBackendFetch(`/api/v1/emails/outbox/${emailId}`, { + method: "GET", + accessType: "server", + }); + if (finalGetResponse.body.status === "sent") break; + if (i >= 50) { + throw new StackAssertionError(`Timed out waiting for email to be sent after unpause`, { + status: finalGetResponse.body.status, + }); + } + await wait(500); + } }); it("should cancel email with MANUALLY_CANCELLED reason", async ({ expect }) => { @@ -1000,7 +1005,7 @@ describe("email outbox API", () => { let emailId: string | null = null; let pauseSucceeded = false; - for (let i = 0; i < 20; i++) { + for (let i = 0; i < 50; i++) { const listResponse = await niceBackendFetch("/api/v1/emails/outbox", { method: "GET", accessType: "server", @@ -1024,7 +1029,7 @@ describe("email outbox API", () => { } } - await wait(25); + await wait(100); } // We need to have successfully paused the email to test cancel diff --git a/apps/e2e/tests/js/email.test.ts b/apps/e2e/tests/js/email.test.ts index cc94b6a192..d498f07eaa 100644 --- a/apps/e2e/tests/js/email.test.ts +++ b/apps/e2e/tests/js/email.test.ts @@ -187,11 +187,15 @@ it("should provide delivery statistics", async ({ expect }) => { subject: "Stats", }); - // wait until the email is sent - // TODO: use the equivalent of waitForMessagesWithSubject - await wait(10_000); - - const info = await serverApp.getEmailDeliveryStats(); + let info; + for (let i = 0; ; i++) { + info = await serverApp.getEmailDeliveryStats(); + if (info.stats.hour.sent >= 1) break; + if (i >= 50) { + throw new Error(`Timed out waiting for email delivery stats to reflect sent email: ${JSON.stringify(info)}`); + } + await wait(500); + } expect(info).toMatchInlineSnapshot(` {