[Fix] Flaky Neon, Email Delivery, and Other Tests

apps/e2e/tests/backend/backend-helpers.ts

@@ -455,7 +455,7 @@ export namespace Auth {
           break;
         }
         await wait(100 + i * 20);
-        if (i >= 30) {
+        if (i >= 40) {
           throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, {
             response,
             messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })),

apps/e2e/tests/backend/endpoints/api/v1/auth/passkey/register.test.ts

@@ -6,14 +6,14 @@ import { Auth, niceBackendFetch, Project } from "../../../../../backend-helpers"
 
 it("should allow initiating passkey registration", async ({ expect }) => {
   await Project.createAndSwitch({ config: { passkey_enabled: true } });
-  const res = await Auth.Password.signUpWithEmail();
+  await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   await Auth.Passkey.initiateRegistration();
 });
 
 
 it("should successfully register a passkey", async ({ expect }) => {
   await Project.createAndSwitch({ config: { passkey_enabled: true } });
-  const res = await Auth.Password.signUpWithEmail();
+  await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   await Auth.Passkey.register();
 });
 

apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts

@@ -38,8 +38,8 @@ const slowTemplate = deindent`
 
   // Artificial delay to make the email slow to render
   const startTime = performance.now();
-  while (performance.now() - startTime < 500) {
-    // Busy wait - 500ms delay
+  while (performance.now() - startTime < 2000) {
+    // Busy wait - 2000ms delay
   }
 
   export function EmailTemplate({ user, project }) {
@@ -648,8 +648,8 @@ describe("email outbox API", () => {
 
                   // Artificial delay to make the email slow to render
                   const startTime = performance.now();
-                  while (performance.now() - startTime < 500) {
-                    // Busy wait - 500ms delay
+                  while (performance.now() - startTime < 2000) {
+                    // Busy wait - 2000ms delay
                   }
 
                   export function EmailTemplate({ user, project }) {
@@ -670,12 +670,12 @@ describe("email outbox API", () => {
           `);
           break;
         } else {
-          if (i >= 20) {
+          if (i >= 50) {
             throw new StackAssertionError(`Timeout waiting for email in the outbox`, {
               outboxEmails: await getOutboxEmails(),
             });
           }
-          await wait(25);
+          await wait(100);
         }
       }
 
@@ -884,7 +884,7 @@ describe("email outbox API", () => {
       let emailId: string | null = null;
       let pauseSucceeded = false;
 
-      for (let i = 0; i < 20; i++) {
+      for (let i = 0; i < 50; i++) {
         const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
           method: "GET",
           accessType: "server",
@@ -908,7 +908,7 @@ describe("email outbox API", () => {
           }
         }
 
-        await wait(25);
+        await wait(100);
       }
 
       // These assertions must always run - test fails if we couldn't pause
@@ -937,15 +937,20 @@ describe("email outbox API", () => {
       // After unpausing, the email should go back to processing (preparing/rendering/scheduled/etc)
       expect(unpauseResponse.body.status).not.toBe("paused");
 
-      // Wait for the email to be sent (since we unpaused it)
-      await wait(7_000);
-
-      // Verify the email was eventually sent
-      const finalGetResponse = await niceBackendFetch(`/api/v1/emails/outbox/${emailId}`, {
-        method: "GET",
-        accessType: "server",
-      });
-      expect(finalGetResponse.body.status).toBe("sent");
+      // Poll until the email is sent (since we unpaused it)
+      for (let i = 0; ; i++) {
+        const finalGetResponse = await niceBackendFetch(`/api/v1/emails/outbox/${emailId}`, {
+          method: "GET",
+          accessType: "server",
+        });
+        if (finalGetResponse.body.status === "sent") break;
+        if (i >= 50) {
+          throw new StackAssertionError(`Timed out waiting for email to be sent after unpause`, {
+            status: finalGetResponse.body.status,
+          });
+        }
+        await wait(500);
+      }
     });
 
     it("should cancel email with MANUALLY_CANCELLED reason", async ({ expect }) => {
@@ -1000,7 +1005,7 @@ describe("email outbox API", () => {
       let emailId: string | null = null;
       let pauseSucceeded = false;
 
-      for (let i = 0; i < 20; i++) {
+      for (let i = 0; i < 50; i++) {
         const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
           method: "GET",
           accessType: "server",
@@ -1024,7 +1029,7 @@ describe("email outbox API", () => {
           }
         }
 
-        await wait(25);
+        await wait(100);
       }
 
       // We need to have successfully paused the email to test cancel

apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision-helpers.ts

@@ -0,0 +1,13 @@
+import { niceBackendFetch } from "../../../../../../backend-helpers";
+
+export async function provisionProject() {
+  return await niceBackendFetch("/api/v1/integrations/custom/projects/provision", {
+    method: "POST",
+    body: {
+      display_name: "Test project",
+    },
+    headers: {
+      "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
+    },
+  });
+}

apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/provision.test.ts

@@ -1,17 +1,6 @@
 import { it } from "../../../../../../../helpers";
 import { backendContext, niceBackendFetch } from "../../../../../../backend-helpers";
-
-export async function provisionProject() {
-  return await niceBackendFetch("/api/v1/integrations/custom/projects/provision", {
-    method: "POST",
-    body: {
-      display_name: "Test project",
-    },
-    headers: {
-      "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
-    },
-  });
-}
+import { provisionProject } from "./provision-helpers";
 
 it("should be able to provision a new project if client details are correct", async ({ expect }) => {
   const response = await provisionProject();

apps/e2e/tests/backend/endpoints/api/v1/integrations/custom/projects/transfer.test.ts

@@ -2,7 +2,7 @@ import { urlString } from "@stackframe/stack-shared/dist/utils/urls";
 import { expect } from "vitest";
 import { it } from "../../../../../../../helpers";
 import { Auth, Project, niceBackendFetch } from "../../../../../../backend-helpers";
-import { provisionProject } from "./provision.test";
+import { provisionProject } from "./provision-helpers";
 
 async function initiateTransfer(projectId: string) {
   const response = await niceBackendFetch("/api/v1/integrations/custom/projects/transfer/initiate", {

apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision-helpers.ts

@@ -0,0 +1,13 @@
+import { niceBackendFetch } from "../../../../../../backend-helpers";
+
+export async function provisionProject() {
+  return await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
+    method: "POST",
+    body: {
+      display_name: "Test project",
+    },
+    headers: {
+      "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
+    },
+  });
+}

apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/provision.test.ts

@@ -1,18 +1,7 @@
 import { decryptValue, hashKey } from "@stackframe/stack-shared/dist/helpers/vault/client-side";
 import { it } from "../../../../../../../helpers";
 import { Auth, InternalApiKey, InternalProjectKeys, backendContext, niceBackendFetch } from "../../../../../../backend-helpers";
-
-export async function provisionProject() {
-  return await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
-    method: "POST",
-    body: {
-      display_name: "Test project",
-    },
-    headers: {
-      "Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
-    },
-  });
-}
+import { provisionProject } from "./provision-helpers";
 
 it("should be able to provision a new project if neon client details are correct", async ({ expect }) => {
   const response = await provisionProject();
@@ -112,7 +101,7 @@ it("should be able to provision a new project if neon client details are correct
   `);
 
   // ensure we can create a user in the new project (make sure it's writable)
-  const signInResponse = await Auth.Password.signUpWithEmail({ password: "test1234" });
+  const signInResponse = await Auth.Password.signUpWithEmail({ password: "test1234", noWaitForEmail: true });
   expect(signInResponse).toMatchInlineSnapshot(`
     {
       "email": "default-mailbox--<stripped UUID>@stack-generated.example.com",

apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/projects/transfer.test.ts

@@ -2,7 +2,7 @@ import { urlString } from "@stackframe/stack-shared/dist/utils/urls";
 import { expect } from "vitest";
 import { it } from "../../../../../../../helpers";
 import { Auth, Project, niceBackendFetch } from "../../../../../../backend-helpers";
-import { provisionProject } from "./provision.test";
+import { provisionProject } from "./provision-helpers";
 
 async function initiateTransfer(projectId: string) {
   const response = await niceBackendFetch("/api/v1/integrations/neon/projects/transfer/initiate", {

apps/e2e/tests/backend/endpoints/api/v1/integrations/neon/webhooks.test.ts

@@ -1,6 +1,6 @@
 import { it } from "../../../../../../helpers";
 import { niceBackendFetch } from "../../../../../backend-helpers";
-import { provisionProject } from "./projects/provision.test";
+import { provisionProject } from "./projects/provision-helpers";
 
 
 it("should be able to create a webhook", async ({ expect }) => {

apps/e2e/tests/helpers.ts

@@ -241,7 +241,7 @@ export class Mailbox {
     };
 
     this.waitForMessagesWithSubjectCount = async (subject: string, minCount: number, options?: { noBody?: boolean }) => {
-      const maxRetries = 25;
+      const maxRetries = 30;
       let messages: MailboxMessage[] = [];
       for (let i = 0; i < maxRetries; i++) {
         messages = await this.fetchMessages(options);

apps/e2e/tests/js/email.test.ts

@@ -187,11 +187,15 @@ it("should provide delivery statistics", async ({ expect }) => {
     subject: "Stats",
   });
 
-  // wait until the email is sent
-  // TODO: use the equivalent of waitForMessagesWithSubject
-  await wait(10_000);
-
-  const info = await serverApp.getEmailDeliveryStats();
+  let info;
+  for (let i = 0; ; i++) {
+    info = await serverApp.getEmailDeliveryStats();
+    if (info.stats.hour.sent >= 1) break;
+    if (i >= 50) {
+      throw new Error(`Timed out waiting for email delivery stats to reflect sent email: ${JSON.stringify(info)}`);
+    }
+    await wait(500);
+  }
 
   expect(info).toMatchInlineSnapshot(`
     {