[Refactor] Improve CI Run Times by Reducing Test Flakiness and Speeding up Test Suite

.github/workflows/e2e-api-tests.yaml

@@ -19,7 +19,6 @@ jobs:
       NODE_ENV: test
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
@@ -160,7 +159,7 @@ jobs:
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} ${{ matrix.freestyle-mode == 'prod' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && 'mail' || '' }}
 
       - name: Run tests again (attempt 1)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -19,7 +19,6 @@ jobs:
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe"
       NEXT_PUBLIC_STACK_PORT_PREFIX: "67"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 

.github/workflows/e2e-source-of-truth-api-tests.yaml

@@ -21,7 +21,6 @@ jobs:
       STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}'
       STACK_TEST_SOURCE_OF_TRUTH: true
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 

.github/workflows/restart-dev-and-test-with-custom-base-port.yaml

@@ -19,7 +19,6 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 

.github/workflows/restart-dev-and-test.yaml

@@ -18,7 +18,6 @@ jobs:
   restart-dev-and-test:
     runs-on: ubicloud-standard-16
     env:
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 

.github/workflows/setup-tests-with-custom-base-port.yaml

@@ -19,7 +19,6 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 

.github/workflows/setup-tests.yaml

@@ -18,7 +18,6 @@ jobs:
   setup-tests:
     runs-on: ubicloud-standard-16
     env:
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
     steps:

apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts

@@ -30,13 +30,6 @@ function parseMaxDurationMs(value: string | undefined): number {
   return parsed;
 }
 
-function parseStopWhenIdle(value: string | undefined): boolean {
-  if (!value) return false;
-  if (value === "true") return true;
-  if (value === "false") return false;
-  throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'");
-}
-
 function directSyncEnabled(): boolean {
   return getEnvVariable(DIRECT_SYNC_ENV, "") === "true";
 }
@@ -69,7 +62,6 @@ export const GET = createSmartRouteHandler({
     }).defined(),
     query: yupObject({
       maxDurationMs: yupString().optional(),
-      stopWhenIdle: yupString().optional(),
     }).defined(),
   }),
   response: yupObject({
@@ -91,13 +83,11 @@ export const GET = createSmartRouteHandler({
     return await traceSpan("external-db-sync.poller", async (span) => {
       const startTime = performance.now();
       const maxDurationMs = parseMaxDurationMs(query.maxDurationMs);
-      const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle);
       const pollIntervalMs = 50;
       const staleClaimIntervalMinutes = 5;
       const pollerClaimLimit = getPollerClaimLimit();
 
       span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs);
-      span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle);
       span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs);
       span.setAttribute("stack.external-db-sync.poller-claim-limit", pollerClaimLimit);
       span.setAttribute("stack.external-db-sync.direct-sync", directSyncEnabled());
@@ -235,7 +225,7 @@ export const GET = createSmartRouteHandler({
       }
 
       type PollerIterationResult = {
-        stopReason: "disabled" | "idle" | null,
+        stopReason: "disabled" | null,
         processed: number,
       };
 
@@ -255,10 +245,6 @@ export const GET = createSmartRouteHandler({
           const pendingRequests = await claimPendingRequests();
           iterationSpan.setAttribute("stack.external-db-sync.pending-count", pendingRequests.length);
 
-          if (stopWhenIdle && pendingRequests.length === 0) {
-            return { stopReason: "idle", processed: 0 };
-          }
-
           const processed = await processRequests(pendingRequests);
           iterationSpan.setAttribute("stack.external-db-sync.processed-count", processed);
           return { stopReason: null, processed };

apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts

@@ -27,13 +27,6 @@ function parseMaxDurationMs(value: string | undefined): number {
   return parsed;
 }
 
-function parseStopWhenIdle(value: string | undefined): boolean {
-  if (!value) return false;
-  if (value === "true") return true;
-  if (value === "false") return false;
-  throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'");
-}
-
 function getSequencerBatchSize(): number {
   const rawValue = getEnvVariable(SEQUENCER_BATCH_SIZE_ENV, "");
   if (!rawValue) return DEFAULT_BATCH_SIZE;
@@ -169,7 +162,6 @@ export const GET = createSmartRouteHandler({
     }).defined(),
     query: yupObject({
       maxDurationMs: yupString().optional(),
-      stopWhenIdle: yupString().optional(),
     }).defined(),
   }),
   response: yupObject({
@@ -190,19 +182,17 @@ export const GET = createSmartRouteHandler({
     return await traceSpan("external-db-sync.sequencer", async (span) => {
       const startTime = performance.now();
       const maxDurationMs = parseMaxDurationMs(query.maxDurationMs);
-      const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle);
       const pollIntervalMs = 50;
       const batchSize = getSequencerBatchSize();
 
       span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs);
-      span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle);
       span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs);
       span.setAttribute("stack.external-db-sync.batch-size", batchSize);
 
       let iterations = 0;
 
       type SequencerIterationResult = {
-        stopReason: "disabled" | "idle" | null,
+        stopReason: "disabled" | null,
       };
 
       while (performance.now() - startTime < maxDurationMs) {
@@ -221,9 +211,6 @@ export const GET = createSmartRouteHandler({
           try {
             const didUpdate = await backfillSequenceIds(batchSize);
             iterationSpan.setAttribute("stack.external-db-sync.did-update", didUpdate);
-            if (stopWhenIdle && !didUpdate) {
-              return { stopReason: "idle" };
-            }
           } catch (error) {
             iterationSpan.setAttribute("stack.external-db-sync.iteration-error", true);
             captureError(

apps/backend/src/lib/external-db-sync-metadata.ts

@@ -11,18 +11,25 @@ const fuseboxSelect = {
   pollerEnabled: true,
 };
 
+// Default values match the Prisma schema defaults
+const defaultFusebox: ExternalDbSyncFusebox = {
+  sequencerEnabled: true,
+  pollerEnabled: true,
+};
+
 export async function getExternalDbSyncFusebox(): Promise<ExternalDbSyncFusebox> {
-  return await globalPrismaClient.externalDbSyncMetadata.upsert({
+  const result = await globalPrismaClient.externalDbSyncMetadata.findFirst({
     where: { singleton: BooleanTrue.TRUE },
-    create: { singleton: BooleanTrue.TRUE },
-    update: {},
     select: fuseboxSelect,
   });
+  // Return defaults if row doesn't exist yet (row is created on first update)
+  return result ?? defaultFusebox;
 }
 
 export async function updateExternalDbSyncFusebox(
   updates: ExternalDbSyncFusebox,
 ): Promise<ExternalDbSyncFusebox> {
+  // Upsert is fine here - updates are infrequent and typically manual/admin actions
   return await globalPrismaClient.externalDbSyncMetadata.upsert({
     where: { singleton: BooleanTrue.TRUE },
     create: { singleton: BooleanTrue.TRUE, ...updates },

apps/e2e/tests/backend/backend-helpers.ts

@@ -186,6 +186,53 @@ export async function bumpEmailAddress(options: { unindexed?: boolean } = {}) {
   return mailbox;
 }
 
+// Type for outbox email items (simplified - full type is EmailOutboxCrud["Server"]["Read"])
+type OutboxEmail = {
+  id: string,
+  subject?: string,
+  status: string,
+  simple_status: string,
+  to?: {
+    type: string,
+    user_id?: string,
+    [key: string]: unknown,
+  },
+  [key: string]: unknown,
+};
+
+// Helper to get emails from the outbox, filtered by subject if provided
+export async function getOutboxEmails(options?: { subject?: string }): Promise<OutboxEmail[]> {
+  const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
+    method: "GET",
+    accessType: "server",
+  });
+  const items = listResponse.body.items as OutboxEmail[];
+  if (options?.subject) {
+    return items.filter((e) => e.subject === options.subject);
+  }
+  return items;
+}
+
+// Helper to poll the outbox until the most recent email with the expected subject has the expected status.
+// Note: emails are returned ordered by createdAt desc (newest first), so we check emails[0] specifically
+// to ensure we're waiting for the MOST RECENT email, not an older one with the same subject.
+export async function waitForOutboxEmailWithStatus(subject: string, status: string): Promise<OutboxEmail[]> {
+  const maxRetries = 24;
+  let emails: OutboxEmail[] = [];
+  for (let i = 0; i < maxRetries; i++) {
+    emails = await getOutboxEmails({ subject });
+    // Check the most recent email (first in the list due to createdAt desc ordering)
+    if (emails.length > 0 && emails[0].status === status) {
+      return emails;
+    }
+    await wait(500);
+  }
+  throw new StackAssertionError(
+    `Timeout waiting for outbox email with subject "${subject}" and status "${status}"`,
+    { foundEmails: emails }
+  );
+}
+
 export namespace Auth {
   export async function fastSignUp(body: any = {}) {
     const { userId } = await User.create(body);
@@ -405,7 +452,11 @@ export namespace Auth {
         }
         await wait(100 + i * 20);
         if (i >= 30) {
-          throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, { response, messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })) });
+          throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, {
+            response,
+            messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })),
+            outboxEmails: await getOutboxEmails(),
+          });
         }
       }
       return {

apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts

@@ -1,18 +1 @@
-import { niceBackendFetch } from "../../../../backend-helpers";
-
-/**
- * Helper to get emails from the outbox, filtered by subject if provided.
- * Shared across email test files to avoid duplication.
- */
-export async function getOutboxEmails(options?: { subject?: string }) {
-  const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
-    method: "GET",
-    accessType: "server",
-  });
-  if (options?.subject) {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    return listResponse.body.items.filter((e: any) => e.subject === options.subject);
-  }
-  return listResponse.body.items;
-}
-
+export { getOutboxEmails, waitForOutboxEmailWithStatus } from "../../../../backend-helpers";