[Refactor] [Fix] Email Rendering Pipeline Refactor, Error Handling, and Bug Fixes

apps/backend/src/lib/email-rendering.tsx

@@ -1,7 +1,6 @@
-import { executeJavascript } from '@/lib/js-execution';
+import { executeJavascript, type ExecuteResult } from '@/lib/js-execution';
 import { emptyEmailTheme } from '@stackframe/stack-shared/dist/helpers/emails';
-import { getNodeEnvironment } from '@stackframe/stack-shared/dist/utils/env';
-import { captureError, StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors';
+import { StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors';
 import { bundleJavaScript } from '@stackframe/stack-shared/dist/utils/esbuild';
 import { get, has } from '@stackframe/stack-shared/dist/utils/objects';
 import { Result } from "@stackframe/stack-shared/dist/utils/results";
@@ -50,7 +49,7 @@ export function createTemplateComponentFromHtml(html: string) {
 const nodeModules = {
   "react-dom": "19.1.1",
   "react": "19.1.1",
-  "@react-email/components": "0.1.1",
+  "@react-email/components": "1.0.6",
   "arktype": "2.1.20",
 };
 
@@ -70,12 +69,9 @@ const entryJs = deindent`
 `;
 
 type EmailRenderResult = { html: string, text: string, subject?: string, notificationCategory?: string };
-type ExecuteResult =
-  | { status: "ok", data: unknown }
-  | { status: "error", error: unknown };
 
 async function bundleAndExecute<T>(
-  files: Record<string, string> & { '/entry.js': string }
+  files: Record<string, string> & { '/entry.js': string },
 ): Promise<Result<T, string>> {
   const bundle = await bundleJavaScript(files, {
     keepAsImports: ['arktype', 'react', 'react/jsx-runtime', '@react-email/components'],
@@ -87,40 +83,11 @@ async function bundleAndExecute<T>(
     return Result.error(bundle.error);
   }
 
-  if (["development", "test"].includes(getNodeEnvironment())) {
-    const executeResult = await executeJavascript(bundle.data, { nodeModules, engine: 'freestyle' }) as ExecuteResult;
-    if (executeResult.status === "error") {
-      return Result.error(JSON.stringify(executeResult.error));
-    }
-    return Result.ok(executeResult.data as T);
-  }
-
-  const executeResult = await executeJavascript(bundle.data, { nodeModules }) as ExecuteResult;
+  const executeResult: ExecuteResult = await executeJavascript(bundle.data, { nodeModules });
 
   if (executeResult.status === "error") {
-    const vercelResult = await executeJavascript(bundle.data, { nodeModules, engine: 'vercel-sandbox' }) as ExecuteResult;
-    if (vercelResult.status === "error") {
-      captureError("email-rendering-freestyle-and-vercel-runtime-error", new StackAssertionError(
-        "Email rendering failed with both freestyle and vercel-sandbox engines",
-        {
-          freestyleError: executeResult.error,
-          vercelError: vercelResult.error,
-          innerCode: bundle.data,
-          innerOptions: [
-            { nodeModules, engine: 'freestyle' },
-            { nodeModules, engine: 'vercel-sandbox' },
-          ],
-        },
-      ));
-      return Result.error(JSON.stringify(vercelResult.error));
-    }
-    captureError("email-rendering-freestyle-runtime-error", new StackAssertionError(
-      "Email rendering failed with freestyle but succeeded with vercel-sandbox",
-      { freestyleError: executeResult.error, innerCode: bundle.data, innerOptions: { nodeModules, engine: 'freestyle' } }
-    ));
-    return Result.ok(vercelResult.data as T);
+    return Result.error(JSON.stringify(executeResult.error));
   }
-
   return Result.ok(executeResult.data as T);
 }
 

apps/backend/src/lib/js-execution.tsx

@@ -8,18 +8,21 @@ import { Freestyle as FreestyleClient } from 'freestyle-sandboxes';
 
 export type ExecuteJavascriptOptions = {
   nodeModules?: Record<string, string>,
-  engine?: 'freestyle' | 'vercel-sandbox',
 };
 
+export type ExecuteResult =
+  | { status: "ok", data: unknown }
+  | { status: "error", error: { message: string, stack?: string, cause?: unknown } };
+
 type JsEngine = {
   name: string,
-  execute: (code: string, options: ExecuteJavascriptOptions) => Promise<unknown>,
+  execute: (code: string, options: ExecuteJavascriptOptions) => Promise<ExecuteResult>,
 };
 
 function createFreestyleEngine(): JsEngine {
   return {
     name: 'freestyle',
-    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<unknown> => {
+    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> => {
       const apiKey = getEnvVariable("STACK_FREESTYLE_API_KEY");
       let baseUrl = getEnvVariable("STACK_FREESTYLE_API_ENDPOINT", "") || undefined;
 
@@ -45,15 +48,15 @@ function createFreestyleEngine(): JsEngine {
         throw new StackAssertionError("Freestyle execution returned undefined result", { response, innerCode: code, innerOptions: options });
       }
 
-      return response.result;
+      return response.result as ExecuteResult;
     },
   };
 }
 
 function createVercelSandboxEngine(): JsEngine {
   return {
     name: 'vercel-sandbox',
-    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<unknown> => {
+    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> => {
       const teamId = getEnvVariable("STACK_VERCEL_SANDBOX_TEAM_ID", "");
       const projectId = getEnvVariable("STACK_VERCEL_SANDBOX_PROJECT_ID", "");
       const token = getEnvVariable("STACK_VERCEL_SANDBOX_TOKEN", "");
@@ -122,51 +125,66 @@ const engineMap = new Map<string, JsEngine>([
   ['vercel-sandbox', createVercelSandboxEngine()],
 ]);
 
-const engines: JsEngine[] = Array.from(engineMap.values());
-
 /**
  * Executes the given code with the given options. Returns the result of the code execution
  * if it is JSON-serializable. Has undefined behavior if it is not JSON-serializable or if
  * the code throws an error.
  */
-export async function executeJavascript(code: string, options: ExecuteJavascriptOptions = {}): Promise<unknown> {
+export async function executeJavascript(code: string, options: ExecuteJavascriptOptions = {}): Promise<ExecuteResult> {
   return await traceSpan({
     description: 'js-execution.executeJavascript',
     attributes: {
       'js-execution.code.length': code.length.toString(),
       'js-execution.nodeModules.count': options.nodeModules ? Object.keys(options.nodeModules).length.toString() : '0',
-      'js-execution.engine': options.engine ?? 'auto',
     }
   }, async () => {
-    if (options.engine) {
-      const engine = engineMap.get(options.engine);
-      if (!engine) {
-        throw new StackAssertionError(`Unknown JS execution engine: ${options.engine}`);
+
+    if (getEnvVariable("STACK_VERCEL_SANDBOX_TOKEN","") != "") {
+      if (!getNodeEnvironment().includes("prod")) {
+        throw new StackAssertionError("STACK_VERCEL_SANDBOX_TOKEN is set in non-production environment. We do not use Vercel Sandbox in non-production environments.");
       }
-      return await engine.execute(code, options);
-    }
 
-    const shouldSanityTest = Math.random() < 0.05;
+      const shouldSanityTest = Math.random() < 0.05;
+      if (shouldSanityTest) {
+        runAsynchronouslyAndWaitUntil(runSanityTest(code, options));
+      }
 
-    if (shouldSanityTest) {
-      runAsynchronouslyAndWaitUntil(runSanityTest(code, options));
+      return await runWithFallback(code, options);
+    } else {
+      return await runWithoutFallback(code, options);
     }
-
-    // Normal execution: try engines in order with retry for first engine
-    return await runWithFallback(code, options);
   });
 }
 
+/**
+ * Compare two execution results for sanity test equality.
+ * For error results, we only compare status and message (not stack traces,
+ * which differ between execution environments).
+ */
+function areResultsEqual(a: ExecuteResult, b: ExecuteResult): boolean {
+  if (a.status !== b.status) return false;
+
+  if (a.status === 'ok' && b.status === 'ok') {
+    return JSON.stringify(a.data) === JSON.stringify(b.data);
+  }
+
+  if (a.status === 'error' && b.status === 'error') {
+    return a.error.message === b.error.message;
+  }
+
+  return false;
+}
+
 async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
   const results: Array<{ engine: string, result: unknown }> = [];
   const failures: Array<{ engine: string, error: unknown }> = [];
 
-  for (const engine of engines) {
+  for (const [name, engine] of engineMap) {
     try {
       const result = await engine.execute(code, options);
-      results.push({ engine: engine.name, result });
+      results.push({ engine: name, result });
     } catch (error) {
-      failures.push({ engine: engine.name, error });
+      failures.push({ engine: name, error });
     }
   }
 
@@ -181,8 +199,8 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
     return;
   }
 
-  const referenceResult = results[0].result;
-  const allEqual = results.every(r => JSON.stringify(r.result) === JSON.stringify(referenceResult));
+  const referenceResult = results[0].result as ExecuteResult;
+  const allEqual = results.every(r => areResultsEqual(r.result as ExecuteResult, referenceResult));
   if (!allEqual) {
     captureError("js-execution-sanity-test-mismatch", new StackAssertionError(
       "JS execution sanity test: engines returned different results",
@@ -191,19 +209,15 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
   }
 }
 
-async function runWithFallback(code: string, options: ExecuteJavascriptOptions): Promise<unknown> {
-  const errors: Array<{ engine: string, error: unknown }> = [];
+async function runWithFallback(code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> {
+  const freestyleEngine = engineMap.get("freestyle")!;
+  const vercelSandboxEngine = engineMap.get("vercel-sandbox")!;
 
-  for (let i = 0; i < engines.length; i++) {
-    const engine = engines[i];
-    const isFirstEngine = i === 0;
-
-    const maxAttempts = isFirstEngine ? 2 : 1;
-
-    const retryResult = await Result.retry(
+  const maxAttempts = 2;
+  const retryResult = await Result.retry(
       async () => {
         try {
-          const result = await engine.execute(code, options);
+          const result = await freestyleEngine.execute(code, options);
           return Result.ok(result);
         } catch (error) {
           return Result.error(error);
@@ -213,20 +227,33 @@ async function runWithFallback(code: string, options: ExecuteJavascriptOptions):
       { exponentialDelayBase: 500 }
     );
 
-    if (retryResult.status === 'ok') {
-      return retryResult.data;
-    }
-
-    const engineError = retryResult.error;
-    errors.push({ engine: engine.name, error: engineError });
+  if (retryResult.status === 'ok') {
+    return retryResult.data;
+  }
 
-    if (i < engines.length - 1) {
-      captureError(`js-execution-${engine.name}-failed`, new StackAssertionError(
-        `JS execution engine '${engine.name}' failed, falling back to next engine`,
-        { error: engineError, attempts: retryResult.attempts, innerCode: code, innerOptions: options }
+  captureError(`js-execution-freestyle-failed`, new StackAssertionError(
+    `JS execution freestyle engine failed, falling back to vercel sandbox engine`,
+    { error: retryResult.error, innerCode: code, innerOptions: options }
+  ));
+
+  try {
+    const result = await vercelSandboxEngine.execute(code, options);
+    return result;
+  } catch (error){
+      captureError(`js-execution-vercel-sandbox-failed`, new StackAssertionError(
+        `JS execution vercel sandbox engine failed after fallback from freestyle engine`,
+        { error: error, innerCode: code, innerOptions: options }
       ));
-    }
+      throw new StackAssertionError("Email rendering service unavailable", { cause: error, innerCode: code, innerOptions: options });
   }
+}
 
-  throw errors[errors.length - 1].error;
+async function runWithoutFallback(code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> {
+  const freestyleEngine = engineMap.get("freestyle")!;
+  try {
+    const result = await freestyleEngine.execute(code, options);
+    return result;
+  } catch (error) {
+    throw new StackAssertionError("Email rendering service unavailable", { cause: error, innerCode: code, innerOptions: options });
+  }
 }