From 4f91839b144d10117cc5b962efb9b0bdd873a75b Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Mon, 23 Mar 2026 19:50:51 -0700 Subject: [PATCH 01/40] Bulldozer DB --- .vscode/settings.json | 13 +- .../migration.sql | 23 + .../tests/ltree-queries.ts | 102 ++++ apps/backend/prisma/schema.prisma | 14 + .../src/lib/bulldozer/bulldozer-schema.ts | 0 .../src/lib/bulldozer/db/index.test.ts | 492 ++++++++++++++++++ apps/backend/src/lib/bulldozer/db/index.ts | 268 ++++++++++ 7 files changed, 907 insertions(+), 5 deletions(-) create mode 100644 apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql create mode 100644 apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts create mode 100644 apps/backend/src/lib/bulldozer/bulldozer-schema.ts create mode 100644 apps/backend/src/lib/bulldozer/db/index.test.ts create mode 100644 apps/backend/src/lib/bulldozer/db/index.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 9937c7581f..72e3892e83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,10 +7,6 @@ "typescript.tsdk": "node_modules/typescript/lib", "editor.tabSize": 2, "cSpell.words": [ - "glassmorphic", - "sparkline", - "Clickhouse", - "pushable", "autoupdate", "backlinks", "Cancelation", @@ -19,14 +15,15 @@ "chinthakagodawita", "Ciphertext", "cjsx", + "Clickhouse", "clsx", - "dbgenerated", "cmdk", "codegen", "crockford", "Crudl", "ctsx", "datapoints", + "dbgenerated", "deindent", "Deindentable", "deindented", @@ -42,6 +39,7 @@ "fkey", "frontends", "geoip", + "glassmorphic", "healthcheck", "hookform", "hostable", @@ -51,6 +49,7 @@ "JWTs", "katex", "localstack", + "ltree", "lucide", "Luma", "midfix", @@ -64,6 +63,7 @@ "nicified", "nicify", "oidc", + "onnotice", "openapi", "opentelemetry", "otel", @@ -81,6 +81,7 @@ "Prefetchers", "Proxied", "psql", + "pushable", "qrcode", "QSTASH", "quetzallabs", @@ -89,6 +90,7 @@ "retryable", "RPID", "simplewebauthn", + "sparkline", "spoofable", "stackable", "stackauth", @@ -109,6 +111,7 @@ "Unmigrated", "unsubscribers", "upsert", + "upserted", "Upvotes", "upvoting", "webapi", diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql new file mode 100644 index 0000000000..042bd7729e --- /dev/null +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql @@ -0,0 +1,23 @@ +-- CreateTable +CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL, + "keyPath" TEXT[] NOT NULL, + "keyPathParent" TEXT[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") > 1 THEN "keyPath"[1:cardinality("keyPath") - 1] + ELSE "keyPath" + END + ) STORED, + "value" JSONB NOT NULL, + + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPath_non_empty_check" CHECK (cardinality("keyPath") >= 1), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE +); + +-- CreateIndex +CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent"); diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts new file mode 100644 index 0000000000..15aa9bd916 --- /dev/null +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts @@ -0,0 +1,102 @@ +import type { Sql } from "postgres"; +import { expect } from "vitest"; + +export const postMigration = async (sql: Sql) => { + await sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + ('00000000-0000-0000-0000-000000000001'::uuid, ARRAY['root']::text[], '{"node":"root"}'::jsonb), + ('00000000-0000-0000-0000-000000000002'::uuid, ARRAY['root', 'branch']::text[], '{"node":"branch"}'::jsonb), + ('00000000-0000-0000-0000-000000000003'::uuid, ARRAY['root', 'branch', 'leaf']::text[], '{"node":"leaf"}'::jsonb), + ('00000000-0000-0000-0000-000000000004'::uuid, ARRAY['root', 'other']::text[], '{"node":"other"}'::jsonb) + `; + + const exactRows = await sql` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY['root', 'branch', 'leaf']::text[] + `; + + expect(exactRows).toHaveLength(1); + expect(exactRows[0].value).toEqual({ node: "leaf" }); + + const nestedRows = await sql` + SELECT array_to_string("keyPath", '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath"[1:cardinality(ARRAY['root', 'branch']::text[])] = ARRAY['root', 'branch']::text[] + ORDER BY "keyPath" + `; + + expect(nestedRows.map((row) => row.keyPath)).toEqual([ + "root.branch", + "root.branch.leaf", + ]); + + const directChildrenRows = await sql` + SELECT array_to_string("keyPath", '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ARRAY['root']::text[] + AND cardinality("keyPath") = cardinality(ARRAY['root']::text[]) + 1 + ORDER BY "keyPath" + `; + + expect(directChildrenRows.map((row) => row.keyPath)).toEqual([ + "root.branch", + "root.other", + ]); + + const indexRows = await sql` + SELECT "indexname" + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = 'BulldozerStorageEngine' + AND indexname IN ( + 'BulldozerStorageEngine_keyPath_key', + 'BulldozerStorageEngine_keyPathParent_idx' + ) + ORDER BY "indexname" + `; + + expect(indexRows.map((row) => row.indexname)).toEqual([ + "BulldozerStorageEngine_keyPathParent_idx", + "BulldozerStorageEngine_keyPath_key", + ]); + + const generatedColumnRows = await sql` + SELECT attname + FROM pg_attribute + WHERE attrelid = 'public."BulldozerStorageEngine"'::regclass + AND attname = 'keyPathParent' + AND attgenerated = 's' + `; + + expect(generatedColumnRows).toHaveLength(1); + + const fkConstraintRows = await sql` + SELECT conname + FROM pg_constraint + WHERE conrelid = 'public."BulldozerStorageEngine"'::regclass + AND conname = 'BulldozerStorageEngine_keyPathParent_fkey' + `; + + expect(fkConstraintRows).toHaveLength(1); + + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "keyPathParent", "value") + VALUES ( + '00000000-0000-0000-0000-000000000005'::uuid, + ARRAY['root', 'mismatch']::text[], + ARRAY[]::text[], + '{"node":"invalid"}'::jsonb + ) + `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); + + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + '00000000-0000-0000-0000-000000000006'::uuid, + ARRAY['missing-parent', 'child']::text[], + '{"node":"invalid-fk"}'::jsonb + ) + `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); +}; diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index ef7b4b6ae9..e143c6689d 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -1244,6 +1244,20 @@ model OutgoingRequest { @@index([startedFulfillingAt, deduplicationKey]) } +model BulldozerStorageEngine { + id String @id @default(uuid()) @db.Uuid + keyPath String[] + keyPathParent String[] + value Json + + @@unique([keyPath], map: "BulldozerStorageEngine_keyPath_key") + // keyPathParent is a generated column in the SQL migration: + // CASE WHEN cardinality(keyPath) > 1 THEN keyPath[1:cardinality(keyPath)-1] ELSE keyPath END + // It also has a self-referencing foreign key to keyPath. + // Prisma schema cannot currently express this generated column + self-reference setup on arrays. + @@index([keyPathParent], map: "BulldozerStorageEngine_keyPathParent_idx") +} + model DeletedRow { id String @id @default(uuid()) @db.Uuid tenancyId String @db.Uuid diff --git a/apps/backend/src/lib/bulldozer/bulldozer-schema.ts b/apps/backend/src/lib/bulldozer/bulldozer-schema.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts new file mode 100644 index 0000000000..f0addb9745 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.test.ts @@ -0,0 +1,492 @@ +import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; +import postgres from "postgres"; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest"; +import { declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_test"; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string }; + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} + +const sqlStringLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`; +const sqlStatement = (strings: TemplateStringsArray, ...values: { sql: string }[]): SqlStatement => ({ + type: "statement", + sql: templateIdentity(strings, ...values.map((value) => value.sql)), +}); + +describe.sequential("declareStoredTable (real postgres)", () => { + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + + async function runStatements(statements: SqlStatement[]) { + await sql.unsafe(toExecutableSqlTransaction(statements)); + } + + async function readBoolean(expression: SqlExpression) { + const rows = await sql.unsafe(`SELECT (${expression.sql}) AS "value"`); + return rows[0].value === true; + } + + async function readRows(query: SqlQuery) { + return await sql.unsafe(toQueryableSqlQuery(query)); + } + + async function readTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerTriggerAudit" + ORDER BY "id" + `); + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerTriggerAudit"`; + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" TEXT[] NOT NULL, + "keyPathParent" TEXT[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") > 1 THEN "keyPath"[1:cardinality("keyPath") - 1] + ELSE "keyPath" + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPath_non_empty_check" CHECK (cardinality("keyPath") >= 1), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + CREATE TABLE "BulldozerTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + function registerAuditTrigger( + table: ReturnType>, + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerTriggerAudit" ( + "event", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + + test("init/isInitialized/delete lifecycle", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + expect(await readBoolean(table.isInitialized())).toBe(false); + await runStatements(table.init()); + expect(await readBoolean(table.isInitialized())).toBe(true); + await runStatements(table.delete()); + expect(await readBoolean(table.isInitialized())).toBe(false); + }); + + test("trigger emits insert change row", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "insert"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "insert", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("trigger emits update change row with old and new values", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "update"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.setRow("alpha", expr(`'{"value":2}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "update", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "update", + rowIdentifier: "alpha", + oldRowData: { value: 1 }, + newRowData: { value: 2 }, + }, + ]); + }); + + test("trigger emits delete change row only when row existed", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "delete"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.deleteRow("missing")); + await runStatements(table.deleteRow("alpha")); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "delete", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "delete", + rowIdentifier: "alpha", + oldRowData: { value: 1 }, + newRowData: null, + }, + ]); + }); + + test("deregistered trigger no longer runs", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + const handle = registerAuditTrigger(table, "deregister"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + handle.deregister(); + await runStatements(table.setRow("beta", expr(`'{"value":2}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "deregister", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("multiple triggers run in one transaction", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "trigger_a"); + registerAuditTrigger(table, "trigger_b"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + expect((await readTriggerAuditRows()).sort((a, b) => stringCompare(a.event, b.event))).toEqual([ + { + event: "trigger_a", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "trigger_b", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("setRow upserts and listRowsInGroup returns raw identifiers", async () => { + const table = declareStoredTable<{ value: number, label: string }>({ tableId: "users" }); + const weirdIdentifier = "row.with/slash and spaces"; + + await runStatements(table.init()); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":1,"label":"first"}'::jsonb`))); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":2,"label":"second"}'::jsonb`))); + await runStatements(table.setRow("plain-row", expr(`'{"value":3,"label":"third"}'::jsonb`))); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + + const mapped = rows + .map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + + expect(mapped).toEqual([ + { + rowIdentifier: "plain-row", + rowData: { label: "third", value: 3 }, + }, + { + rowIdentifier: weirdIdentifier, + rowData: { label: "second", value: 2 }, + }, + ]); + }); + + test("table contents snapshot after init + upserts", async () => { + const table = declareStoredTable<{ value: number, label: string }>({ tableId: "users" }); + const weirdIdentifier = "row.with/slash and spaces"; + + await runStatements(table.init()); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":1,"label":"first"}'::jsonb`))); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":2,"label":"second"}'::jsonb`))); + await runStatements(table.setRow("plain-row", expr(`'{"value":3,"label":"third"}'::jsonb`))); + + const rows = await sql.unsafe(` + SELECT array_to_string("keyPath", ' -> ') AS "keyPath", "value" + FROM "BulldozerStorageEngine" + ORDER BY "keyPath" + `); + const snapshotRows = [...rows].map((row) => ({ keyPath: row.keyPath, value: row.value })); + + expect(snapshotRows).toMatchInlineSnapshot(` + [ + { + "keyPath": "table", + "value": null, + }, + { + "keyPath": "table -> external:users", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage -> metadata", + "value": { + "version": 1, + }, + }, + { + "keyPath": "table -> external:users -> storage -> rows", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage -> rows -> plain-row", + "value": { + "rowData": { + "label": "third", + "value": 3, + }, + }, + }, + { + "keyPath": "table -> external:users -> storage -> rows -> row.with/slash and spaces", + "value": { + "rowData": { + "label": "second", + "value": 2, + }, + }, + }, + ] + `); + }); + + test("generated keyPathParent rejects explicit writes", async () => { + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "keyPathParent", "value") + VALUES ( + ARRAY['table', 'external:users', 'storage', 'rows', 'x']::text[], + ARRAY['table', 'external:users', 'storage']::text[], + '{"rowData":{"value":1}}'::jsonb + ) + `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); + }); + + test("keyPathParent foreign key rejects missing parent rows", async () => { + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES ( + ARRAY['missing-parent', 'child']::text[], + '{"rowData":{"value":1}}'::jsonb + ) + `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); + }); + + test("deleteRow removes only the target row and missing rows are no-op", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + + await runStatements(table.init()); + await runStatements(table.setRow("a", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.setRow("b", expr(`'{"value":2}'::jsonb`))); + await runStatements(table.deleteRow("missing")); + await runStatements(table.deleteRow("a")); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(rows).toHaveLength(1); + expect(rows[0].rowdata).toEqual({ value: 2 }); + expect(await readBoolean(table.isInitialized())).toBe(true); + }); + + test("exclusive start/end excludes the single null group and rowSortKey", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("row", expr(`'{"value":1}'::jsonb`))); + + const groups = await readRows(table.listGroups({ + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: false, + endInclusive: false, + })); + expect(groups).toHaveLength(0); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: false, + endInclusive: false, + })); + expect(rows).toHaveLength(0); + }); + + test("table paths are isolated by tableId", async () => { + const left = declareStoredTable<{ value: number }>({ tableId: "left" }); + const right = declareStoredTable<{ value: number }>({ tableId: "right" }); + + await runStatements(left.init()); + await runStatements(right.init()); + await runStatements(left.setRow("shared", expr(`'{"value":1}'::jsonb`))); + await runStatements(right.setRow("shared", expr(`'{"value":2}'::jsonb`))); + await runStatements(left.delete()); + + const rightRows = await readRows(right.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + + expect(await readBoolean(left.isInitialized())).toBe(false); + expect(await readBoolean(right.isInitialized())).toBe(true); + expect(rightRows).toHaveLength(1); + expect(rightRows[0].rowdata).toEqual({ value: 2 }); + }); + + test("rowIdentifier from listRowsInGroup can be passed to deleteRow", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("plain-row", expr(`'{"value":1}'::jsonb`))); + + const listedRows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(listedRows).toHaveLength(1); + + await runStatements(table.deleteRow(listedRows[0].rowidentifier)); + + const remainingRows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(remainingRows).toHaveLength(0); + }); + + test("toExecutableSqlTransaction handles empty statements", async () => { + await runStatements([]); + }); + + test("toQueryableSqlQuery returns executable SQL", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + const query = table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + }); + const rows = await sql.unsafe(toQueryableSqlQuery(query)); + expect(rows).toHaveLength(1); + expect(rows[0].rowdata).toEqual({ value: 1 }); + }); +}); diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts new file mode 100644 index 0000000000..4a5c51b904 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.ts @@ -0,0 +1,268 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { deindent, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; + +export type Table = { + tableId: TableId, + + // Query groups and rows + listGroups(options: { start: SqlExpression, end: SqlExpression, startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + listRowsInGroup(options: { groupKey: SqlExpression, start: SqlExpression, end: SqlExpression, startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + + // Sorting and grouping + compareGroupKeys(a: SqlExpression, b: SqlExpression): SqlExpression, + compareSortKeys(a: SqlExpression, b: SqlExpression): SqlExpression, + + // Lifecycle/migration methods + /** Called when the table should be created on the storage engine. */ + init(): SqlStatement[], + /** Called when the table should be deleted from the storage engine. */ + delete(): SqlStatement[], + isInitialized(): SqlExpression, + + // Internal methods, used only by table constructors to create relationships between them + /** + * @param trigger A SQL statement that can reference the changes table with columns `groupKey: GK`, `rowIdentifier: RowIdentifier`, `oldRowSortKey: SK | null`, `newRowSortKey: SK | null`, `oldRowData: RowData | null`, `newRowData: RowData | null`. Note that this trigger should be a no-op if the table that created this trigger is not initialized. + */ + registerRowChangeTrigger(trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[]): { deregister: () => void }, +}; + +export function declareStoredTable(options: { + tableId: TableId, +}): Table & { + setRow(rowIdentifier: RowIdentifier, rowData: SqlExpression): SqlStatement[], + deleteRow(rowIdentifier: RowIdentifier): SqlStatement[], +} { + const triggers = new Map) => SqlStatement[]>(); + const ensureRowsHierarchyStatement = createInsertPathRowsStatement(options.tableId, ["rows"]); + + // Note that this table has only one group and sort key (null), so all groups and rows are always returned by every filter. + return { + tableId: options.tableId, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT 'null'::jsonb AS groupKey + WHERE ${startInclusive && endInclusive ? sqlExpression`1 = 1` : sqlExpression`1 = 0`} + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT + "keyPath"[cardinality("keyPath")] AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["rows"])}::text[] + AND ${startInclusive && endInclusive ? sqlExpression`1 = 1` : sqlExpression`1 = 0`} + `, + compareGroupKeys: (a, b) => sqlExpression` 0 `, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => [ensureRowsHierarchyStatement, sqlStatement` + -- Create metadata about the table. + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES ( + ${getStorageEnginePath(options.tableId, ["metadata"])}::text[], + '{ "version": 1 }'::jsonb + ) + `], + delete: () => [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getStorageEnginePath(options.tableId, [])}::text[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `], + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::text[] + ) + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + setRow: (rowIdentifier, rowData) => { + const oldRowsTableName = `old_rows_${generateSecureRandomString()}`; + const upsertedRowsTableName = `upserted_rows_${generateSecureRandomString()}`; + const changesTableName = `changes_${generateSecureRandomString()}`; + const rowIdentifierLiteral = quoteSqlStringLiteral(rowIdentifier); + const rowValue = sqlExpression` + jsonb_build_object( + 'rowData', ${rowData}::jsonb + ) + `; + return [ + ensureRowsHierarchyStatement, + sqlQuery` + SELECT "value"->'rowData' AS "oldRowData" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[] + `.toStatement(oldRowsTableName), + sqlQuery` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES ( + ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[], + ${rowValue}::jsonb + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = ${rowValue}::jsonb + RETURNING "value"->'rowData' AS "newRowData" + `.toStatement(upsertedRowsTableName), + sqlQuery` + SELECT + 'null'::jsonb AS "groupKey", + ${rowIdentifierLiteral}::text AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + ${quoteSqlIdentifier(oldRowsTableName)}."oldRowData" AS "oldRowData", + ${quoteSqlIdentifier(upsertedRowsTableName)}."newRowData" AS "newRowData" + FROM ${quoteSqlIdentifier(upsertedRowsTableName)} + LEFT JOIN ${quoteSqlIdentifier(oldRowsTableName)} ON true + `.toStatement(changesTableName), + ...[...triggers.values()].flatMap(trigger => trigger(quoteSqlIdentifier(changesTableName))) + ]; + }, + deleteRow: (rowIdentifier) => { + const deletedRowsTableName = `deleted_rows_${generateSecureRandomString()}`; + const changesTableName = `changes_${generateSecureRandomString()}`; + const rowIdentifierLiteral = quoteSqlStringLiteral(rowIdentifier); + return [ + sqlQuery` + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[] + RETURNING "value"->'rowData' AS "oldRowData" + `.toStatement(deletedRowsTableName), + sqlQuery` + SELECT + 'null'::jsonb AS "groupKey", + ${rowIdentifierLiteral}::text AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + ${quoteSqlIdentifier(deletedRowsTableName)}."oldRowData" AS "oldRowData", + 'null'::jsonb AS "newRowData" + FROM ${quoteSqlIdentifier(deletedRowsTableName)} + `.toStatement(changesTableName), + ...[...triggers.values()].flatMap(trigger => trigger(quoteSqlIdentifier(changesTableName))) + ]; + }, + }; +} + +declare function declareMapTable< + GK extends Json, + SK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + mapper: SqlMapper, +}): Table; + +declare function declareFilterTable< + GK extends Json, + SK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + filter: SqlPredicate, +}): Table; + + +// ====== Executing SQL Statements ====== +const BULLDOZER_LOCK_ID = 7857391; // random number to avoid conflicts with other applications +export function toQueryableSqlQuery(query: SqlQuery): string { + return query.sql; +} +export function toExecutableSqlTransaction(statements: SqlStatement[]): string { + return deindent` + BEGIN; + + SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID}); + + WITH __dummy_statement_1__ AS (SELECT 1), + ${statements.map(statement => deindent` + ${quoteSqlIdentifier(statement.outputName ?? `unnamed_statement_${generateSecureRandomString().slice(0, 8)}`).sql} AS ( + ${statement.sql} + ), + `).join("\n")} + __dummy_statement_2__ AS (SELECT 1) + SELECT 1; + + COMMIT; + `; +} + +// ====== Utilities ====== +const sqlTemplateLiteral = (type: T) => (strings: TemplateStringsArray, ...values: { sql: string }[]) => ({ type, sql: templateIdentity(strings, ...values.map(v => v.sql)) }); +type SqlStatement = { type: "statement", outputName?: string, sql: string }; +const sqlStatement = sqlTemplateLiteral<"statement">("statement"); +type SqlQuery = void> = { type: "query", outputName?: string, sql: string }; +const sqlQuery = (...args: Parameters>>) => { + return { + ...sqlTemplateLiteral<"query">("query")(...args), + toStatement(outputName?: string) { + return { type: "statement", outputName, sql: this.sql } as const; + } + }; +}; +type SqlExpression = { type: "expression", sql: string }; +const sqlExpression = sqlTemplateLiteral<"expression">("expression"); +type SqlMapper = { type: "mapper", sql: string }; // ex.: "row.id AS id, row.old_value + 1 AS new_value" +const sqlMapper = sqlTemplateLiteral<"mapper">("mapper"); +type SqlPredicate = { type: "predicate", sql: string }; // ex.: "user_id = 123" +const sqlPredicate = sqlTemplateLiteral<"predicate">("predicate"); +type RowData = Record; +type Json = string | number | boolean | null | { [key: string]: Json } | Json[]; +type RowIdentifier = string; +type TableId = string | { "tableType": "internal", "internalId": string, "parent": null | TableId }; +function quoteSqlIdentifier(input: string): SqlExpression { + if (input.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) == null) { + throw new StackAssertionError("Invalid SQL identifier", { input }); + } + return { type: "expression", sql: `"${input}"` }; +} +function quoteSqlStringLiteral(input: string): SqlExpression { + return { type: "expression", sql: `'${input.replaceAll("'", "''")}'` }; +} +function getStorageEnginePath(tableId: TableId, path: string[]): SqlExpression { + return pathSegmentsToSqlExpression(getStorageEnginePathSegments(tableId, path)); +} +function getStorageEnginePathSegments(tableId: TableId, path: string[]): string[] { + const tableIdWithParents = []; + let currentTableId = tableId; + while (true) { + if (typeof currentTableId === "string") { + tableIdWithParents.push(`external:${currentTableId}`); + break; + } else { + tableIdWithParents.push(`internal:${currentTableId.internalId}`); + if (currentTableId.parent === null) break; + currentTableId = currentTableId.parent; + } + } + return [...tableIdWithParents.reverse().flatMap(id => ["table", id]), "storage", ...path]; +} +function pathSegmentsToSqlExpression(pathSegments: string[]): SqlExpression { + return { + type: "expression", + sql: `ARRAY[${pathSegments.map((segment) => quoteSqlStringLiteral(segment).sql).join(", ")}]::text[]`, + }; +} +function createInsertPathRowsStatement(tableId: TableId, path: string[]): SqlStatement { + const segments = getStorageEnginePathSegments(tableId, path); + const uniquePathSqlExpressions = [...new Set(segments.map((_, idx) => pathSegmentsToSqlExpression(segments.slice(0, idx + 1)).sql))]; + return { + type: "statement", + sql: deindent` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + ${uniquePathSqlExpressions.map((pathSql) => `(${pathSql}, 'null'::jsonb)`).join(",\n")} + ON CONFLICT ("keyPath") DO NOTHING + `, + }; +} From 9694c3377f993704b759b0d795e27a5aa4f2347f Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 10:11:50 -0700 Subject: [PATCH 02/40] declareGroupByTable --- .../migration.sql | 18 +- .../tests/ltree-queries.ts | 37 +- apps/backend/prisma/schema.prisma | 65 ++- .../src/lib/bulldozer/db/index.test.ts | 378 ++++++++++++++++- apps/backend/src/lib/bulldozer/db/index.ts | 380 +++++++++++++++--- 5 files changed, 756 insertions(+), 122 deletions(-) diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql index 042bd7729e..18a23438e8 100644 --- a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql @@ -1,23 +1,31 @@ -- CreateTable CREATE TABLE "BulldozerStorageEngine" ( "id" UUID NOT NULL, - "keyPath" TEXT[] NOT NULL, - "keyPathParent" TEXT[] GENERATED ALWAYS AS ( + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( CASE - WHEN cardinality("keyPath") > 1 THEN "keyPath"[1:cardinality("keyPath") - 1] - ELSE "keyPath" + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] END ) STORED, "value" JSONB NOT NULL, CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), - CONSTRAINT "BulldozerStorageEngine_keyPath_non_empty_check" CHECK (cardinality("keyPath") >= 1), CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" FOREIGN KEY ("keyPathParent") REFERENCES "BulldozerStorageEngine"("keyPath") ON DELETE CASCADE ); +-- Seed root hierarchy rows used by all tables. +INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") +VALUES + ('00000000-0000-0000-0000-000000000100'::uuid, ARRAY[]::jsonb[], 'null'::jsonb); + +INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") +VALUES + ('00000000-0000-0000-0000-000000000101'::uuid, ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb); + -- CreateIndex CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent"); diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts index 15aa9bd916..bfcb577b1e 100644 --- a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts @@ -5,25 +5,25 @@ export const postMigration = async (sql: Sql) => { await sql` INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") VALUES - ('00000000-0000-0000-0000-000000000001'::uuid, ARRAY['root']::text[], '{"node":"root"}'::jsonb), - ('00000000-0000-0000-0000-000000000002'::uuid, ARRAY['root', 'branch']::text[], '{"node":"branch"}'::jsonb), - ('00000000-0000-0000-0000-000000000003'::uuid, ARRAY['root', 'branch', 'leaf']::text[], '{"node":"leaf"}'::jsonb), - ('00000000-0000-0000-0000-000000000004'::uuid, ARRAY['root', 'other']::text[], '{"node":"other"}'::jsonb) + ('00000000-0000-0000-0000-000000000001'::uuid, ARRAY[to_jsonb('root'::text)]::jsonb[], '{"node":"root"}'::jsonb), + ('00000000-0000-0000-0000-000000000002'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[], '{"node":"branch"}'::jsonb), + ('00000000-0000-0000-0000-000000000003'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[], '{"node":"leaf"}'::jsonb), + ('00000000-0000-0000-0000-000000000004'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('other'::text)]::jsonb[], '{"node":"other"}'::jsonb) `; const exactRows = await sql` SELECT "value" FROM "BulldozerStorageEngine" - WHERE "keyPath" = ARRAY['root', 'branch', 'leaf']::text[] + WHERE "keyPath" = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[] `; expect(exactRows).toHaveLength(1); expect(exactRows[0].value).toEqual({ node: "leaf" }); const nestedRows = await sql` - SELECT array_to_string("keyPath", '.') AS "keyPath" + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" FROM "BulldozerStorageEngine" - WHERE "keyPath"[1:cardinality(ARRAY['root', 'branch']::text[])] = ARRAY['root', 'branch']::text[] + WHERE "keyPath"[1:cardinality(ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[])] = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[] ORDER BY "keyPath" `; @@ -33,10 +33,9 @@ export const postMigration = async (sql: Sql) => { ]); const directChildrenRows = await sql` - SELECT array_to_string("keyPath", '.') AS "keyPath" + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" FROM "BulldozerStorageEngine" - WHERE "keyPathParent" = ARRAY['root']::text[] - AND cardinality("keyPath") = cardinality(ARRAY['root']::text[]) + 1 + WHERE "keyPathParent" = ARRAY[to_jsonb('root'::text)]::jsonb[] ORDER BY "keyPath" `; @@ -62,6 +61,18 @@ export const postMigration = async (sql: Sql) => { "BulldozerStorageEngine_keyPath_key", ]); + const seededRootRows = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (ARRAY[]::jsonb[], ARRAY[to_jsonb('table'::text)]::jsonb[]) + ORDER BY cardinality("keyPath") + `; + + expect(seededRootRows.map((row) => row.keyPath)).toEqual([ + "", + "table", + ]); + const generatedColumnRows = await sql` SELECT attname FROM pg_attribute @@ -85,8 +96,8 @@ export const postMigration = async (sql: Sql) => { INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "keyPathParent", "value") VALUES ( '00000000-0000-0000-0000-000000000005'::uuid, - ARRAY['root', 'mismatch']::text[], - ARRAY[]::text[], + ARRAY[to_jsonb('root'::text), to_jsonb('mismatch'::text)]::jsonb[], + ARRAY[]::jsonb[], '{"node":"invalid"}'::jsonb ) `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); @@ -95,7 +106,7 @@ export const postMigration = async (sql: Sql) => { INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") VALUES ( '00000000-0000-0000-0000-000000000006'::uuid, - ARRAY['missing-parent', 'child']::text[], + ARRAY[to_jsonb('missing-parent'::text), to_jsonb('child'::text)]::jsonb[], '{"node":"invalid-fk"}'::jsonb ) `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index e143c6689d..8e3dba15fc 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -69,10 +69,10 @@ model Tenancy { branchId String // If organizationId is NULL, hasNoOrganization must be TRUE. If organizationId is not NULL, hasNoOrganization must be NULL. - organizationId String? @db.Uuid - hasNoOrganization BooleanTrue? - emailOutboxes EmailOutbox[] - sessionReplays SessionReplay[] + organizationId String? @db.Uuid + hasNoOrganization BooleanTrue? + emailOutboxes EmailOutbox[] + sessionReplays SessionReplay[] sessionReplayChunks SessionReplayChunk[] managedEmailDomains ManagedEmailDomain[] @@ -94,24 +94,24 @@ enum ManagedEmailDomainStatus { model ManagedEmailDomain { id String @id @default(uuid()) @db.Uuid - tenancyId String @db.Uuid - tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) + tenancyId String @db.Uuid + tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) projectId String - branchId String + branchId String - subdomain String - senderLocalPart String - resendDomainId String @unique + subdomain String + senderLocalPart String + resendDomainId String @unique nameServerRecords Json - status ManagedEmailDomainStatus @default(PENDING_VERIFICATION) + status ManagedEmailDomainStatus @default(PENDING_VERIFICATION) providerStatusRaw String? - isActive Boolean @default(true) - lastError String? - verifiedAt DateTime? - appliedAt DateTime? - lastWebhookAt DateTime? + isActive Boolean @default(true) + lastError String? + verifiedAt DateTime? + appliedAt DateTime? + lastWebhookAt DateTime? createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -276,15 +276,15 @@ model ProjectUser { restrictedByAdminPrivateDetails String? // Private details (server access only) // Sign-up metadata - signedUpAt DateTime - signUpIp String? - signUpIpTrusted Boolean? - signUpEmailNormalized String? - signUpEmailBase String? + signedUpAt DateTime + signUpIp String? + signUpIpTrusted Boolean? + signUpEmailNormalized String? + signUpEmailBase String? // Sign-up risk scores (0-100, set at sign-up time) - signUpRiskScoreBot Int @default(0) @db.SmallInt - signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt + signUpRiskScoreBot Int @default(0) @db.SmallInt + signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt projectUserOAuthAccounts ProjectUserOAuthAccount[] teamMembers TeamMember[] @@ -367,18 +367,18 @@ model SessionReplay { chunks SessionReplayChunk[] @@id([tenancyId, id]) - @@map("SessionReplay") @@index([tenancyId, projectUserId, startedAt]) @@index([tenancyId, lastEventAt]) // index by updatedAt instead of lastEventAt because event timing can be spoofed @@index([tenancyId, refreshTokenId, updatedAt]) + @@map("SessionReplay") } model SessionReplayChunk { id String @id @default(uuid()) @db.Uuid - tenancyId String @db.Uuid - sessionReplayId String @db.Uuid @map("sessionReplayId") + tenancyId String @db.Uuid + sessionReplayId String @map("sessionReplayId") @db.Uuid // Unique per uploaded batch for a given session id. batchId String @db.Uuid @@ -402,8 +402,8 @@ model SessionReplayChunk { tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@unique([tenancyId, sessionReplayId, batchId]) - @@map("SessionReplayChunk") @@index([tenancyId, sessionReplayId, createdAt]) + @@map("SessionReplayChunk") } enum ContactChannelType { @@ -1245,17 +1245,14 @@ model OutgoingRequest { } model BulldozerStorageEngine { - id String @id @default(uuid()) @db.Uuid - keyPath String[] - keyPathParent String[] + id String @id @default(uuid()) @db.Uuid + keyPath Json[] + keyPathParent Unsupported("jsonb[]") value Json @@unique([keyPath], map: "BulldozerStorageEngine_keyPath_key") - // keyPathParent is a generated column in the SQL migration: - // CASE WHEN cardinality(keyPath) > 1 THEN keyPath[1:cardinality(keyPath)-1] ELSE keyPath END - // It also has a self-referencing foreign key to keyPath. - // Prisma schema cannot currently express this generated column + self-reference setup on arrays. @@index([keyPathParent], map: "BulldozerStorageEngine_keyPathParent_idx") + @@ignore } model DeletedRow { diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts index f0addb9745..74dae4fe97 100644 --- a/apps/backend/src/lib/bulldozer/db/index.test.ts +++ b/apps/backend/src/lib/bulldozer/db/index.test.ts @@ -1,7 +1,7 @@ import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; import postgres from "postgres"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest"; -import { declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; +import { declareGroupByTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; type TestDb = { full: string, base: string }; @@ -24,11 +24,15 @@ function getTestDbUrls(): TestDb { type SqlExpression = { type: "expression", sql: string }; type SqlStatement = { type: "statement", sql: string, outputName?: string }; -type SqlQuery = { type: "query", sql: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; +type SqlMapper = { type: "mapper", sql: string }; function expr(sql: string): SqlExpression { return { type: "expression", sql }; } +function mapper(sql: string): SqlMapper { + return { type: "mapper", sql }; +} const sqlStringLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`; const sqlStatement = (strings: TemplateStringsArray, ...values: { sql: string }[]): SqlStatement => ({ @@ -66,6 +70,18 @@ describe.sequential("declareStoredTable (real postgres)", () => { ORDER BY "id" `); } + async function readGroupTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "groupKey"#>>'{}' AS "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerGroupTriggerAudit" + ORDER BY "id" + `); + } beforeAll(async () => { await adminSql.unsafe(`CREATE DATABASE ${dbName}`); @@ -73,22 +89,22 @@ describe.sequential("declareStoredTable (real postgres)", () => { beforeEach(async () => { await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerGroupTriggerAudit"`; await sql`DROP TABLE IF EXISTS "BulldozerTriggerAudit"`; await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; await sql` CREATE TABLE "BulldozerStorageEngine" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), - "keyPath" TEXT[] NOT NULL, - "keyPathParent" TEXT[] GENERATED ALWAYS AS ( + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( CASE - WHEN cardinality("keyPath") > 1 THEN "keyPath"[1:cardinality("keyPath") - 1] - ELSE "keyPath" + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] END ) STORED, "value" JSONB NOT NULL, CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), - CONSTRAINT "BulldozerStorageEngine_keyPath_non_empty_check" CHECK (cardinality("keyPath") >= 1), CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" FOREIGN KEY ("keyPathParent") REFERENCES "BulldozerStorageEngine"("keyPath") @@ -96,6 +112,12 @@ describe.sequential("declareStoredTable (real postgres)", () => { ) `; await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; await sql` CREATE TABLE "BulldozerTriggerAudit" ( "id" SERIAL PRIMARY KEY, @@ -105,6 +127,16 @@ describe.sequential("declareStoredTable (real postgres)", () => { "newRowData" JSONB ) `; + await sql` + CREATE TABLE "BulldozerGroupTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "groupKey" JSONB, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; }); afterAll(async () => { @@ -140,6 +172,38 @@ describe.sequential("declareStoredTable (real postgres)", () => { `, ]); } + function createGroupedTable() { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users" }); + const groupedTable = declareGroupByTable({ + tableId: "users-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + return { fromTable, groupedTable }; + } + function registerGroupAuditTrigger( + table: ReturnType["groupedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerGroupTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } test("init/isInitialized/delete lifecycle", async () => { const table = declareStoredTable<{ value: number }>({ tableId: "users" }); @@ -302,7 +366,7 @@ describe.sequential("declareStoredTable (real postgres)", () => { await runStatements(table.setRow("plain-row", expr(`'{"value":3,"label":"third"}'::jsonb`))); const rows = await sql.unsafe(` - SELECT array_to_string("keyPath", ' -> ') AS "keyPath", "value" + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), ' -> ') AS "keyPath", "value" FROM "BulldozerStorageEngine" ORDER BY "keyPath" `); @@ -310,6 +374,10 @@ describe.sequential("declareStoredTable (real postgres)", () => { expect(snapshotRows).toMatchInlineSnapshot(` [ + { + "keyPath": "", + "value": null, + }, { "keyPath": "table", "value": null, @@ -358,8 +426,8 @@ describe.sequential("declareStoredTable (real postgres)", () => { await expect(sql` INSERT INTO "BulldozerStorageEngine" ("keyPath", "keyPathParent", "value") VALUES ( - ARRAY['table', 'external:users', 'storage', 'rows', 'x']::text[], - ARRAY['table', 'external:users', 'storage']::text[], + ARRAY[to_jsonb('table'::text), to_jsonb('external:users'::text), to_jsonb('storage'::text), to_jsonb('rows'::text), to_jsonb('x'::text)]::jsonb[], + ARRAY[to_jsonb('table'::text), to_jsonb('external:users'::text), to_jsonb('storage'::text)]::jsonb[], '{"rowData":{"value":1}}'::jsonb ) `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); @@ -369,7 +437,7 @@ describe.sequential("declareStoredTable (real postgres)", () => { await expect(sql` INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") VALUES ( - ARRAY['missing-parent', 'child']::text[], + ARRAY[to_jsonb('missing-parent'::text), to_jsonb('child'::text)]::jsonb[], '{"rowData":{"value":1}}'::jsonb ) `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); @@ -469,6 +537,294 @@ describe.sequential("declareStoredTable (real postgres)", () => { expect(remainingRows).toHaveLength(0); }); + test("groupBy init backfills groups and rows from source table", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + + await runStatements(groupedTable.init()); + + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1", rowData: { team: "alpha", value: 1 } }, + { rowIdentifier: "u3", rowData: { team: "alpha", value: 3 } }, + ]); + + const allRows = await readRows(groupedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1" }, + { groupKey: "alpha", rowIdentifier: "u3" }, + { groupKey: "beta", rowIdentifier: "u2" }, + ]); + }); + + test("groupBy registerRowChangeTrigger emits insert/update/move/delete changes", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerGroupAuditTrigger(groupedTable, "group_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", value: 1 }, + }, + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", value: 1 }, + newRowData: { team: "alpha", value: 2 }, + }, + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", value: 2 }, + newRowData: null, + }, + { + event: "group_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "beta", value: 3 }, + }, + { + event: "group_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: { team: "beta", value: 3 }, + newRowData: null, + }, + ]); + }); + + test("groupBy deregistered trigger no longer runs", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + const handle = registerGroupAuditTrigger(groupedTable, "group_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + handle.deregister(); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", value: 1 }, + }, + ]); + }); + + test("groupBy stays no-op while uninitialized", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + registerGroupAuditTrigger(groupedTable, "group_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readBoolean(groupedTable.isInitialized())).toBe(false); + expect(await readGroupTriggerAuditRows()).toEqual([]); + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + }); + + test("groupBy delete cleans up and re-init backfills from source", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(groupedTable.delete()); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readBoolean(groupedTable.isInitialized())).toBe(false); + const groupsBeforeReinit = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsBeforeReinit).toEqual([]); + + await runStatements(groupedTable.init()); + const groupsAfterReinit = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + }); + + test("groupBy listGroups returns all groups when the range is inclusive", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + + const inclusive = await readRows(groupedTable.listGroups({ + start: expr(`to_jsonb('beta'::text)`), + end: expr(`to_jsonb('gamma'::text)`), + startInclusive: true, + endInclusive: true, + })); + expect(inclusive.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta", "gamma"]); + + const exclusive = await readRows(groupedTable.listGroups({ + start: expr(`to_jsonb('beta'::text)`), + end: expr(`to_jsonb('gamma'::text)`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusive).toEqual([]); + }); + + test("groupBy removes empty groups after moves and deletes", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + const groupsAfterInsert = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterInsert.map((row) => row.groupkey)).toEqual(["alpha"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`))); + const groupsAfterMove = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["beta"]); + + await runStatements(fromTable.deleteRow("u1")); + const groupsAfterDelete = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterDelete).toEqual([]); + }); + + test("groupBy listRowsInGroup handles missing groups and exclusive bounds", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const missingGroupRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('missing'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(missingGroupRows).toEqual([]); + + const exclusiveRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusiveRows).toEqual([]); + + const inclusiveRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: true, + endInclusive: true, + })); + expect(inclusiveRows).toHaveLength(2); + }); + + test("groupBy multiple triggers run in one transaction", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerGroupAuditTrigger(groupedTable, "group_trigger_a"); + registerGroupAuditTrigger(groupedTable, "group_trigger_b"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + const rows = await readGroupTriggerAuditRows(); + expect(rows.map((row) => row.event).sort(stringCompare)).toEqual(["group_trigger_a", "group_trigger_b"]); + }); + + test("groupBy supports null group keys and transitions away cleanly", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":null,"value":1}'::jsonb`))); + const nullGroupRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`'null'::jsonb`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(nullGroupRows.map((row) => row.rowidentifier)).toEqual(["u1"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + test("toExecutableSqlTransaction handles empty statements", async () => { await runStatements([]); }); diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts index 4a5c51b904..61e4e84897 100644 --- a/apps/backend/src/lib/bulldozer/db/index.ts +++ b/apps/backend/src/lib/bulldozer/db/index.ts @@ -6,8 +6,8 @@ export type Table = { tableId: TableId, // Query groups and rows - listGroups(options: { start: SqlExpression, end: SqlExpression, startInclusive: boolean, endInclusive: boolean }): SqlQuery>, - listRowsInGroup(options: { groupKey: SqlExpression, start: SqlExpression, end: SqlExpression, startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + listGroups(options: { start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + listRowsInGroup(options: { groupKey?: SqlExpression, start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery>, // Sorting and grouping compareGroupKeys(a: SqlExpression, b: SqlExpression): SqlExpression, @@ -34,37 +34,23 @@ export function declareStoredTable(options: { deleteRow(rowIdentifier: RowIdentifier): SqlStatement[], } { const triggers = new Map) => SqlStatement[]>(); - const ensureRowsHierarchyStatement = createInsertPathRowsStatement(options.tableId, ["rows"]); // Note that this table has only one group and sort key (null), so all groups and rows are always returned by every filter. return { tableId: options.tableId, - listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` - SELECT 'null'::jsonb AS groupKey - WHERE ${startInclusive && endInclusive ? sqlExpression`1 = 1` : sqlExpression`1 = 0`} - `, - listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => sqlQuery` - SELECT - "keyPath"[cardinality("keyPath")] AS rowIdentifier, - 'null'::jsonb AS rowSortKey, - "value"->'rowData' AS rowData - FROM "BulldozerStorageEngine" - WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["rows"])}::text[] - AND ${startInclusive && endInclusive ? sqlExpression`1 = 1` : sqlExpression`1 = 0`} - `, compareGroupKeys: (a, b) => sqlExpression` 0 `, compareSortKeys: (a, b) => sqlExpression` 0 `, - init: () => [ensureRowsHierarchyStatement, sqlStatement` - -- Create metadata about the table. + init: () => [sqlStatement` INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") - VALUES ( - ${getStorageEnginePath(options.tableId, ["metadata"])}::text[], - '{ "version": 1 }'::jsonb - ) + VALUES + (${getTablePath(options.tableId)}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["rows"])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) `], delete: () => [sqlStatement` WITH RECURSIVE "pathsToDelete" AS ( - SELECT ${getStorageEnginePath(options.tableId, [])}::text[] AS "path" + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" UNION ALL SELECT "BulldozerStorageEngine"."keyPath" AS "path" FROM "BulldozerStorageEngine" @@ -76,9 +62,22 @@ export function declareStoredTable(options: { isInitialized: () => sqlExpression` EXISTS ( SELECT 1 FROM "BulldozerStorageEngine" - WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::text[] + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] ) `, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT 'null'::jsonb AS groupKey + WHERE ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, registerRowChangeTrigger: (trigger) => { const id = generateSecureRandomString(); triggers.set(id, trigger); @@ -95,16 +94,15 @@ export function declareStoredTable(options: { ) `; return [ - ensureRowsHierarchyStatement, sqlQuery` SELECT "value"->'rowData' AS "oldRowData" FROM "BulldozerStorageEngine" - WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[] + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[] `.toStatement(oldRowsTableName), sqlQuery` INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") VALUES ( - ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[], + ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[], ${rowValue}::jsonb ) ON CONFLICT ("keyPath") DO UPDATE @@ -132,7 +130,7 @@ export function declareStoredTable(options: { return [ sqlQuery` DELETE FROM "BulldozerStorageEngine" - WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::text[] + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[] RETURNING "value"->'rowData' AS "oldRowData" `.toStatement(deletedRowsTableName), sqlQuery` @@ -151,26 +149,286 @@ export function declareStoredTable(options: { }; } -declare function declareMapTable< +export function declareGroupByTable< GK extends Json, - SK extends Json, OldRD extends RowData, NewRD extends RowData, >(options: { tableId: TableId, - fromTable: Table, + fromTable: Table, + groupBy: SqlMapper<{ rowIdentifier: RowIdentifier, rowData: OldRD }, { groupKey: GK }>, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + + options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`; + const groupedChangesTableName = `grouped_changes_${generateSecureRandomString()}`; + + return [ + sqlQuery` + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow", + "oldGroup"."groupKey" AS "oldGroupKey", + "newGroup"."groupKey" AS "newGroupKey" + FROM ${fromChangesTable} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') + WHERE ${isInitializedExpression} + `.toStatement(mappedChangesTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"newGroupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"newGroupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "target"."keyPath" = ${getGroupRowPath( + sqlExpression`"changes"."oldGroupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT + ${getGroupRowPath( + sqlExpression`"newGroupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "newRowData") + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRow" + WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[] + ) + `, + sqlQuery` + SELECT + "oldGroupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + "oldRowData" AS "oldRowData", + CASE + WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData" + ELSE 'null'::jsonb + END AS "newRowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasOldRow" + UNION ALL + SELECT + "newGroupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + 'null'::jsonb AS "oldRowData", + "newRowData" AS "newRowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey") + `.toStatement(groupedChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(groupedChangesTableName))), + ]; + }); + + return { + tableId: options.tableId, + compareGroupKeys: (a, b) => sqlExpression` 0 `, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => { + const fromTableAllRowsTableName = `from_table_all_rows_${generateSecureRandomString()}`; + const fromTableRowsWithGroupKeyTableName = `from_table_rows_with_group_key_${generateSecureRandomString()}`; + + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (${getTablePath(options.tableId)}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.fromTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromTableAllRowsTableName), + sqlQuery` + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData", + "mapped"."groupKey" AS "groupKey" + FROM ${quoteSqlIdentifier(fromTableAllRowsTableName)} AS "rows" + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(fromTableRowsWithGroupKeyTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + UNION + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "rowData") + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + `, + ]; + }, + delete: () => [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `], + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + -- Get all rows from all groups + SELECT + "groupRows"."keyPath"[cardinality("groupRows"."keyPath") - 1] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupRows" + INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath" + WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} + +export declare function declareMapTable< + GK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, mapper: SqlMapper, -}): Table; +}): Table; -declare function declareFilterTable< +export declare function declareFilterTable< GK extends Json, - SK extends Json, RD extends RowData, >(options: { tableId: TableId, - fromTable: Table, + fromTable: Table, filter: SqlPredicate, -}): Table; +}): Table; // ====== Executing SQL Statements ====== @@ -201,7 +459,7 @@ export function toExecutableSqlTransaction(statements: SqlStatement[]): string { const sqlTemplateLiteral = (type: T) => (strings: TemplateStringsArray, ...values: { sql: string }[]) => ({ type, sql: templateIdentity(strings, ...values.map(v => v.sql)) }); type SqlStatement = { type: "statement", outputName?: string, sql: string }; const sqlStatement = sqlTemplateLiteral<"statement">("statement"); -type SqlQuery = void> = { type: "query", outputName?: string, sql: string }; +type SqlQuery = void> = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; const sqlQuery = (...args: Parameters>>) => { return { ...sqlTemplateLiteral<"query">("query")(...args), @@ -216,6 +474,7 @@ type SqlMapper = { type: "mapper", const sqlMapper = sqlTemplateLiteral<"mapper">("mapper"); type SqlPredicate = { type: "predicate", sql: string }; // ex.: "user_id = 123" const sqlPredicate = sqlTemplateLiteral<"predicate">("predicate"); +const sqlArray = (exprs: (SqlExpression | SqlMapper)[]) => ({ type: "expression", sql: `ARRAY[${exprs.map(e => e.sql).join(", ")}]` } as const); type RowData = Record; type Json = string | number | boolean | null | { [key: string]: Json } | Json[]; type RowIdentifier = string; @@ -229,10 +488,20 @@ function quoteSqlIdentifier(input: string): SqlExpression { function quoteSqlStringLiteral(input: string): SqlExpression { return { type: "expression", sql: `'${input.replaceAll("'", "''")}'` }; } -function getStorageEnginePath(tableId: TableId, path: string[]): SqlExpression { - return pathSegmentsToSqlExpression(getStorageEnginePathSegments(tableId, path)); +function quoteSqlJsonbLiteral(input: Json): SqlExpression { + return { type: "expression", sql: `${quoteSqlStringLiteral(JSON.stringify(input)).sql}::jsonb` }; +} +function getTablePath(tableId: TableId): SqlExpression { + return sqlArray(getTablePathSegments(tableId)); +} +function getStorageEnginePath(tableId: TableId, path: (string | SqlExpression | SqlMapper)[]): SqlExpression { + return sqlArray([ + ...getTablePathSegments(tableId), + quoteSqlJsonbLiteral("storage"), + ...path.map(p => typeof p === "string" ? quoteSqlJsonbLiteral(p) : p), + ]); } -function getStorageEnginePathSegments(tableId: TableId, path: string[]): string[] { +function getTablePathSegments(tableId: TableId): SqlExpression[] { const tableIdWithParents = []; let currentTableId = tableId; while (true) { @@ -245,24 +514,17 @@ function getStorageEnginePathSegments(tableId: TableId, path: string[]): string[ currentTableId = currentTableId.parent; } } - return [...tableIdWithParents.reverse().flatMap(id => ["table", id]), "storage", ...path]; -} -function pathSegmentsToSqlExpression(pathSegments: string[]): SqlExpression { - return { - type: "expression", - sql: `ARRAY[${pathSegments.map((segment) => quoteSqlStringLiteral(segment).sql).join(", ")}]::text[]`, - }; + return [ + ...tableIdWithParents.reverse().flatMap(id => ["table", id]), + ].map(id => quoteSqlJsonbLiteral(id)); } -function createInsertPathRowsStatement(tableId: TableId, path: string[]): SqlStatement { - const segments = getStorageEnginePathSegments(tableId, path); - const uniquePathSqlExpressions = [...new Set(segments.map((_, idx) => pathSegmentsToSqlExpression(segments.slice(0, idx + 1)).sql))]; - return { - type: "statement", - sql: deindent` - INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") - VALUES - ${uniquePathSqlExpressions.map((pathSql) => `(${pathSql}, 'null'::jsonb)`).join(",\n")} - ON CONFLICT ("keyPath") DO NOTHING - `, - }; +function singleNullSortKeyRangePredicate(options: { + start: SqlExpression | "start", + end: SqlExpression | "end", + startInclusive: boolean, + endInclusive: boolean, +}): SqlExpression { + return (options.start === "start" || options.startInclusive) && (options.end === "end" || options.endInclusive) + ? sqlExpression`1 = 1` + : sqlExpression`1 = 0`; } From 09ba416d6fca2aef9df56ac692ed79047f3eb057 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 10:12:07 -0700 Subject: [PATCH 03/40] Fix Prisma schema --- apps/backend/prisma/schema.prisma | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 8e3dba15fc..4efeca7ea1 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -1245,14 +1245,13 @@ model OutgoingRequest { } model BulldozerStorageEngine { - id String @id @default(uuid()) @db.Uuid + id String @id @default(uuid()) @db.Uuid keyPath Json[] keyPathParent Unsupported("jsonb[]") value Json @@unique([keyPath], map: "BulldozerStorageEngine_keyPath_key") @@index([keyPathParent], map: "BulldozerStorageEngine_keyPathParent_idx") - @@ignore } model DeletedRow { From 4232a943d3a8d97cde693a73e38eccda83aee0ee Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 10:43:09 -0700 Subject: [PATCH 04/40] declareMapTable --- AGENTS.md | 1 + .../src/lib/bulldozer/db/index.fuzz.test.ts | 479 +++++++++++++ .../src/lib/bulldozer/db/index.test.ts | 653 +++++++++++++++++- apps/backend/src/lib/bulldozer/db/index.ts | 288 +++++++- 4 files changed, 1418 insertions(+), 3 deletions(-) create mode 100644 apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts diff --git a/AGENTS.md b/AGENTS.md index fde90ebf10..4bdb14eef6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,7 @@ To see all development ports, refer to the index.html of `apps/dev-launchpad/pub - Any design components you add or modify in the dashboard, update the Playground page accordingly to showcase the changes. - Unless very clearly equivalent from types, prefer explicit null/undefinedness checks over boolean checks, eg. `foo == null` instead of `!foo`. - Ensure **aggressively** that all code has low coupling and high cohesion. This is really important as it makes sure our code remains consistent and maintainable. Eagerly refactor things into better abstractions and look out for them actively. +- Always let me know about the tradeoffs and decisions you make while implementing a non-trivial change. ### Code-related - Use ES6 maps instead of records wherever you can. diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts new file mode 100644 index 0000000000..9a7241637b --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts @@ -0,0 +1,479 @@ +import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; +import postgres from "postgres"; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest"; +import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_fuzz_test"; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; +type SqlMapper = { type: "mapper", sql: string }; +type QueryableTable = { + listGroups(options: { start: "start", end: "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + listRowsInGroup(options: { groupKey?: SqlExpression, start: "start", end: "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, +}; +type SourceRow = { team: string | null, value: number }; +type TeamMappedRow = { team: string | null, valuePlusTen: number }; +type TeamBucketRow = { team: string | null, valueScaled: number, bucket: string }; +type GroupedRows> = Map }>; + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} +function mapper(sql: string): SqlMapper { + return { type: "mapper", sql }; +} + +function createRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; +} +function choose(rng: () => number, values: readonly T[]): T { + return values[Math.floor(rng() * values.length)] ?? values[0]; +} +function sqlStringLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} +function jsonbLiteral(value: unknown): string { + return `${sqlStringLiteral(JSON.stringify(value))}::jsonb`; +} +function groupDiscriminator(groupKey: string | null): string { + return groupKey === null ? "__NULL__" : `S:${groupKey}`; +} +function nullableStringCompare(a: string | null, b: string | null): number { + if (a === b) return 0; + if (a === null) return -1; + if (b === null) return 1; + return stringCompare(a, b); +} +function groupKeyExpression(groupKey: string | null): SqlExpression { + return groupKey === null + ? expr(`'null'::jsonb`) + : expr(`to_jsonb(${sqlStringLiteral(groupKey)}::text)`); +} + +function computeTeamGroups(rows: Map): GroupedRows<{ team: string | null, value: number }> { + const groups: GroupedRows<{ team: string | null, value: number }> = new Map(); + for (const [rowIdentifier, row] of rows) { + const key = groupDiscriminator(row.team); + const existing = groups.get(key); + if (existing != null) { + existing.rows.set(rowIdentifier, { team: row.team, value: row.value }); + } else { + groups.set(key, { + groupKey: row.team, + rows: new Map([[rowIdentifier, { team: row.team, value: row.value }]]), + }); + } + } + return groups; +} +function mapGroups, NewRow extends Record>( + groups: GroupedRows, + mapperFn: (row: OldRow) => NewRow, +): GroupedRows { + const mapped: GroupedRows = new Map(); + for (const [groupKey, group] of groups) { + mapped.set(groupKey, { + groupKey: group.groupKey, + rows: new Map([...group.rows.entries()].map(([rowIdentifier, rowData]) => [rowIdentifier, mapperFn(rowData)])), + }); + } + return mapped; +} +function regroupByField>( + groups: GroupedRows, + groupKeySelector: (row: T) => string | null, +): GroupedRows { + const regrouped: GroupedRows = new Map(); + for (const group of groups.values()) { + for (const [rowIdentifier, rowData] of group.rows) { + const groupKey = groupKeySelector(rowData); + const key = groupDiscriminator(groupKey); + const existing = regrouped.get(key); + if (existing != null) { + existing.rows.set(rowIdentifier, rowData); + } else { + regrouped.set(key, { + groupKey, + rows: new Map([[rowIdentifier, rowData]]), + }); + } + } + } + return regrouped; +} + +describe.sequential("bulldozer db fuzz composition (real postgres)", () => { + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + + async function runStatements(statements: SqlStatement[]) { + await sql.unsafe(toExecutableSqlTransaction(statements)); + } + async function readBoolean(expression: SqlExpression) { + const rows = await sql.unsafe(`SELECT (${expression.sql}) AS "value"`); + return rows[0].value === true; + } + async function readRows(query: SqlQuery) { + return await sql.unsafe(toQueryableSqlQuery(query)); + } + + async function assertTableMatches>(table: QueryableTable, expected: GroupedRows) { + const expectedGroups = [...expected.values()] + .filter((group) => group.rows.size > 0) + .map((group) => group.groupKey) + .sort(nullableStringCompare); + + const actualGroups = (await readRows(table.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))) + .map((row) => row.groupkey as string | null) + .sort(nullableStringCompare); + + expect(actualGroups).toEqual(expectedGroups); + + const expectedAllRows = [...expected.values()] + .flatMap((group) => [...group.rows.entries()].map(([rowIdentifier, rowData]) => ({ groupKey: group.groupKey, rowIdentifier, rowData }))) + .sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + return byGroup !== 0 ? byGroup : stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + + const actualAllRows = (await readRows(table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))) + .map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: row.rowidentifier as string, + rowData: row.rowdata as Record, + })) + .sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + return byGroup !== 0 ? byGroup : stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + + expect(actualAllRows).toEqual(expectedAllRows); + + for (const expectedGroup of expected.values()) { + if (expectedGroup.rows.size === 0) continue; + const expectedRows = [...expectedGroup.rows.entries()] + .map(([rowIdentifier, rowData]) => ({ rowIdentifier, rowData })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + const actualRows = (await readRows(table.listRowsInGroup({ + groupKey: groupKeyExpression(expectedGroup.groupKey), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))) + .map((row) => ({ rowIdentifier: row.rowidentifier as string, rowData: row.rowdata as Record })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + expect(actualRows).toEqual(expectedRows); + } + + const missingRows = await readRows(table.listRowsInGroup({ + groupKey: groupKeyExpression("__missing_group__"), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(missingRows).toEqual([]); + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + test("fuzz: stacked group/map/group pipelines preserve invariants under random mutations", async () => { + const identifiers = ["u1", "u2", "u3", "u4", "u:5", "u 6", "u/7", "u'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [101, 202, 303]) { + const rng = createRng(seed); + const sourceRows = new Map(); + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const mapTable1 = declareMapTable({ + tableId: `fuzz-users-map-level-1-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen" + `), + }); + const mapTable2 = declareMapTable({ + tableId: `fuzz-users-map-level-2-${seed}`, + fromTable: mapTable1, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", + ( + CASE + WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + `), + }); + const groupedByBucket = declareGroupByTable({ + tableId: `fuzz-users-by-bucket-${seed}`, + fromTable: mapTable2, + groupBy: mapper(`"rowData"->'bucket' AS "groupKey"`), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTable1.init()); + await runStatements(mapTable2.init()); + await runStatements(groupedByBucket.init()); + + for (let step = 0; step < 60; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 50), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.94) { + await runStatements(groupedByBucket.delete()); + await runStatements(mapTable2.delete()); + await runStatements(mapTable1.delete()); + await runStatements(mapTable1.init()); + await runStatements(mapTable2.init()); + await runStatements(groupedByBucket.init()); + } else { + const rowIdentifier = choose(rng, identifiers); + const rowData = sourceRows.get(rowIdentifier); + if (rowData != null) { + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else { + await runStatements(fromTable.deleteRow(rowIdentifier)); + } + } + + const expectedGrouped = computeTeamGroups(sourceRows); + const expectedMap1 = mapGroups(expectedGrouped, (row): TeamMappedRow => ({ + team: (row.team as string | null), + valuePlusTen: (row.value as number) + 10, + })); + const expectedMap2 = mapGroups(expectedMap1, (row): TeamBucketRow => { + const valueScaled = (row.valuePlusTen as number) * 2; + return { + team: (row.team as string | null), + valueScaled, + bucket: valueScaled >= 30 ? "high" : "low", + }; + }); + const expectedBucket = regroupByField(expectedMap2, (row) => row.bucket as string); + + await assertTableMatches(groupedTable, expectedGrouped); + await assertTableMatches(mapTable1, expectedMap1); + await assertTableMatches(mapTable2, expectedMap2); + await assertTableMatches(groupedByBucket, expectedBucket); + } + } + }); + + test("fuzz: parallel map tables remain isolated with independent re-inits", async () => { + const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const; + const teams = ["alpha", "beta", null] as const; + + for (const seed of [401, 402]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let mapAInitialized = true; + let mapBInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `parallel-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `parallel-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const mapTableA = declareMapTable({ + tableId: `parallel-users-map-a-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValueA" + `), + }); + const mapTableB = declareMapTable({ + tableId: `parallel-users-map-b-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ((("rowData"->>'value')::int) * -1) AS "mappedValueB" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTableA.init()); + await runStatements(mapTableB.init()); + + for (let step = 0; step < 50; step++) { + const roll = rng(); + if (roll < 0.6) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 40), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.82) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.9) { + if (mapAInitialized) { + await runStatements(mapTableA.delete()); + mapAInitialized = false; + } + } else if (roll < 0.94) { + if (!mapAInitialized) { + await runStatements(mapTableA.init()); + mapAInitialized = true; + } + } else if (roll < 0.98) { + if (mapBInitialized) { + await runStatements(mapTableB.delete()); + mapBInitialized = false; + } + } else { + if (!mapBInitialized) { + await runStatements(mapTableB.init()); + mapBInitialized = true; + } + } + + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + const expectedMapA = mapGroups(expectedGrouped, (row) => ({ + team: row.team as string | null, + mappedValueA: (row.value as number) + 100, + })); + const expectedMapB = mapGroups(expectedGrouped, (row) => ({ + team: row.team as string | null, + mappedValueB: -1 * (row.value as number), + })); + + if (mapAInitialized) { + expect(await readBoolean(mapTableA.isInitialized())).toBe(true); + await assertTableMatches(mapTableA, expectedMapA); + } else { + expect(await readBoolean(mapTableA.isInitialized())).toBe(false); + const groups = await readRows(mapTableA.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + + if (mapBInitialized) { + expect(await readBoolean(mapTableB.isInitialized())).toBe(true); + await assertTableMatches(mapTableB, expectedMapB); + } else { + expect(await readBoolean(mapTableB.isInitialized())).toBe(false); + const groups = await readRows(mapTableB.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + } + } + }); +}); diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts index 74dae4fe97..6ee4dbac33 100644 --- a/apps/backend/src/lib/bulldozer/db/index.test.ts +++ b/apps/backend/src/lib/bulldozer/db/index.test.ts @@ -1,7 +1,7 @@ import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; import postgres from "postgres"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest"; -import { declareGroupByTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; +import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; type TestDb = { full: string, base: string }; @@ -82,6 +82,18 @@ describe.sequential("declareStoredTable (real postgres)", () => { ORDER BY "id" `); } + async function readMapTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "groupKey"#>>'{}' AS "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerMapTriggerAudit" + ORDER BY "id" + `); + } beforeAll(async () => { await adminSql.unsafe(`CREATE DATABASE ${dbName}`); @@ -89,6 +101,7 @@ describe.sequential("declareStoredTable (real postgres)", () => { beforeEach(async () => { await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerMapTriggerAudit"`; await sql`DROP TABLE IF EXISTS "BulldozerGroupTriggerAudit"`; await sql`DROP TABLE IF EXISTS "BulldozerTriggerAudit"`; await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; @@ -137,6 +150,16 @@ describe.sequential("declareStoredTable (real postgres)", () => { "newRowData" JSONB ) `; + await sql` + CREATE TABLE "BulldozerMapTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "groupKey" JSONB, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; }); afterAll(async () => { @@ -181,6 +204,53 @@ describe.sequential("declareStoredTable (real postgres)", () => { }); return { fromTable, groupedTable }; } + function createMappedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const mappedTable = declareMapTable({ + tableId: "users-by-team-mapped", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValue" + `), + }); + return { fromTable, groupedTable, mappedTable }; + } + function createStackedMappedTables() { + const { fromTable, groupedTable } = createGroupedTable(); + const mappedTableLevel1 = declareMapTable({ + tableId: "users-by-team-map-level-1", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen" + `), + }); + const mappedTableLevel2 = declareMapTable({ + tableId: "users-by-team-map-level-2", + fromTable: mappedTableLevel1, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", + ( + CASE + WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + `), + }); + return { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 }; + } + function createGroupMapGroupPipeline() { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + const groupedByBucketTable = declareGroupByTable({ + tableId: "users-by-bucket", + fromTable: mappedTableLevel2, + groupBy: mapper(`"rowData"->'bucket' AS "groupKey"`), + }); + return { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable }; + } function registerGroupAuditTrigger( table: ReturnType["groupedTable"], event: string, @@ -204,6 +274,29 @@ describe.sequential("declareStoredTable (real postgres)", () => { `, ]); } + function registerMapAuditTrigger( + table: ReturnType["mappedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } test("init/isInitialized/delete lifecycle", async () => { const table = declareStoredTable<{ value: number }>({ tableId: "users" }); @@ -825,6 +918,564 @@ describe.sequential("declareStoredTable (real postgres)", () => { expect(groups.map((row) => row.groupkey)).toEqual(["alpha"]); }); + test("mapTable init backfills groups and mapped rows", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + + const groups = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1", rowData: { team: "alpha", mappedValue: 101 } }, + { rowIdentifier: "u3", rowData: { team: "alpha", mappedValue: 103 } }, + ]); + + const allRows = await readRows(mappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1" }, + { groupKey: "alpha", rowIdentifier: "u3" }, + { groupKey: "beta", rowIdentifier: "u2" }, + ]); + }); + + test("mapTable registerRowChangeTrigger emits mapped insert/update/move/delete changes", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + registerMapAuditTrigger(mappedTable, "map_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", mappedValue: 101 }, + }, + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", mappedValue: 101 }, + newRowData: { team: "alpha", mappedValue: 102 }, + }, + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", mappedValue: 102 }, + newRowData: null, + }, + { + event: "map_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "beta", mappedValue: 103 }, + }, + { + event: "map_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: { team: "beta", mappedValue: 103 }, + newRowData: null, + }, + ]); + }); + + test("mapTable deregistered trigger no longer runs", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + const handle = registerMapAuditTrigger(mappedTable, "map_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + handle.deregister(); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", mappedValue: 101 }, + }, + ]); + }); + + test("mapTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerMapAuditTrigger(mappedTable, "map_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readBoolean(mappedTable.isInitialized())).toBe(false); + expect(await readMapTriggerAuditRows()).toEqual([]); + const groups = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + }); + + test("mapTable delete cleans up and re-init backfills from source", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(mappedTable.delete()); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readBoolean(mappedTable.isInitialized())).toBe(false); + const groupsBeforeReinit = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsBeforeReinit).toEqual([]); + + await runStatements(mappedTable.init()); + const groupsAfterReinit = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + }); + + test("mapTable listRowsInGroup handles missing groups and exclusive bounds", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + const missingGroupRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('missing'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(missingGroupRows).toEqual([]); + + const exclusiveRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusiveRows).toEqual([]); + }); + + test("stacked map tables propagate updates across multiple mapping layers", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":7}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":4}'::jsonb`))); + + const groupsAfterMove = await readRows(mappedTableLevel2.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["alpha"]); + + const alphaRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } }, + { rowIdentifier: "u2", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } }, + ]); + + await runStatements(fromTable.deleteRow("u1")); + const alphaRowsAfterDelete = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsAfterDelete.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } }, + ]); + }); + + test("stacked map tables handle special row identifiers and null group transitions", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + + const specialIdentifier = "user/one:two space"; + await runStatements(fromTable.setRow(specialIdentifier, expr(`'{"team":null,"value":3}'::jsonb`))); + + const nullGroupRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`'null'::jsonb`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(nullGroupRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: specialIdentifier, rowData: { team: null, valueScaled: 26, bucket: "low" } }, + ]); + + await runStatements(fromTable.setRow(specialIdentifier, expr(`'{"team":"alpha","value":3}'::jsonb`))); + const groupsAfterMove = await readRows(mappedTableLevel2.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + + test("stacked map tables backfill correctly with staggered initialization order", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + await runStatements(mappedTableLevel1.init()); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + + await runStatements(mappedTableLevel2.init()); + const allRowsAfterInit = await readRows(mappedTableLevel2.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRowsAfterInit.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 22, bucket: "low" } }, + { groupKey: "alpha", rowIdentifier: "u3", rowData: { team: "alpha", valueScaled: 26, bucket: "low" } }, + { groupKey: "beta", rowIdentifier: "u2", rowData: { team: "beta", valueScaled: 24, bucket: "low" } }, + ]); + + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + const betaRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(betaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2", rowData: { team: "beta", valueScaled: 60, bucket: "high" } }, + ]); + }); + + test("groupBy over a stacked map table stays consistent on mapped key transitions", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":2}'::jsonb`))); + + const initialGroups = await readRows(groupedByBucketTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(initialGroups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["high", "low"]); + + const lowRows = await readRows(groupedByBucketTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('low'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(lowRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1", "u3"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":30}'::jsonb`))); + await runStatements(fromTable.deleteRow("u3")); + + const finalGroups = await readRows(groupedByBucketTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(finalGroups.map((row) => row.groupkey)).toEqual(["high"]); + + const highRows = await readRows(groupedByBucketTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('high'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(highRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1", "u2"]); + }); + + test("composed trigger fanout works for stacked map and downstream groupBy tables", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + mappedTableLevel2.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("map_level_2_change"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + groupedByBucketTable.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerGroupTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("bucket_group_change"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":30}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + }, + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + newRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + }, + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + newRowData: null, + }, + ]); + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "bucket_group_change", + groupKey: "low", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + }, + { + event: "bucket_group_change", + groupKey: "low", + rowIdentifier: "u1", + oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + newRowData: null, + }, + { + event: "bucket_group_change", + groupKey: "high", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + }, + { + event: "bucket_group_change", + groupKey: "high", + rowIdentifier: "u1", + oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + newRowData: null, + }, + ]); + }); + + test("deep pipeline delete and re-init restores exact source truth", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":2}'::jsonb`))); + + await runStatements(groupedByBucketTable.delete()); + await runStatements(mappedTableLevel2.delete()); + await runStatements(mappedTableLevel1.delete()); + + expect(await readBoolean(mappedTableLevel1.isInitialized())).toBe(false); + expect(await readBoolean(mappedTableLevel2.isInitialized())).toBe(false); + expect(await readBoolean(groupedByBucketTable.isInitialized())).toBe(false); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.deleteRow("u2")); + await runStatements(fromTable.setRow("u4", expr(`'{"team":"delta","value":0}'::jsonb`))); + + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + const allBucketRows = await readRows(groupedByBucketTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allBucketRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "high", rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } }, + { groupKey: "low", rowIdentifier: "u3", rowData: { team: "gamma", valueScaled: 24, bucket: "low" } }, + { groupKey: "low", rowIdentifier: "u4", rowData: { team: "delta", valueScaled: 20, bucket: "low" } }, + ]); + }); + + test("parallel map tables on the same grouped source stay isolated", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + const mapTableA = declareMapTable({ + tableId: "users-map-a", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValueA" + `), + }); + const mapTableB = declareMapTable({ + tableId: "users-map-b", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ((("rowData"->>'value')::int) * -1) AS "mappedValueB" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTableA.init()); + await runStatements(mapTableB.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":4}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":6}'::jsonb`))); + await runStatements(fromTable.deleteRow("u2")); + + const alphaRowsA = await readRows(mapTableA.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsA.map((row) => row.rowdata)).toEqual([{ team: "alpha", mappedValueA: 106 }]); + + const alphaRowsB = await readRows(mapTableB.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsB.map((row) => row.rowdata)).toEqual([{ team: "alpha", mappedValueB: -6 }]); + + const groupsA = await readRows(mapTableA.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const groupsB = await readRows(mapTableB.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsA.map((row) => row.groupkey)).toEqual(["alpha"]); + expect(groupsB.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + test("toExecutableSqlTransaction handles empty statements", async () => { await runStatements([]); }); diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts index 61e4e84897..61d0341d36 100644 --- a/apps/backend/src/lib/bulldozer/db/index.ts +++ b/apps/backend/src/lib/bulldozer/db/index.ts @@ -411,7 +411,7 @@ export function declareGroupByTable< }; } -export declare function declareMapTable< +export function declareMapTable< GK extends Json, OldRD extends RowData, NewRD extends RowData, @@ -419,7 +419,269 @@ export declare function declareMapTable< tableId: TableId, fromTable: Table, mapper: SqlMapper, -}): Table; +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + + options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`; + const mapChangesTableName = `map_changes_${generateSecureRandomString()}`; + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow", + "oldMapped"."rowData" AS "oldMappedRowData", + "newMapped"."rowData" AS "newMappedRowData" + FROM ${fromChangesTable} AS "changes" + LEFT JOIN LATERAL ( + SELECT to_jsonb("mapped") AS "rowData" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') + LEFT JOIN LATERAL ( + SELECT to_jsonb("mapped") AS "rowData" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') + WHERE ${isInitializedExpression} + `.toStatement(mappedChangesTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "target"."keyPath" = ${getGroupRowPath( + sqlExpression`"changes"."groupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "newMappedRowData") + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRow" + WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + `, + sqlQuery` + SELECT + "groupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + CASE WHEN "hasOldRow" THEN "oldMappedRowData" ELSE 'null'::jsonb END AS "oldRowData", + CASE WHEN "hasNewRow" THEN "newMappedRowData" ELSE 'null'::jsonb END AS "newRowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasOldRow" OR "hasNewRow" + `.toStatement(mapChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(mapChangesTableName))), + ]; + }); + + return { + tableId: options.tableId, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => { + const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`; + const fromRowsTableName = `from_rows_${generateSecureRandomString()}`; + const mappedRowsTableName = `mapped_rows_${generateSecureRandomString()}`; + + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (${getTablePath(options.tableId)}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb), + (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.fromTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromGroupsTableName), + sqlQuery` + SELECT + "groups"."groupkey" AS "groupKey", + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupkey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + `.toStatement(fromRowsTableName), + sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "rowIdentifier", + "mapped"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows" + LEFT JOIN LATERAL ( + SELECT to_jsonb("mapped") AS "rowData" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "rows"."rowIdentifier" AS "rowIdentifier", + "rows"."rowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(mappedRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[], + 'null'::jsonb + FROM ${quoteSqlIdentifier(mappedRowsTableName)} + UNION + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "rowData") + FROM ${quoteSqlIdentifier(mappedRowsTableName)} + `, + ]; + }, + delete: () => [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `], + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + SELECT + "groupRows"."keyPath"[cardinality("groupRows"."keyPath") - 1] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupRows" + INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath" + WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} export declare function declareFilterTable< GK extends Json, @@ -430,6 +692,28 @@ export declare function declareFilterTable< filter: SqlPredicate, }): Table; +export declare function declareSortTable< + GK extends Json, + SK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + getSortKey: SqlMapper<{ rowData: RD }, { sortKey: SK }>, + compareSortKeys: (a: SqlExpression, b: SqlExpression) => SqlExpression, +}): Table; + +export declare function declareLFoldTable< + GK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + initialState: SqlExpression, + reducer: SqlMapper<{ state: Json, oldRowData: OldRD }, { newState: Json, newRowData: NewRD }>, +}): Table; + // ====== Executing SQL Statements ====== const BULLDOZER_LOCK_ID = 7857391; // random number to avoid conflicts with other applications From 31b6ac6013e5de40e9751e55dc5bdf6779c33978 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 10:52:00 -0700 Subject: [PATCH 05/40] Performance tests --- .../src/lib/bulldozer/db/index.perf.test.ts | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 apps/backend/src/lib/bulldozer/db/index.perf.test.ts diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts new file mode 100644 index 0000000000..30ae6df336 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts @@ -0,0 +1,275 @@ +import postgres from "postgres"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; + +type WorkloadOperation = + | { type: "upsert", rowIdentifier: string, team: string | null, value: number } + | { type: "delete", rowIdentifier: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_perf_test"; +const DEFAULT_WARMUP_OPS = 80; +const DEFAULT_MEASURED_OPS = 500; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} + +function jsonbLiteral(value: unknown): string { + return `'${JSON.stringify(value).replaceAll("'", "''")}'::jsonb`; +} + +function createRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; +} + +function choose(rng: () => number, values: readonly T[]): T { + return values[Math.floor(rng() * values.length)] ?? values[0]; +} + +function createWorkload(seed: number, operationCount: number): WorkloadOperation[] { + const rng = createRng(seed); + const identifiers = ["u1", "u2", "u3", "u4", "u:5", "u 6", "u/7", "u'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + const existing = new Set(); + const operations: WorkloadOperation[] = []; + + for (let i = 0; i < operationCount; i++) { + const roll = rng(); + if (roll < 0.74) { + const rowIdentifier = choose(rng, identifiers); + const team = choose(rng, teams); + const value = Math.floor(rng() * 100); + operations.push({ type: "upsert", rowIdentifier, team, value }); + existing.add(rowIdentifier); + } else { + const rowIdentifier = existing.size > 0 + ? choose(rng, [...existing]) + : choose(rng, identifiers); + operations.push({ type: "delete", rowIdentifier }); + existing.delete(rowIdentifier); + } + } + + return operations; +} + +function operationCountFromEnv(varName: string, fallback: number): number { + const env = Reflect.get(import.meta, "env"); + const raw = Reflect.get(env, varName); + if (typeof raw !== "string") return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 1) return fallback; + return Math.floor(parsed); +} + +function logLine(message: string): void { + console.log(`${message}\n`); +} + +describe.sequential("bulldozer db performance (real postgres)", () => { + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + + async function runStatements(statements: SqlStatement[]) { + await sql.unsafe(toExecutableSqlTransaction(statements)); + } + + async function readRows(query: SqlQuery) { + return await sql.unsafe(toQueryableSqlQuery(query)); + } + + async function executeWorkload( + fromTable: ReturnType>, + operations: WorkloadOperation[], + ): Promise { + for (const operation of operations) { + if (operation.type === "upsert") { + await runStatements(fromTable.setRow( + operation.rowIdentifier, + expr(jsonbLiteral({ team: operation.team, value: operation.value })), + )); + } else { + await runStatements(fromTable.deleteRow(operation.rowIdentifier)); + } + } + } + + async function benchmarkScenario(options: { + name: string, + warmupOperations: WorkloadOperation[], + measuredOperations: WorkloadOperation[], + beforeRun: () => Promise<{ fromTable: ReturnType>, validate: () => Promise }>, + }) { + const setup = await options.beforeRun(); + await executeWorkload(setup.fromTable, options.warmupOperations); + const startedAt = performance.now(); + await executeWorkload(setup.fromTable, options.measuredOperations); + const elapsedMs = performance.now() - startedAt; + await setup.validate(); + + const operationsPerSecond = options.measuredOperations.length / (elapsedMs / 1000); + logLine(`[bulldozer-perf] ${options.name}: ${operationsPerSecond.toFixed(1)} ops/s (${options.measuredOperations.length} ops in ${elapsedMs.toFixed(1)} ms)`); + return { operationsPerSecond, elapsedMs }; + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + it("reports ops/sec for baseline and composed example setup", async () => { + const warmupCount = operationCountFromEnv("STACK_BULLDOZER_PERF_WARMUP_OPS", DEFAULT_WARMUP_OPS); + const measuredCount = operationCountFromEnv("STACK_BULLDOZER_PERF_MEASURED_OPS", DEFAULT_MEASURED_OPS); + const warmupOperations = createWorkload(111, warmupCount); + const measuredOperations = createWorkload(222, measuredCount); + + const baseline = await benchmarkScenario({ + name: "stored-table baseline", + warmupOperations, + measuredOperations, + beforeRun: async () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-baseline-users" }); + await runStatements(fromTable.init()); + return { + fromTable, + validate: async () => { + const rows = await readRows(fromTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(Array.isArray(rows)).toBe(true); + }, + }; + }, + }); + + const composed = await benchmarkScenario({ + name: "group+map+group composed pipeline", + warmupOperations, + measuredOperations, + beforeRun: async () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-composed-users" }); + const groupedByTeam = declareGroupByTable({ + tableId: "perf-composed-users-by-team", + fromTable, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const mapped = declareMapTable({ + tableId: "perf-composed-users-mapped", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen", + ( + CASE + WHEN (("rowData"->>'value')::int + 10) >= 40 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + ` }, + }); + const groupedByBucket = declareGroupByTable({ + tableId: "perf-composed-users-by-bucket", + fromTable: mapped, + groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` }, + }); + + await runStatements(fromTable.init()); + await runStatements(groupedByTeam.init()); + await runStatements(mapped.init()); + await runStatements(groupedByBucket.init()); + + return { + fromTable, + validate: async () => { + const rows = await readRows(groupedByBucket.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(Array.isArray(rows)).toBe(true); + }, + }; + }, + }); + + const slowdownFactor = baseline.operationsPerSecond / composed.operationsPerSecond; + logLine(`[bulldozer-perf] slowdown factor (baseline/composed): ${slowdownFactor.toFixed(2)}x`); + logLine(`[bulldozer-perf] config: warmup=${warmupCount}, measured=${measuredCount}`); + + expect(baseline.operationsPerSecond).toBeGreaterThan(0); + expect(composed.operationsPerSecond).toBeGreaterThan(0); + }); +}); + From 2f7f09afca8314985193094594fdef8023528658 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 11:08:38 -0700 Subject: [PATCH 06/40] Load tests --- .../src/lib/bulldozer/db/index.perf.test.ts | 266 +++++++++++++++++- 1 file changed, 252 insertions(+), 14 deletions(-) diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts index 30ae6df336..9f3ec1619a 100644 --- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts +++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts @@ -1,3 +1,4 @@ +import { getEnvBoolean } from "@stackframe/stack-shared/dist/utils/env"; import postgres from "postgres"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; @@ -14,6 +15,15 @@ type WorkloadOperation = const TEST_DB_PREFIX = "stack_bulldozer_db_perf_test"; const DEFAULT_WARMUP_OPS = 80; const DEFAULT_MEASURED_OPS = 500; +const DEFAULT_LOAD_ROW_COUNT = getEnvBoolean("CI") ? 200_000 : 20_000; +const LOAD_PREFILL_MAX_MS = 30_000; +const LOAD_COUNT_QUERY_MAX_MS = 5_000; +const LOAD_POINT_MUTATION_MAX_MS = 400; +const LOAD_SET_ROW_AVG_ITERATIONS = 10; +const LOAD_SET_ROW_AVG_MAX_MS = 50; +const LOAD_TABLE_DELETE_MAX_MS = 20_000; +const LOAD_DERIVED_INIT_MAX_MS = 90_000; +const LOAD_DERIVED_COUNT_QUERY_MAX_MS = 10_000; function getTestDbUrls(): TestDb { const env = Reflect.get(import.meta, "env"); @@ -77,15 +87,6 @@ function createWorkload(seed: number, operationCount: number): WorkloadOperation return operations; } -function operationCountFromEnv(varName: string, fallback: number): number { - const env = Reflect.get(import.meta, "env"); - const raw = Reflect.get(env, varName); - if (typeof raw !== "string") return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed < 1) return fallback; - return Math.floor(parsed); -} - function logLine(message: string): void { console.log(`${message}\n`); } @@ -104,6 +105,53 @@ describe.sequential("bulldozer db performance (real postgres)", () => { return await sql.unsafe(toQueryableSqlQuery(query)); } + async function measureMs(label: string, fn: () => Promise): Promise<{ result: T, elapsedMs: number }> { + const startedAt = performance.now(); + const result = await fn(); + const elapsedMs = performance.now() - startedAt; + logLine(`[bulldozer-perf] ${label}: ${elapsedMs.toFixed(1)} ms`); + return { result, elapsedMs }; + } + + async function prefillStoredTableInSingleStatement(tableId: string, rowCount: number): Promise { + const externalId = `external:${tableId}`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT "seedRows"."keyPath", "seedRows"."value" + FROM ( + VALUES + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text), to_jsonb('rows'::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text), to_jsonb('metadata'::text)]::jsonb[], '{ "version": 1 }'::jsonb) + ) AS "seedRows"("keyPath", "value") + UNION ALL + SELECT + ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalId}::text), + to_jsonb('storage'::text), + to_jsonb('rows'::text), + to_jsonb(('seed-' || "n"::text)::text) + ]::jsonb[], + jsonb_build_object( + 'rowData', + jsonb_build_object( + 'team', + CASE + WHEN "n" % 4 = 0 THEN 'null'::jsonb + WHEN "n" % 4 = 1 THEN to_jsonb('alpha'::text) + WHEN "n" % 4 = 2 THEN to_jsonb('beta'::text) + ELSE to_jsonb('gamma'::text) + END, + 'value', + to_jsonb(("n" % 1000)::int) + ) + ) + FROM generate_series(1, ${rowCount}) AS "n" + `; + } + async function executeWorkload( fromTable: ReturnType>, operations: WorkloadOperation[], @@ -186,10 +234,8 @@ describe.sequential("bulldozer db performance (real postgres)", () => { }); it("reports ops/sec for baseline and composed example setup", async () => { - const warmupCount = operationCountFromEnv("STACK_BULLDOZER_PERF_WARMUP_OPS", DEFAULT_WARMUP_OPS); - const measuredCount = operationCountFromEnv("STACK_BULLDOZER_PERF_MEASURED_OPS", DEFAULT_MEASURED_OPS); - const warmupOperations = createWorkload(111, warmupCount); - const measuredOperations = createWorkload(222, measuredCount); + const warmupOperations = createWorkload(111, DEFAULT_WARMUP_OPS); + const measuredOperations = createWorkload(222, DEFAULT_MEASURED_OPS); const baseline = await benchmarkScenario({ name: "stored-table baseline", @@ -266,10 +312,202 @@ describe.sequential("bulldozer db performance (real postgres)", () => { const slowdownFactor = baseline.operationsPerSecond / composed.operationsPerSecond; logLine(`[bulldozer-perf] slowdown factor (baseline/composed): ${slowdownFactor.toFixed(2)}x`); - logLine(`[bulldozer-perf] config: warmup=${warmupCount}, measured=${measuredCount}`); + logLine(`[bulldozer-perf] config: warmup=${DEFAULT_WARMUP_OPS}, measured=${DEFAULT_MEASURED_OPS}`); expect(baseline.operationsPerSecond).toBeGreaterThan(0); expect(composed.operationsPerSecond).toBeGreaterThan(0); }); + + it("load test: prefilled stored table with hundreds of thousands of rows stays functional and fast", async () => { + const loadRowCount = DEFAULT_LOAD_ROW_COUNT; + const tableId = "load-prefilled-users"; + const externalTableId = `external:${tableId}`; + const table = declareStoredTable<{ value: number, team: string | null }>({ tableId }); + + const prefill = await measureMs(`load prefill (${loadRowCount} rows)`, async () => { + await prefillStoredTableInSingleStatement(tableId, loadRowCount); + }); + expect(prefill.elapsedMs).toBeLessThan(LOAD_PREFILL_MAX_MS); + + const metadataInitializedRows = await sql` + SELECT EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalTableId}::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + ) AS "initialized" + `; + expect(metadataInitializedRows[0].initialized).toBe(true); + + const listRowsQuery = table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const countRows = await measureMs("load count via listRowsInGroup", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(listRowsQuery)}) AS "rows" + `); + }); + expect(countRows.elapsedMs).toBeLessThan(LOAD_COUNT_QUERY_MAX_MS); + expect(Number(countRows.result[0].count)).toBe(loadRowCount); + + const setRowIterationTimes: number[] = []; + for (let i = 0; i < LOAD_SET_ROW_AVG_ITERATIONS; i++) { + const startedAt = performance.now(); + await runStatements(table.setRow( + `seed-${Math.floor(loadRowCount / 2) + i}`, + expr(jsonbLiteral({ team: "beta", value: 777 + i })), + )); + setRowIterationTimes.push(performance.now() - startedAt); + } + const setRowAverageMs = setRowIterationTimes.reduce((acc, value) => acc + value, 0) / setRowIterationTimes.length; + logLine(`[bulldozer-perf] load setRow average (${LOAD_SET_ROW_AVG_ITERATIONS} iterations): ${setRowAverageMs.toFixed(1)} ms`); + expect(setRowAverageMs).toBeLessThanOrEqual(LOAD_SET_ROW_AVG_MAX_MS); + + const pointDelete = await measureMs("load point delete (deleteRow existing)", async () => { + await runStatements(table.deleteRow(`seed-${Math.floor(loadRowCount / 2) - 1}`)); + }); + expect(pointDelete.elapsedMs).toBeLessThan(LOAD_POINT_MUTATION_MAX_MS); + + const countAfterDelete = await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(listRowsQuery)}) AS "rows" + `); + expect(Number(countAfterDelete[0].count)).toBe(loadRowCount - 1); + + const groupedByTeam = declareGroupByTable({ + tableId: "load-prefilled-users-by-team", + fromTable: table, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const mappedByTeam = declareMapTable({ + tableId: "load-prefilled-users-mapped", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen", + ( + CASE + WHEN (("rowData"->>'value')::int + 10) >= 700 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + ` }, + }); + const mappedTwice = declareMapTable({ + tableId: "load-prefilled-users-mapped-twice", + fromTable: mappedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + ("rowData"->'bucket') AS "bucket", + ((("rowData"->>'valuePlusTen')::int * 2)) AS "valueScaled" + ` }, + }); + const groupedByBucket = declareGroupByTable({ + tableId: "load-prefilled-users-by-bucket", + fromTable: mappedTwice, + groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` }, + }); + + const groupInit = await measureMs("load init groupedByTeam", async () => { + await runStatements(groupedByTeam.init()); + }); + expect(groupInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const mapInit = await measureMs("load init mappedByTeam", async () => { + await runStatements(mappedByTeam.init()); + }); + expect(mapInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const mapTwiceInit = await measureMs("load init mappedTwice", async () => { + await runStatements(mappedTwice.init()); + }); + expect(mapTwiceInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const bucketInit = await measureMs("load init groupedByBucket", async () => { + await runStatements(groupedByBucket.init()); + }); + expect(bucketInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + + const groupedCountQuery = groupedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const mappedCountQuery = mappedTwice.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const bucketCountQuery = groupedByBucket.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const derivedCounts = await measureMs("load count derived tables", async () => { + return await Promise.all([ + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(groupedCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`), + ]); + }); + expect(derivedCounts.elapsedMs).toBeLessThan(LOAD_DERIVED_COUNT_QUERY_MAX_MS); + expect(Number(derivedCounts.result[0][0].count)).toBe(loadRowCount - 1); + expect(Number(derivedCounts.result[1][0].count)).toBe(loadRowCount - 1); + expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1); + + await runStatements(table.setRow( + "seed-100000", + expr(jsonbLiteral({ team: "delta", value: 999 })), + )); + const deltaGroupedRows = await readRows(groupedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(deltaGroupedRows.some((row) => row.rowidentifier === "seed-100000")).toBe(true); + const highBucketRows = await readRows(groupedByBucket.listRowsInGroup({ + groupKey: expr(`to_jsonb('high'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const highBucketRow = highBucketRows.find((row) => row.rowidentifier === "seed-100000"); + expect(highBucketRow).toBeDefined(); + expect(highBucketRow?.rowdata).toEqual({ + team: "delta", + bucket: "high", + valueScaled: 2018, + }); + + const bulkDelete = await measureMs("load full table delete", async () => { + await runStatements(table.delete()); + }); + expect(bulkDelete.elapsedMs).toBeLessThan(LOAD_TABLE_DELETE_MAX_MS); + + const isInitializedRows = await sql` + SELECT EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalTableId}::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + ) AS "initialized" + `; + expect(isInitializedRows[0].initialized).toBe(false); + + logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`); + }, 180_000); }); From 863ee05f0224ff2da7fe3f8644f7e09931dd7196 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 11:32:38 -0700 Subject: [PATCH 07/40] Interface updates --- apps/backend/src/lib/bulldozer/db/index.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts index 61d0341d36..b655dc329c 100644 --- a/apps/backend/src/lib/bulldozer/db/index.ts +++ b/apps/backend/src/lib/bulldozer/db/index.ts @@ -694,24 +694,26 @@ export declare function declareFilterTable< export declare function declareSortTable< GK extends Json, - SK extends Json, + OldSK extends Json, + NewSK extends Json, RD extends RowData, >(options: { tableId: TableId, - fromTable: Table, - getSortKey: SqlMapper<{ rowData: RD }, { sortKey: SK }>, - compareSortKeys: (a: SqlExpression, b: SqlExpression) => SqlExpression, -}): Table; + fromTable: Table, + getSortKey: SqlMapper<{ rowIdentifier: RowIdentifier, oldSortKey: OldSK, rowData: RD }, { newSortKey: NewSK }>, + compareSortKeys: (a: SqlExpression, b: SqlExpression) => SqlExpression, +}): Table; export declare function declareLFoldTable< GK extends Json, OldRD extends RowData, NewRD extends RowData, + S extends Json, >(options: { tableId: TableId, fromTable: Table, - initialState: SqlExpression, - reducer: SqlMapper<{ state: Json, oldRowData: OldRD }, { newState: Json, newRowData: NewRD }>, + initialState: SqlExpression, + reducer: SqlMapper<{ oldState: S, oldRowData: OldRD }, { newState: S, newRowData: NewRD }>, }): Table; From 109cf5d01f03621bfe7b1e1ee5063ad0c145bfdc Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Tue, 24 Mar 2026 18:15:27 -0700 Subject: [PATCH 08/40] Bulldozer Studio --- AGENTS.md | 2 +- apps/backend/package.json | 5 +- .../migration.sql | 2 +- apps/backend/scripts/run-bulldozer-studio.ts | 1663 +++++++++++++++++ apps/backend/scripts/run-cron-jobs.ts | 1 + .../src/lib/bulldozer/db/example-schema.ts | 95 + .../src/lib/bulldozer/db/index.test.ts | 94 + apps/backend/src/lib/bulldozer/db/index.ts | 239 ++- apps/backend/src/prisma-client.tsx | 2 +- apps/dev-launchpad/public/index.html | 10 + 10 files changed, 2032 insertions(+), 81 deletions(-) create mode 100644 apps/backend/scripts/run-bulldozer-studio.ts create mode 100644 apps/backend/src/lib/bulldozer/db/example-schema.ts diff --git a/AGENTS.md b/AGENTS.md index 4bdb14eef6..30a32c4458 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ This file provides guidance to coding agents when working with code in this repo ### Essential Commands - **Install dependencies**: `pnpm install` - **Run tests**: `pnpm test run` (uses Vitest). You can filter with `pnpm test run `. The `run` is important to not trigger watch mode -- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually. +- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually. Use `pnpm -C lint` to lint a specific package. - **Type check**: `pnpm typecheck` #### Extra commands diff --git a/apps/backend/package.json b/apps/backend/package.json index 26b5f8a795..139951b88a 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -11,7 +11,7 @@ "with-env:dev": "dotenv -c development --", "with-env:prod": "dotenv -c production --", "with-env:test": "dotenv -c test --", - "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"", + "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs,bulldozer-studio\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\" \"pnpm run run-bulldozer-studio\"", "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev", "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev", "build": "pnpm run codegen && next build", @@ -48,7 +48,8 @@ "run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts", "run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts", "verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts", - "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts" + "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts", + "run-bulldozer-studio": "pnpm run with-env:dev tsx watch --clear-screen=false scripts/run-bulldozer-studio.ts" }, "prisma": { "seed": "pnpm run db-seed-script" diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql index 18a23438e8..388283cfcb 100644 --- a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql @@ -1,6 +1,6 @@ -- CreateTable CREATE TABLE "BulldozerStorageEngine" ( - "id" UUID NOT NULL, + "id" UUID NOT NULL DEFAULT gen_random_uuid(), "keyPath" JSONB[] NOT NULL, "keyPathParent" JSONB[] GENERATED ALWAYS AS ( CASE diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts new file mode 100644 index 0000000000..3fc40c9b65 --- /dev/null +++ b/apps/backend/scripts/run-bulldozer-studio.ts @@ -0,0 +1,1663 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings"; +import http from "node:http"; +import { exampleFungibleLedgerSchema } from "../src/lib/bulldozer/db/example-schema"; +import { toQueryableSqlQuery } from "../src/lib/bulldozer/db/index"; +import { globalPrismaClient, retryTransaction } from "../src/prisma-client"; + +type JsonPrimitive = string | number | boolean | null; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; + +type StudioTable = { + tableId: unknown, + inputTables?: StudioTable[], + debugArgs?: Record, + listGroups(options: { start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + listRowsInGroup(options: { groupKey?: SqlExpression, start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + init(): SqlStatement[], + delete(): SqlStatement[], + isInitialized(): SqlExpression, + registerRowChangeTrigger(trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[]): { deregister: () => void }, +}; + +type StudioStoredTable = StudioTable & { + setRow(rowIdentifier: string, rowData: SqlExpression>): SqlStatement[], + deleteRow(rowIdentifier: string): SqlStatement[], +}; + +type StudioTableRecord = { + id: string, + name: string, + table: StudioTable, +}; + +const STUDIO_PORT = Number(`${getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81")}39`); +const BULLDOZER_LOCK_ID = 7857391; +const STUDIO_INSTANCE_ID = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isStudioTable(value: unknown): value is StudioTable { + if (!isRecord(value)) return false; + return typeof Reflect.get(value, "listGroups") === "function" + && typeof Reflect.get(value, "listRowsInGroup") === "function" + && typeof Reflect.get(value, "init") === "function" + && typeof Reflect.get(value, "delete") === "function" + && typeof Reflect.get(value, "isInitialized") === "function" + && typeof Reflect.get(value, "registerRowChangeTrigger") === "function"; +} + +function isStudioStoredTable(value: StudioTable): value is StudioStoredTable { + return typeof Reflect.get(value, "setRow") === "function" + && typeof Reflect.get(value, "deleteRow") === "function"; +} + +function requireRecord(value: unknown, errorMessage: string): Record { + if (!isRecord(value)) throw new StackAssertionError(errorMessage); + return value; +} + +function requireString(value: unknown, errorMessage: string): string { + if (typeof value !== "string") throw new StackAssertionError(errorMessage); + return value; +} + +function requireStringArray(value: unknown, errorMessage: string): string[] { + if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) { + throw new StackAssertionError(errorMessage); + } + return value; +} + +function quoteSqlStringLiteral(input: string): string { + return `'${input.replaceAll("'", "''")}'`; +} +function quoteSqlIdentifier(input: string): string { + if (input.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) == null) { + throw new StackAssertionError("Invalid SQL identifier for Bulldozer Studio query.", { input }); + } + return `"${input}"`; +} + +function quoteSqlJsonbLiteral(input: unknown): string { + return `${quoteSqlStringLiteral(JSON.stringify(input))}::jsonb`; +} + +function keyPathSqlLiteral(pathSegments: string[]): string { + if (pathSegments.length === 0) return "ARRAY[]::jsonb[]"; + return `ARRAY[${pathSegments.map((segment) => quoteSqlJsonbLiteral(segment)).join(", ")}]::jsonb[]`; +} + +function tableIdToString(tableId: unknown): string { + if (typeof tableId === "string") return tableId; + return JSON.stringify(tableId); +} + +function createTableRegistry(schema: Record): { + tables: StudioTableRecord[], + tableById: Map, + idByTable: Map, +} { + const tables: StudioTableRecord[] = []; + const idByTable = new Map(); + + for (const [name, value] of Object.entries(schema)) { + if (!isStudioTable(value)) continue; + const id = name; + const record: StudioTableRecord = { id, name, table: value }; + tables.push(record); + idByTable.set(value, id); + } + + if (tables.length === 0) { + throw new StackAssertionError("No studio-compatible tables found in schema object."); + } + + const tableById = new Map(tables.map((table) => [table.id, table])); + return { tables, tableById, idByTable }; +} + +const schemaObject: Record = exampleFungibleLedgerSchema; +const registry = createTableRegistry(schemaObject); + +function toExecutableSqlCteStatement(statements: SqlStatement[]): string { + const cteStatements = statements.map((statement, index) => { + const outputName = statement.outputName ?? `unnamed_statement_${index}`; + return `${quoteSqlIdentifier(outputName)} AS (\n${statement.sql}\n)`; + }).join(",\n"); + + return `WITH __dummy_statement_1__ AS (SELECT 1),\n${cteStatements},\n__dummy_statement_2__ AS (SELECT 1)\nSELECT 1;`; +} + +async function executeStatements(statements: SqlStatement[]): Promise { + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID})`); + await tx.$executeRawUnsafe(toExecutableSqlCteStatement(statements)); + }); +} + +async function queryRows(query: SqlQuery): Promise { + const rows = await retryTransaction(globalPrismaClient, async (tx) => { + return await tx.$queryRawUnsafe(toQueryableSqlQuery(query)); + }); + if (!Array.isArray(rows)) throw new StackAssertionError("Expected SQL query to return an array of rows."); + return rows; +} + +async function readBoolean(expression: SqlExpression): Promise { + const rows = await retryTransaction(globalPrismaClient, async (tx) => { + return await tx.$queryRawUnsafe>>(`SELECT (${expression.sql}) AS "value"`); + }); + if (!Array.isArray(rows) || rows.length === 0 || !isRecord(rows[0])) { + throw new StackAssertionError("Expected boolean expression query to return one row."); + } + return Reflect.get(rows[0], "value") === true; +} + +function valueFromRow(row: unknown, key: string): unknown { + if (!isRecord(row)) return null; + return Reflect.get(row, key); +} + +async function getTableSnapshot(record: StudioTableRecord): Promise<{ + id: string, + name: string, + tableId: string, + operator: string, + dependencies: string[], + debugArgs: Record, + supportsSetRow: boolean, + supportsDeleteRow: boolean, + initialized: boolean, +}> { + const inputTables = record.table.inputTables ?? []; + const debugArgs = record.table.debugArgs ?? {}; + const dependsOn = inputTables.map((inputTable) => { + return registry.idByTable.get(inputTable) ?? tableIdToString(inputTable.tableId); + }); + const operatorValue = Reflect.get(debugArgs, "operator"); + const operator = typeof operatorValue === "string" ? operatorValue : "unknown"; + + return { + id: record.id, + name: record.name, + tableId: tableIdToString(record.table.tableId), + operator, + dependencies: dependsOn, + debugArgs, + supportsSetRow: isStudioStoredTable(record.table), + supportsDeleteRow: isStudioStoredTable(record.table), + initialized: await readBoolean(record.table.isInitialized()), + }; +} + +async function getTableDetails(record: StudioTableRecord): Promise<{ + table: Awaited>, + groups: Array<{ groupKey: unknown, rows: Array<{ rowIdentifier: unknown, rowSortKey: unknown, rowData: unknown }> }>, + totalRows: number, +}> { + const table = record.table; + const tableSnapshot = await getTableSnapshot(record); + const groupsRaw = await queryRows(table.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const allRowsRaw = await queryRows(table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + + const rowsByGroup = new Map }>(); + + for (const groupRow of groupsRaw) { + const groupKey = valueFromRow(groupRow, "groupkey"); + const key = JSON.stringify(groupKey); + rowsByGroup.set(key, { groupKey, rows: [] }); + } + + for (const row of allRowsRaw) { + const hasGroupKey = isRecord(row) && Reflect.has(row, "groupkey"); + const groupKey = hasGroupKey ? valueFromRow(row, "groupkey") : null; + const key = JSON.stringify(groupKey); + const existing = rowsByGroup.get(key) ?? { groupKey, rows: [] }; + existing.rows.push({ + rowIdentifier: valueFromRow(row, "rowidentifier"), + rowSortKey: valueFromRow(row, "rowsortkey"), + rowData: valueFromRow(row, "rowdata"), + }); + rowsByGroup.set(key, existing); + } + + const groups = [...rowsByGroup.values()].sort((a, b) => { + return stringCompare(JSON.stringify(a.groupKey), JSON.stringify(b.groupKey)); + }); + + return { + table: tableSnapshot, + groups, + totalRows: allRowsRaw.length, + }; +} + +async function getRawNode(pathSegments: string[]): Promise<{ + path: string[], + value: unknown, + children: Array<{ segment: string, hasChildren: boolean }>, +}> { + const keyPathLiteral = keyPathSqlLiteral(pathSegments); + const { valueRows, childrenRows } = await retryTransaction(globalPrismaClient, async (tx) => { + const valueRows = await tx.$queryRawUnsafe>>(` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${keyPathLiteral} + `); + const childrenRows = await tx.$queryRawUnsafe>>(` + SELECT + ("child"."keyPath"[cardinality("child"."keyPath")] #>> '{}') AS "segment", + EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "grandChild" + WHERE "grandChild"."keyPathParent" = "child"."keyPath" + ) AS "hasChildren" + FROM "BulldozerStorageEngine" AS "child" + WHERE "child"."keyPathParent" = ${keyPathLiteral} + ORDER BY "segment" + `); + return { valueRows, childrenRows }; + }); + + const children = childrenRows + .filter((row) => isRecord(row) && typeof Reflect.get(row, "segment") === "string") + .map((row) => ({ + segment: requireString(Reflect.get(row, "segment"), "Expected segment to be a string."), + hasChildren: Reflect.get(row, "hasChildren") === true, + })); + + return { + path: pathSegments, + value: Array.isArray(valueRows) && valueRows.length > 0 ? valueFromRow(valueRows[0], "value") : null, + children, + }; +} + +async function readRequestBody(request: http.IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of request) { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === "string") { + chunks.push(Buffer.from(chunk)); + } + } + return Buffer.concat(chunks).toString("utf8"); +} + +async function readJsonBody(request: http.IncomingMessage): Promise { + const rawBody = await readRequestBody(request); + if (rawBody.trim() === "") return {}; + return JSON.parse(rawBody); +} + +function sendJson(response: http.ServerResponse, statusCode: number, payload: unknown): void { + response.statusCode = statusCode; + response.setHeader("Content-Type", "application/json; charset=utf-8"); + response.end(JSON.stringify(payload)); +} + +function sendHtml(response: http.ServerResponse, html: string): void { + response.statusCode = 200; + response.setHeader("Content-Type", "text/html; charset=utf-8"); + response.end(html); +} + +function getStudioPageHtml(): string { + return ` + + + + + Bulldozer Studio + + + +
+
+
+
Bulldozer Studio
+ + + + + +
+
+
ready
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+ + +
+
Action failed
+ 

+      

+        
+      

+    

+  
+
+  
+
+`;
+}
+
+async function handleRequest(request: http.IncomingMessage, response: http.ServerResponse): Promise {
+  const requestUrl = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
+  const pathname = requestUrl.pathname;
+  const method = request.method ?? "GET";
+
+  if (method === "GET" && pathname === "/") {
+    sendHtml(response, getStudioPageHtml());
+    return;
+  }
+
+  if (method === "GET" && pathname === "/api/version") {
+    sendJson(response, 200, { version: STUDIO_INSTANCE_ID });
+    return;
+  }
+
+  if (method === "GET" && pathname === "/api/schema") {
+    const tables = await Promise.all(registry.tables.map((table) => getTableSnapshot(table)));
+    sendJson(response, 200, { tables });
+    return;
+  }
+
+  if (pathname.startsWith("/api/table/")) {
+    const pathParts = pathname.split("/").filter(Boolean);
+    const tableId = decodeURIComponent(pathParts[2] ?? "");
+    const record = registry.tableById.get(tableId);
+    if (!record) {
+      sendJson(response, 404, { error: `Unknown table: ${tableId}` });
+      return;
+    }
+
+    if (method === "GET" && pathParts[3] === "details") {
+      const details = await getTableDetails(record);
+      sendJson(response, 200, details);
+      return;
+    }
+
+    if (method === "POST" && pathParts[3] === "init") {
+      await executeStatements(record.table.init());
+      sendJson(response, 200, { ok: true });
+      return;
+    }
+
+    if (method === "POST" && pathParts[3] === "delete") {
+      await executeStatements(record.table.delete());
+      sendJson(response, 200, { ok: true });
+      return;
+    }
+
+    if (method === "POST" && pathParts[3] === "set-row") {
+      if (!isStudioStoredTable(record.table)) {
+        sendJson(response, 400, { error: "This table does not support setRow." });
+        return;
+      }
+      const body = requireRecord(await readJsonBody(request), "set-row body must be an object.");
+      const rowIdentifier = requireString(Reflect.get(body, "rowIdentifier"), "rowIdentifier must be a string.");
+      const rowData = requireRecord(Reflect.get(body, "rowData"), "rowData must be a JSON object.");
+      await executeStatements(record.table.setRow(
+        rowIdentifier,
+        { type: "expression", sql: quoteSqlJsonbLiteral(rowData) },
+      ));
+      sendJson(response, 200, { ok: true });
+      return;
+    }
+
+    if (method === "POST" && pathParts[3] === "delete-row") {
+      if (!isStudioStoredTable(record.table)) {
+        sendJson(response, 400, { error: "This table does not support deleteRow." });
+        return;
+      }
+      const body = requireRecord(await readJsonBody(request), "delete-row body must be an object.");
+      const rowIdentifier = requireString(Reflect.get(body, "rowIdentifier"), "rowIdentifier must be a string.");
+      await executeStatements(record.table.deleteRow(rowIdentifier));
+      sendJson(response, 200, { ok: true });
+      return;
+    }
+  }
+
+  if (method === "GET" && pathname === "/api/raw/node") {
+    const pathParam = requestUrl.searchParams.get("path") ?? "[]";
+    const parsedPath = JSON.parse(pathParam);
+    const pathSegments = requireStringArray(parsedPath, "path must be a string[]");
+    const node = await getRawNode(pathSegments);
+    sendJson(response, 200, node);
+    return;
+  }
+
+  if (method === "POST" && pathname === "/api/raw/upsert") {
+    const body = requireRecord(await readJsonBody(request), "raw upsert body must be an object.");
+    const pathSegments = requireStringArray(Reflect.get(body, "pathSegments"), "pathSegments must be a string[]");
+    const value = Reflect.get(body, "value") ?? null;
+    await retryTransaction(globalPrismaClient, async (tx) => {
+      await tx.$executeRawUnsafe(`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        VALUES (gen_random_uuid(), ${keyPathSqlLiteral(pathSegments)}, ${quoteSqlJsonbLiteral(value)})
+        ON CONFLICT ("keyPath") DO UPDATE
+        SET "value" = EXCLUDED."value"
+      `);
+    });
+    sendJson(response, 200, { ok: true });
+    return;
+  }
+
+  if (method === "POST" && pathname === "/api/raw/delete") {
+    const body = requireRecord(await readJsonBody(request), "raw delete body must be an object.");
+    const pathSegments = requireStringArray(Reflect.get(body, "pathSegments"), "pathSegments must be a string[]");
+    await retryTransaction(globalPrismaClient, async (tx) => {
+      await tx.$executeRawUnsafe(`
+        DELETE FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ${keyPathSqlLiteral(pathSegments)}
+      `);
+    });
+    sendJson(response, 200, { ok: true });
+    return;
+  }
+
+  sendJson(response, 404, { error: `Route not found: ${method} ${pathname}` });
+}
+
+async function main(): Promise {
+  const server = http.createServer((request, response) => {
+    handleRequest(request, response).then(
+      () => undefined,
+      (error) => {
+        console.error(error);
+        const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
+        sendJson(response, 500, { error: message });
+      },
+    );
+  });
+
+  server.listen(STUDIO_PORT, () => {
+    console.log(`Bulldozer Studio running on http://localhost:${STUDIO_PORT}`);
+  });
+
+  const shutdown = async () => {
+    server.close();
+  };
+  process.on("SIGINT", () => {
+    shutdown().then(() => process.exit(0), () => process.exit(1));
+  });
+  process.on("SIGTERM", () => {
+    shutdown().then(() => process.exit(0), () => process.exit(1));
+  });
+}
+
+main().then(
+  () => undefined,
+  (error) => {
+    console.error(error);
+    process.exit(1);
+  },
+);
diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts
index 98b9680ce5..df61f311c4 100644
--- a/apps/backend/scripts/run-cron-jobs.ts
+++ b/apps/backend/scripts/run-cron-jobs.ts
@@ -25,6 +25,7 @@ async function main() {
 
   for (const endpoint of endpoints) {
     runAsynchronously(async () => {
+      await wait(30_000); // Wait a few seconds to make sure the server is fully started
       while (true) {
         const runResult = await Result.fromPromise(run(endpoint));
         if (runResult.status === "error") {
diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts
new file mode 100644
index 0000000000..f95b986aa7
--- /dev/null
+++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts
@@ -0,0 +1,95 @@
+import { declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
+
+const mapper = (sql: string) => ({ type: "mapper" as const, sql });
+
+/**
+ * Example fungible-asset ledger schema composed from Bulldozer table operators.
+ *
+ * This file intentionally declares tables only; it does not call init/delete.
+ */
+export const exampleFungibleLedgerSchema = (() => {
+  // Base append/update table for raw ledger entries.
+  const ledgerEntries = declareStoredTable<{
+    accountId: string,
+    asset: string,
+    amount: string,
+    side: "credit" | "debit",
+    txHash: string,
+    blockNumber: number,
+    timestamp: string,
+    counterparty: string | null,
+    memo: string | null,
+  }>({
+    tableId: "bulldozer-example-ledger-entries",
+  });
+
+  // Group the ledger by account.
+  const entriesByAccount = declareGroupByTable({
+    tableId: "bulldozer-example-ledger-entries-by-account",
+    fromTable: ledgerEntries,
+    groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`),
+  });
+
+  // Group the ledger by asset symbol.
+  const entriesByAsset = declareGroupByTable({
+    tableId: "bulldozer-example-ledger-entries-by-asset",
+    fromTable: ledgerEntries,
+    groupBy: mapper(`"rowData"->'asset' AS "groupKey"`),
+  });
+
+  // Enrich account-grouped rows with normalized direction and numeric amount.
+  const accountEntriesNormalized = declareMapTable({
+    tableId: "bulldozer-example-ledger-account-entries-normalized",
+    fromTable: entriesByAccount,
+    mapper: mapper(`
+      ("rowData"->'accountId') AS "accountId",
+      ("rowData"->'asset') AS "asset",
+      ("rowData"->'side') AS "side",
+      (("rowData"->>'amount')::numeric) AS "amountNumeric",
+      CASE
+        WHEN "rowData"->>'side' = 'credit' THEN 'inflow'
+        ELSE 'outflow'
+      END AS "flowDirection",
+      ("rowData"->'txHash') AS "txHash",
+      ("rowData"->'timestamp') AS "timestamp"
+    `),
+  });
+
+  // Build an account+asset partition from normalized entries.
+  const accountAssetPartitions = declareGroupByTable({
+    tableId: "bulldozer-example-ledger-account-asset-partitions",
+    fromTable: accountEntriesNormalized,
+    groupBy: mapper(`
+      jsonb_build_object(
+        'accountId', "rowData"->'accountId',
+        'asset', "rowData"->'asset'
+      ) AS "groupKey"
+    `),
+  });
+
+  // Enrich asset-grouped rows for downstream analytics views.
+  const assetEntriesNormalized = declareMapTable({
+    tableId: "bulldozer-example-ledger-asset-entries-normalized",
+    fromTable: entriesByAsset,
+    mapper: mapper(`
+      ("rowData"->'asset') AS "asset",
+      ("rowData"->'accountId') AS "accountId",
+      (("rowData"->>'amount')::numeric) AS "amountNumeric",
+      CASE
+        WHEN "rowData"->>'side' = 'credit' THEN 1
+        ELSE -1
+      END AS "signedDirection",
+      ("rowData"->'blockNumber') AS "blockNumber",
+      ("rowData"->'txHash') AS "txHash"
+    `),
+  });
+
+  return {
+    ledgerEntries,
+    entriesByAccount,
+    entriesByAsset,
+    accountEntriesNormalized,
+    accountAssetPartitions,
+    assetEntriesNormalized,
+  };
+})();
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index 6ee4dbac33..b425468700 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -845,6 +845,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(groupsAfterDelete).toEqual([]);
   });
 
+  test("groupBy deletes stale group paths from storage", async () => {
+    const { fromTable, groupedTable } = createGroupedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("u1"));
+
+    const staleGroupPaths = await sql`
+      SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPath"[1:4] = ARRAY[
+        to_jsonb('table'::text),
+        to_jsonb('external:users-by-team'::text),
+        to_jsonb('storage'::text),
+        to_jsonb('groups'::text)
+      ]::jsonb[]
+      AND cardinality("keyPath") > 4
+      ORDER BY "keyPath"
+    `;
+    expect(staleGroupPaths).toEqual([]);
+  });
+
   test("groupBy listRowsInGroup handles missing groups and exclusive bounds", async () => {
     const { fromTable, groupedTable } = createGroupedTable();
     await runStatements(fromTable.init());
@@ -880,6 +903,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(inclusiveRows).toHaveLength(2);
   });
 
+  test("groupBy listRowsInGroup (all groups) handles 'rows' collisions in group key and row identifier", async () => {
+    const { fromTable, groupedTable } = createGroupedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+
+    const allRows = await readRows(groupedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    const normalizedRows = allRows
+      .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata }))
+      .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`));
+
+    expect(normalizedRows).toEqual([
+      { groupKey: "alpha", rowIdentifier: "rows", rowData: { team: "alpha", value: 2 } },
+      { groupKey: "rows", rowIdentifier: "u1", rowData: { team: "rows", value: 1 } },
+    ]);
+  });
+
   test("groupBy multiple triggers run in one transaction", async () => {
     const { fromTable, groupedTable } = createGroupedTable();
     await runStatements(fromTable.init());
@@ -1105,6 +1151,54 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(exclusiveRows).toEqual([]);
   });
 
+  test("mapTable listRowsInGroup (all groups) handles 'rows' collisions in group key and row identifier", async () => {
+    const { fromTable, groupedTable, mappedTable } = createMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(mappedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+
+    const allRows = await readRows(mappedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    const normalizedRows = allRows
+      .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata }))
+      .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`));
+
+    expect(normalizedRows).toEqual([
+      { groupKey: "alpha", rowIdentifier: "rows", rowData: { team: "alpha", mappedValue: 102 } },
+      { groupKey: "rows", rowIdentifier: "u1", rowData: { team: "rows", mappedValue: 101 } },
+    ]);
+  });
+
+  test("mapTable deletes stale group paths from storage", async () => {
+    const { fromTable, groupedTable, mappedTable } = createMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(mappedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("u1"));
+
+    const staleGroupPaths = await sql`
+      SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPath"[1:4] = ARRAY[
+        to_jsonb('table'::text),
+        to_jsonb('external:users-by-team-mapped'::text),
+        to_jsonb('storage'::text),
+        to_jsonb('groups'::text)
+      ]::jsonb[]
+      AND cardinality("keyPath") > 4
+      ORDER BY "keyPath"
+    `;
+    expect(staleGroupPaths).toEqual([]);
+  });
+
   test("stacked map tables propagate updates across multiple mapping layers", async () => {
     const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables();
     await runStatements(fromTable.init());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index b655dc329c..3ee7abf203 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -4,6 +4,8 @@ import { deindent, templateIdentity } from "@stackframe/stack-shared/dist/utils/
 
 export type Table = {
   tableId: TableId,
+  inputTables: Table[],
+  debugArgs: Record,
 
   // Query groups and rows
   listGroups(options: { start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery>,
@@ -38,15 +40,20 @@ export function declareStoredTable(options: {
   // Note that this table has only one group and sort key (null), so all groups and rows are always returned by every filter.
   return {
     tableId: options.tableId,
+    inputTables: [],
+    debugArgs: {
+      operator: "stored",
+      tableId: tableIdToDebugString(options.tableId),
+    },
     compareGroupKeys: (a, b) => sqlExpression` 0 `,
     compareSortKeys: (a, b) => sqlExpression` 0 `,
     init: () => [sqlStatement`
-      INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+      INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
       VALUES
-      (${getTablePath(options.tableId)}, 'null'::jsonb),
-      (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
-      (${getStorageEnginePath(options.tableId, ["rows"])}, 'null'::jsonb),
-      (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
+      (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+      (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
+      (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["rows"])}, 'null'::jsonb),
+      (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
     `],
     delete: () => [sqlStatement`
       WITH RECURSIVE "pathsToDelete" AS (
@@ -100,8 +107,9 @@ export function declareStoredTable(options: {
           WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[]
         `.toStatement(oldRowsTableName),
         sqlQuery`
-          INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
           VALUES (
+            gen_random_uuid(),
             ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[],
             ${rowValue}::jsonb
           )
@@ -209,18 +217,24 @@ export function declareGroupByTable<
         WHERE ${isInitializedExpression}
       `.toStatement(mappedChangesTableName),
       sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
-        SELECT DISTINCT
-          ${getGroupKeyPath(sqlExpression`"newGroupKey"`)}::jsonb[],
-          'null'::jsonb
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasNewRow"
-        UNION
-        SELECT DISTINCT
-          ${getGroupRowsPath(sqlExpression`"newGroupKey"`)}::jsonb[],
-          'null'::jsonb
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasNewRow"
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          "insertRows"."keyPath",
+          "insertRows"."value"
+        FROM (
+          SELECT DISTINCT
+            ${getGroupKeyPath(sqlExpression`"newGroupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
+          WHERE "hasNewRow"
+          UNION
+          SELECT DISTINCT
+            ${getGroupRowsPath(sqlExpression`"newGroupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
+          WHERE "hasNewRow"
+        ) AS "insertRows"
         ON CONFLICT ("keyPath") DO NOTHING
       `,
       sqlStatement`
@@ -233,8 +247,9 @@ export function declareGroupByTable<
           )}::jsonb[]
       `,
       sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
         SELECT
+          gen_random_uuid(),
           ${getGroupRowPath(
             sqlExpression`"newGroupKey"`,
             sqlExpression`to_jsonb("rowIdentifier"::text)`,
@@ -257,6 +272,22 @@ export function declareGroupByTable<
             SELECT 1
             FROM "BulldozerStorageEngine" AS "groupRow"
             WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[]
+              AND NOT EXISTS (
+                SELECT 1
+                FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "deletingRow"
+                WHERE "deletingRow"."hasOldRow"
+                  AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
+                  AND "groupRow"."keyPath" = ${getGroupRowPath(
+                    sqlExpression`"deletingRow"."oldGroupKey"`,
+                    sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`,
+                  )}::jsonb[]
+              )
+          )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "insertingRow"
+            WHERE "insertingRow"."hasNewRow"
+              AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
           )
       `,
       sqlQuery`
@@ -290,6 +321,13 @@ export function declareGroupByTable<
 
   return {
     tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "groupBy",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      groupBySql: options.groupBy.sql,
+    },
     compareGroupKeys: (a, b) => sqlExpression` 0 `,
     compareSortKeys: (a, b) => sqlExpression` 0 `,
     init: () => {
@@ -298,12 +336,12 @@ export function declareGroupByTable<
 
       return [
         sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
           VALUES
-          (${getTablePath(options.tableId)}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
+          (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
         `,
         options.fromTable.listRowsInGroup({
           start: "start",
@@ -330,24 +368,30 @@ export function declareGroupByTable<
           ) AS "mapped" ON true
         `.toStatement(fromTableRowsWithGroupKeyTableName),
         sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
-          SELECT DISTINCT
-            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[],
-            'null'::jsonb
-          FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
-          UNION
-          SELECT DISTINCT
-            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[],
-            'null'::jsonb
-          FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
-          UNION
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
           SELECT
-            ${getGroupRowPath(
-              sqlExpression`"groupKey"`,
-              sqlExpression`to_jsonb("rowIdentifier"::text)`,
-            )}::jsonb[],
-            jsonb_build_object('rowData', "rowData")
-          FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
+            UNION
+            SELECT DISTINCT
+              ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
+            UNION
+            SELECT
+              ${getGroupRowPath(
+                sqlExpression`"groupKey"`,
+                sqlExpression`to_jsonb("rowIdentifier"::text)`,
+              )}::jsonb[] AS "keyPath",
+              jsonb_build_object('rowData', "rowData") AS "value"
+            FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)}
+          ) AS "insertRows"
         `,
       ];
     },
@@ -400,6 +444,7 @@ export function declareGroupByTable<
       FROM "BulldozerStorageEngine" AS "groupRows"
       INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath"
       WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND cardinality("groupRows"."keyPath") = cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]) + 2
         AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text)
         AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
     `,
@@ -471,18 +516,24 @@ export function declareMapTable<
         WHERE ${isInitializedExpression}
       `.toStatement(mappedChangesTableName),
       sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
-        SELECT DISTINCT
-          ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[],
-          'null'::jsonb
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasNewRow"
-        UNION
-        SELECT DISTINCT
-          ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[],
-          'null'::jsonb
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasNewRow"
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          "insertRows"."keyPath",
+          "insertRows"."value"
+        FROM (
+          SELECT DISTINCT
+            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
+          WHERE "hasNewRow"
+          UNION
+          SELECT DISTINCT
+            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
+          WHERE "hasNewRow"
+        ) AS "insertRows"
         ON CONFLICT ("keyPath") DO NOTHING
       `,
       sqlStatement`
@@ -495,8 +546,9 @@ export function declareMapTable<
           )}::jsonb[]
       `,
       sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
         SELECT
+          gen_random_uuid(),
           ${getGroupRowPath(
             sqlExpression`"groupKey"`,
             sqlExpression`to_jsonb("rowIdentifier"::text)`,
@@ -519,6 +571,22 @@ export function declareMapTable<
             SELECT 1
             FROM "BulldozerStorageEngine" AS "groupRow"
             WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[]
+              AND NOT EXISTS (
+                SELECT 1
+                FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "deletingRow"
+                WHERE "deletingRow"."hasOldRow"
+                  AND "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+                  AND "groupRow"."keyPath" = ${getGroupRowPath(
+                    sqlExpression`"deletingRow"."groupKey"`,
+                    sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`,
+                  )}::jsonb[]
+              )
+          )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "insertingRow"
+            WHERE "insertingRow"."hasNewRow"
+              AND "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
           )
       `,
       sqlQuery`
@@ -538,6 +606,13 @@ export function declareMapTable<
 
   return {
     tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "map",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      mapperSql: options.mapper.sql,
+    },
     compareGroupKeys: options.fromTable.compareGroupKeys,
     compareSortKeys: (a, b) => sqlExpression` 0 `,
     init: () => {
@@ -547,12 +622,12 @@ export function declareMapTable<
 
       return [
         sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
           VALUES
-          (${getTablePath(options.tableId)}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
-          (${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
+          (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
         `,
         options.fromTable.listGroups({
           start: "start",
@@ -595,24 +670,30 @@ export function declareMapTable<
           ) AS "mapped" ON true
         `.toStatement(mappedRowsTableName),
         sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
-          SELECT DISTINCT
-            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[],
-            'null'::jsonb
-          FROM ${quoteSqlIdentifier(mappedRowsTableName)}
-          UNION
-          SELECT DISTINCT
-            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[],
-            'null'::jsonb
-          FROM ${quoteSqlIdentifier(mappedRowsTableName)}
-          UNION
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
           SELECT
-            ${getGroupRowPath(
-              sqlExpression`"groupKey"`,
-              sqlExpression`to_jsonb("rowIdentifier"::text)`,
-            )}::jsonb[],
-            jsonb_build_object('rowData', "rowData")
-          FROM ${quoteSqlIdentifier(mappedRowsTableName)}
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
+            UNION
+            SELECT DISTINCT
+              ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
+            UNION
+            SELECT
+              ${getGroupRowPath(
+                sqlExpression`"groupKey"`,
+                sqlExpression`to_jsonb("rowIdentifier"::text)`,
+              )}::jsonb[] AS "keyPath",
+              jsonb_build_object('rowData', "rowData") AS "value"
+            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
+          ) AS "insertRows"
         `,
       ];
     },
@@ -672,6 +753,7 @@ export function declareMapTable<
       FROM "BulldozerStorageEngine" AS "groupRows"
       INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath"
       WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND cardinality("groupRows"."keyPath") = cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]) + 2
         AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text)
         AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
     `,
@@ -804,6 +886,11 @@ function getTablePathSegments(tableId: TableId): SqlExpression[] {
     ...tableIdWithParents.reverse().flatMap(id => ["table", id]),
   ].map(id => quoteSqlJsonbLiteral(id));
 }
+function tableIdToDebugString(tableId: TableId): string {
+  return typeof tableId === "string"
+    ? tableId
+    : JSON.stringify(tableId);
+}
 function singleNullSortKeyRangePredicate(options: {
   start: SqlExpression | "start",
   end: SqlExpression | "end",
diff --git a/apps/backend/src/prisma-client.tsx b/apps/backend/src/prisma-client.tsx
index c06fe3a21a..2fef5229c7 100644
--- a/apps/backend/src/prisma-client.tsx
+++ b/apps/backend/src/prisma-client.tsx
@@ -1,5 +1,4 @@
 import { Prisma, PrismaClient } from "@/generated/prisma/client";
-import { getStackServerApp } from "@/stack";
 import { PrismaNeon } from "@prisma/adapter-neon";
 import { PrismaPg } from '@prisma/adapter-pg';
 import { readReplicas } from '@prisma/extension-read-replicas';
@@ -60,6 +59,7 @@ async function resolveNeonConnectionString(entry: string): Promise {
   if (!isUuid(entry)) {
     return entry;
   }
+  const { getStackServerApp } = await import("@/stack");
   const store = await getStackServerApp().getDataVaultStore('neon-connection-strings');
   const secret = "no client side encryption";
   const value = await store.getValue(entry, { secret });
diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html
index 828ddfec1b..b65ecb695e 100644
--- a/apps/dev-launchpad/public/index.html
+++ b/apps/dev-launchpad/public/index.html
@@ -324,6 +324,16 @@ 
Background services

           importance: 1,
           img: "https://www.svgrepo.com/show/374002/replication.svg",
         },
+        {
+          name: "Bulldozer Studio",
+          portSuffix: "39",
+          description: [
+            "Bulldozer table graph and editor",
+            "Includes raw storage debug browser",
+          ],
+          importance: 1,
+          img: "https://www.svgrepo.com/show/349299/database.svg",
+        },
         {
           name: "JS example",
           portSuffix: "19",

From 53f73025506a9c39f0f8e57a7438060a5f15cc30 Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Tue, 24 Mar 2026 21:06:44 -0700
Subject: [PATCH 09/40] Remove unnecessary table

---
 apps/backend/src/lib/bulldozer/db/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index 3ee7abf203..eaa4881c37 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -798,7 +798,6 @@ export declare function declareLFoldTable<
   reducer: SqlMapper<{ oldState: S, oldRowData: OldRD }, { newState: S, newRowData: NewRD }>,
 }): Table;
 
-
 // ====== Executing SQL Statements ======
 const BULLDOZER_LOCK_ID = 7857391;  // random number to avoid conflicts with other applications
 export function toQueryableSqlQuery(query: SqlQuery): string {

From 006cf5ef5e65d44961e16be9d94445e09cd1f8ab Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 10:08:02 -0700
Subject: [PATCH 10/40] Add flat map interface

---
 apps/backend/src/lib/bulldozer/db/index.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index eaa4881c37..539844bff7 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -765,6 +765,16 @@ export function declareMapTable<
   };
 }
 
+export declare function declareFlatMapTable<
+  GK extends Json,
+  OldRD extends RowData,
+  NewRD extends RowData,
+>(options: {
+  tableId: TableId,
+  fromTable: Table,
+  mapper: SqlMapper,
+}): Table;
+
 export declare function declareFilterTable<
   GK extends Json,
   RD extends RowData,

From 4cdc0571b4f948c6c6008fe6fe7c70b3eac4b784 Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 10:23:27 -0700
Subject: [PATCH 11/40] FlatMap table

---
 .../src/lib/bulldozer/db/example-schema.ts    |  35 +-
 .../src/lib/bulldozer/db/index.perf.test.ts   |  84 ++++-
 .../src/lib/bulldozer/db/index.test.ts        | 338 ++++++++++++++++-
 apps/backend/src/lib/bulldozer/db/index.ts    | 349 +++++++++++++++++-
 4 files changed, 798 insertions(+), 8 deletions(-)

diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts
index f95b986aa7..ad139118f7 100644
--- a/apps/backend/src/lib/bulldozer/db/example-schema.ts
+++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts
@@ -1,4 +1,4 @@
-import { declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
+import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
 
 const mapper = (sql: string) => ({ type: "mapper" as const, sql });
 
@@ -55,6 +55,38 @@ export const exampleFungibleLedgerSchema = (() => {
     `),
   });
 
+  // Fan out each ledger entry into two directional legs for downstream views.
+  const accountEntryLegs = declareFlatMapTable({
+    tableId: "bulldozer-example-ledger-account-entry-legs",
+    fromTable: entriesByAccount,
+    mapper: mapper(`
+      jsonb_build_array(
+        jsonb_build_object(
+          'accountId', "rowData"->'accountId',
+          'asset', "rowData"->'asset',
+          'legType', 'entry',
+          'signedAmount',
+            CASE
+              WHEN "rowData"->>'side' = 'credit' THEN (("rowData"->>'amount')::numeric)
+              ELSE -(("rowData"->>'amount')::numeric)
+            END,
+          'txHash', "rowData"->'txHash'
+        ),
+        jsonb_build_object(
+          'accountId', "rowData"->'accountId',
+          'asset', "rowData"->'asset',
+          'legType', 'counterparty',
+          'signedAmount',
+            CASE
+              WHEN "rowData"->>'side' = 'credit' THEN -(("rowData"->>'amount')::numeric)
+              ELSE (("rowData"->>'amount')::numeric)
+            END,
+          'txHash', "rowData"->'txHash'
+        )
+      ) AS "rows"
+    `),
+  });
+
   // Build an account+asset partition from normalized entries.
   const accountAssetPartitions = declareGroupByTable({
     tableId: "bulldozer-example-ledger-account-asset-partitions",
@@ -89,6 +121,7 @@ export const exampleFungibleLedgerSchema = (() => {
     entriesByAccount,
     entriesByAsset,
     accountEntriesNormalized,
+    accountEntryLegs,
     accountAssetPartitions,
     assetEntriesNormalized,
   };
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index 9f3ec1619a..ec075a4a29 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -1,7 +1,7 @@
-import { getEnvBoolean } from "@stackframe/stack-shared/dist/utils/env";
+import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 type SqlExpression = { type: "expression", sql: string };
@@ -15,7 +15,12 @@ type WorkloadOperation =
 const TEST_DB_PREFIX = "stack_bulldozer_db_perf_test";
 const DEFAULT_WARMUP_OPS = 80;
 const DEFAULT_MEASURED_OPS = 500;
-const DEFAULT_LOAD_ROW_COUNT = getEnvBoolean("CI") ? 200_000 : 20_000;
+const IS_CI = (() => {
+  const env = Reflect.get(import.meta, "env");
+  const ci = Reflect.get(env, "CI");
+  return ci === true || ci === "true" || ci === "1";
+})();
+const DEFAULT_LOAD_ROW_COUNT = IS_CI ? 200_000 : 20_000;
 const LOAD_PREFILL_MAX_MS = 30_000;
 const LOAD_COUNT_QUERY_MAX_MS = 5_000;
 const LOAD_POINT_MUTATION_MAX_MS = 400;
@@ -24,6 +29,9 @@ const LOAD_SET_ROW_AVG_MAX_MS = 50;
 const LOAD_TABLE_DELETE_MAX_MS = 20_000;
 const LOAD_DERIVED_INIT_MAX_MS = 90_000;
 const LOAD_DERIVED_COUNT_QUERY_MAX_MS = 10_000;
+const LOAD_EXPANDING_INIT_MAX_MS = 120_000;
+const LOAD_EXPANDING_COUNT_QUERY_MAX_MS = 15_000;
+const LOAD_FILTERED_QUERY_MAX_MS = 4_000;
 
 function getTestDbUrls(): TestDb {
   const env = Reflect.get(import.meta, "env");
@@ -414,6 +422,24 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       fromTable: mappedTwice,
       groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` },
     });
+    const expandedByTeam = declareFlatMapTable({
+      tableId: "load-prefilled-users-expanded",
+      fromTable: groupedByTeam,
+      mapper: { type: "mapper", sql: `
+        jsonb_build_array(
+          jsonb_build_object(
+            'team', "rowData"->'team',
+            'kind', 'base',
+            'mappedValue', (("rowData"->>'value')::int + 10)
+          ),
+          jsonb_build_object(
+            'team', "rowData"->'team',
+            'kind', 'double',
+            'mappedValue', (("rowData"->>'value')::int * 2)
+          )
+        ) AS "rows"
+      ` },
+    });
 
     const groupInit = await measureMs("load init groupedByTeam", async () => {
       await runStatements(groupedByTeam.init());
@@ -431,6 +457,10 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       await runStatements(groupedByBucket.init());
     });
     expect(bucketInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS);
+    const expandInit = await measureMs("load init expandedByTeam", async () => {
+      await runStatements(expandedByTeam.init());
+    });
+    expect(expandInit.elapsedMs).toBeLessThan(LOAD_EXPANDING_INIT_MAX_MS);
 
     const groupedCountQuery = groupedByTeam.listRowsInGroup({
       start: "start",
@@ -450,17 +480,52 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     });
+    const expandedCountQuery = expandedByTeam.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    });
     const derivedCounts = await measureMs("load count derived tables", async () => {
       return await Promise.all([
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(groupedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`),
+        sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"`),
       ]);
     });
     expect(derivedCounts.elapsedMs).toBeLessThan(LOAD_DERIVED_COUNT_QUERY_MAX_MS);
     expect(Number(derivedCounts.result[0][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[1][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1);
+    expect(Number(derivedCounts.result[3][0].count)).toBe((loadRowCount - 1) * 2);
+
+    const expandedCountOnly = await measureMs("load count expanded table only", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"
+      `);
+    });
+    expect(expandedCountOnly.elapsedMs).toBeLessThan(LOAD_EXPANDING_COUNT_QUERY_MAX_MS);
+    expect(Number(expandedCountOnly.result[0].count)).toBe((loadRowCount - 1) * 2);
+
+    const filteredExpandedBetaBase = await measureMs("load filtered expanded query (team=beta, kind=base)", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (
+          ${toQueryableSqlQuery(expandedByTeam.listRowsInGroup({
+            groupKey: expr(`to_jsonb('beta'::text)`),
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }))}
+        ) AS "rows"
+        WHERE "rows"."rowdata"->>'kind' = 'base'
+      `);
+    });
+    expect(filteredExpandedBetaBase.elapsedMs).toBeLessThan(LOAD_FILTERED_QUERY_MAX_MS);
+    expect(Number(filteredExpandedBetaBase.result[0].count)).toBeGreaterThan(0);
 
     await runStatements(table.setRow(
       "seed-100000",
@@ -488,6 +553,17 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       bucket: "high",
       valueScaled: 2018,
     });
+    const expandedDeltaRows = await readRows(expandedByTeam.listRowsInGroup({
+      groupKey: expr(`to_jsonb('delta'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(expandedDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(String(a.rowIdentifier), String(b.rowIdentifier)))).toEqual([
+      { rowIdentifier: "seed-100000:1", rowData: { team: "delta", kind: "base", mappedValue: 1009 } },
+      { rowIdentifier: "seed-100000:2", rowData: { team: "delta", kind: "double", mappedValue: 1998 } },
+    ]);
 
     const bulkDelete = await measureMs("load full table delete", async () => {
       await runStatements(table.delete());
@@ -507,7 +583,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     `;
     expect(isInitializedRows[0].initialized).toBe(false);
 
-    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
+    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
   }, 180_000);
 });
 
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index b425468700..d7b5de5030 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -216,6 +216,49 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     });
     return { fromTable, groupedTable, mappedTable };
   }
+  function createFlatMappedTable() {
+    const { fromTable, groupedTable } = createGroupedTable();
+    const flatMappedTable = declareFlatMapTable({
+      tableId: "users-by-team-flat-mapped",
+      fromTable: groupedTable,
+      mapper: mapper(`
+        CASE
+          WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb
+          ELSE jsonb_build_array(
+            jsonb_build_object(
+              'team', "rowData"->'team',
+              'kind', 'base',
+              'mappedValue', (("rowData"->>'value')::int + 100)
+            ),
+            jsonb_build_object(
+              'team', "rowData"->'team',
+              'kind', 'double',
+              'mappedValue', (("rowData"->>'value')::int * 2)
+            )
+          )
+        END AS "rows"
+      `),
+    });
+    return { fromTable, groupedTable, flatMappedTable };
+  }
+  function createFlatMapMapGroupPipeline() {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    const mappedAfterFlatMap = declareMapTable({
+      tableId: "users-by-team-flat-map-then-map",
+      fromTable: flatMappedTable,
+      mapper: mapper(`
+        ("rowData"->'team') AS "team",
+        ("rowData"->'kind') AS "kind",
+        (("rowData"->>'mappedValue')::int + 1) AS "mappedValuePlusOne"
+      `),
+    });
+    const groupedByKind = declareGroupByTable({
+      tableId: "users-by-kind",
+      fromTable: mappedAfterFlatMap,
+      groupBy: mapper(`"rowData"->'kind' AS "groupKey"`),
+    });
+    return { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind };
+  }
   function createStackedMappedTables() {
     const { fromTable, groupedTable } = createGroupedTable();
     const mappedTableLevel1 = declareMapTable({
@@ -297,6 +340,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       `,
     ]);
   }
+  function registerFlatMapAuditTrigger(
+    table: ReturnType["flatMappedTable"],
+    event: string,
+  ) {
+    return table.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral(event))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+  }
 
   test("init/isInitialized/delete lifecycle", async () => {
     const table = declareStoredTable<{ value: number }>({ tableId: "users" });
@@ -1199,6 +1265,276 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(staleGroupPaths).toEqual([]);
   });
 
+  test("flatMapTable init backfills fan-out rows and skips empty expansions", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":-1}'::jsonb`)));
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+
+    const groups = await readRows(flatMappedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]);
+
+    const alphaRows = await readRows(flatMappedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([
+      { rowIdentifier: "u1:1", rowData: { team: "alpha", kind: "base", mappedValue: 101 } },
+      { rowIdentifier: "u1:2", rowData: { team: "alpha", kind: "double", mappedValue: 2 } },
+    ]);
+
+    const allRows = await readRows(flatMappedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
+      { groupKey: "alpha", rowIdentifier: "u1:1" },
+      { groupKey: "alpha", rowIdentifier: "u1:2" },
+      { groupKey: "beta", rowIdentifier: "u2:1" },
+      { groupKey: "beta", rowIdentifier: "u2:2" },
+    ]);
+  });
+
+  test("flatMapTable registerRowChangeTrigger emits per-expanded-row inserts, updates, moves, and removals", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+    registerFlatMapAuditTrigger(flatMappedTable, "flat_map_change");
+
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":-1}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("u1"));
+
+    const normalizedAuditRows = (await readMapTriggerAuditRows())
+      .map((row) => ({
+        groupKey: row.groupKey,
+        rowIdentifier: row.rowIdentifier,
+        oldRowData: row.oldRowData,
+        newRowData: row.newRowData,
+      }))
+      .sort((a, b) => stringCompare(
+        `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`,
+        `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`,
+      ));
+    expect(normalizedAuditRows).toEqual([
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: null,
+        newRowData: { team: "alpha", kind: "base", mappedValue: 101 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: { team: "alpha", kind: "base", mappedValue: 101 },
+        newRowData: { team: "alpha", kind: "base", mappedValue: 102 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: { team: "alpha", kind: "base", mappedValue: 102 },
+        newRowData: null,
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:2",
+        oldRowData: null,
+        newRowData: { team: "alpha", kind: "double", mappedValue: 2 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:2",
+        oldRowData: { team: "alpha", kind: "double", mappedValue: 2 },
+        newRowData: { team: "alpha", kind: "double", mappedValue: 4 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:2",
+        oldRowData: { team: "alpha", kind: "double", mappedValue: 4 },
+        newRowData: null,
+      },
+      {
+        groupKey: "beta",
+        rowIdentifier: "u1:1",
+        oldRowData: null,
+        newRowData: { team: "beta", kind: "base", mappedValue: 103 },
+      },
+      {
+        groupKey: "beta",
+        rowIdentifier: "u1:1",
+        oldRowData: { team: "beta", kind: "base", mappedValue: 103 },
+        newRowData: null,
+      },
+      {
+        groupKey: "beta",
+        rowIdentifier: "u1:2",
+        oldRowData: null,
+        newRowData: { team: "beta", kind: "double", mappedValue: 6 },
+      },
+      {
+        groupKey: "beta",
+        rowIdentifier: "u1:2",
+        oldRowData: { team: "beta", kind: "double", mappedValue: 6 },
+        newRowData: null,
+      },
+    ]);
+  });
+
+  test("flatMapTable stays no-op while uninitialized", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    registerFlatMapAuditTrigger(flatMappedTable, "flat_map_change");
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+
+    expect(await readBoolean(flatMappedTable.isInitialized())).toBe(false);
+    expect(await readMapTriggerAuditRows()).toEqual([]);
+    const groups = await readRows(flatMappedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups).toEqual([]);
+  });
+
+  test("flatMapTable delete cleans up and re-init backfills from source", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(flatMappedTable.delete());
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+
+    expect(await readBoolean(flatMappedTable.isInitialized())).toBe(false);
+    const groupsBeforeReinit = await readRows(flatMappedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groupsBeforeReinit).toEqual([]);
+
+    await runStatements(flatMappedTable.init());
+    const groupsAfterReinit = await readRows(flatMappedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]);
+  });
+
+  test("flatMapTable listRowsInGroup (all groups) handles 'rows' collisions in group key and source row identifier", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+
+    const allRows = await readRows(flatMappedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    const normalizedRows = allRows
+      .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata }))
+      .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`));
+
+    expect(normalizedRows).toEqual([
+      { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", kind: "base", mappedValue: 102 } },
+      { groupKey: "alpha", rowIdentifier: "rows:2", rowData: { team: "alpha", kind: "double", mappedValue: 4 } },
+      { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", kind: "base", mappedValue: 101 } },
+      { groupKey: "rows", rowIdentifier: "u1:2", rowData: { team: "rows", kind: "double", mappedValue: 2 } },
+    ]);
+  });
+
+  test("flatMapTable deletes stale group paths from storage", async () => {
+    const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":-1}'::jsonb`)));
+
+    const staleGroupPaths = await sql`
+      SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPath"[1:4] = ARRAY[
+        to_jsonb('table'::text),
+        to_jsonb('external:users-by-team-flat-mapped'::text),
+        to_jsonb('storage'::text),
+        to_jsonb('groups'::text)
+      ]::jsonb[]
+      AND cardinality("keyPath") > 4
+      ORDER BY "keyPath"
+    `;
+    expect(staleGroupPaths).toEqual([]);
+  });
+
+  test("flatMap -> map -> groupBy composition stays consistent across updates", async () => {
+    const { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind } = createFlatMapMapGroupPipeline();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(flatMappedTable.init());
+    await runStatements(mappedAfterFlatMap.init());
+    await runStatements(groupedByKind.init());
+
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":-1}'::jsonb`)));
+
+    const groups = await readRows(groupedByKind.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["base", "double"]);
+
+    const baseRows = await readRows(groupedByKind.listRowsInGroup({
+      groupKey: expr(`to_jsonb('base'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(baseRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "u2:1", rowData: { team: "beta", kind: "base", mappedValuePlusOne: 103 } },
+    ]);
+
+    const doubleRows = await readRows(groupedByKind.listRowsInGroup({
+      groupKey: expr(`to_jsonb('double'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(doubleRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "u2:2", rowData: { team: "beta", kind: "double", mappedValuePlusOne: 5 } },
+    ]);
+  });
+
   test("stacked map tables propagate updates across multiple mapping layers", async () => {
     const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables();
     await runStatements(fromTable.init());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index 539844bff7..1156919f59 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -765,7 +765,7 @@ export function declareMapTable<
   };
 }
 
-export declare function declareFlatMapTable<
+export function declareFlatMapTable<
   GK extends Json,
   OldRD extends RowData,
   NewRD extends RowData,
@@ -773,7 +773,352 @@ export declare function declareFlatMapTable<
   tableId: TableId,
   fromTable: Table,
   mapper: SqlMapper,
-}): Table;
+}): Table {
+  const triggers = new Map) => SqlStatement[]>();
+  const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]);
+  const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]);
+  const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]);
+  const isInitializedExpression = sqlExpression`
+    EXISTS (
+      SELECT 1 FROM "BulldozerStorageEngine"
+      WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
+    )
+  `;
+
+  options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
+    const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`;
+    const oldFlatRowsTableName = `old_flat_rows_${generateSecureRandomString()}`;
+    const newFlatRowsTableName = `new_flat_rows_${generateSecureRandomString()}`;
+    const flatMapChangesTableName = `flat_map_changes_${generateSecureRandomString()}`;
+    return [
+      sqlQuery`
+        SELECT
+          "changes"."groupKey" AS "groupKey",
+          "changes"."rowIdentifier" AS "sourceRowIdentifier",
+          ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+          ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
+          "oldMapped"."rows" AS "oldMappedRows",
+          "newMapped"."rows" AS "newMappedRows"
+        FROM ${fromChangesTable} AS "changes"
+        LEFT JOIN LATERAL (
+          SELECT "mapped"."rows" AS "rows"
+          FROM (
+            SELECT ${options.mapper}
+            FROM (
+              SELECT
+                "changes"."rowIdentifier" AS "rowIdentifier",
+                "changes"."oldRowData" AS "rowData"
+            ) AS "mapperInput"
+          ) AS "mapped"
+        ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
+        LEFT JOIN LATERAL (
+          SELECT "mapped"."rows" AS "rows"
+          FROM (
+            SELECT ${options.mapper}
+            FROM (
+              SELECT
+                "changes"."rowIdentifier" AS "rowIdentifier",
+                "changes"."newRowData" AS "rowData"
+            ) AS "mapperInput"
+          ) AS "mapped"
+        ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
+        WHERE ${isInitializedExpression}
+      `.toStatement(mappedChangesTableName),
+      sqlQuery`
+        SELECT
+          "changes"."groupKey" AS "groupKey",
+          ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+          "flatRow"."rowData" AS "rowData"
+        FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
+        CROSS JOIN LATERAL jsonb_array_elements(
+          CASE
+            WHEN "changes"."hasOldRow" THEN (
+              CASE
+                WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows"
+                ELSE '[]'::jsonb
+              END
+            )
+            ELSE '[]'::jsonb
+          END
+        ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+      `.toStatement(oldFlatRowsTableName),
+      sqlQuery`
+        SELECT
+          "changes"."groupKey" AS "groupKey",
+          ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+          "flatRow"."rowData" AS "rowData"
+        FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
+        CROSS JOIN LATERAL jsonb_array_elements(
+          CASE
+            WHEN "changes"."hasNewRow" THEN (
+              CASE
+                WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows"
+                ELSE '[]'::jsonb
+              END
+            )
+            ELSE '[]'::jsonb
+          END
+        ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+      `.toStatement(newFlatRowsTableName),
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          "insertRows"."keyPath",
+          "insertRows"."value"
+        FROM (
+          SELECT DISTINCT
+            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(newFlatRowsTableName)}
+          UNION
+          SELECT DISTINCT
+            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(newFlatRowsTableName)}
+        ) AS "insertRows"
+        ON CONFLICT ("keyPath") DO NOTHING
+      `,
+      sqlStatement`
+        DELETE FROM "BulldozerStorageEngine" AS "target"
+        USING ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "changes"
+        WHERE "target"."keyPath" = ${getGroupRowPath(
+          sqlExpression`"changes"."groupKey"`,
+          sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`,
+        )}::jsonb[]
+      `,
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          ${getGroupRowPath(
+            sqlExpression`"groupKey"`,
+            sqlExpression`to_jsonb("rowIdentifier"::text)`,
+          )}::jsonb[],
+          jsonb_build_object('rowData', "rowData")
+        FROM ${quoteSqlIdentifier(newFlatRowsTableName)}
+        ON CONFLICT ("keyPath") DO UPDATE
+        SET "value" = EXCLUDED."value"
+      `,
+      sqlStatement`
+        DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+        USING ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "changes"
+        WHERE "staleGroupPath"."keyPath" IN (
+          ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[],
+          ${getGroupKeyPath(sqlExpression`"changes"."groupKey"`)}::jsonb[]
+        )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM "BulldozerStorageEngine" AS "groupRow"
+            WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[]
+              AND NOT EXISTS (
+                SELECT 1
+                FROM ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "deletingRow"
+                WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+                  AND "groupRow"."keyPath" = ${getGroupRowPath(
+                    sqlExpression`"deletingRow"."groupKey"`,
+                    sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`,
+                  )}::jsonb[]
+              )
+          )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${quoteSqlIdentifier(newFlatRowsTableName)} AS "insertingRow"
+            WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+          )
+      `,
+      sqlQuery`
+        SELECT
+          COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
+          COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
+          'null'::jsonb AS "oldRowSortKey",
+          'null'::jsonb AS "newRowSortKey",
+          CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
+          CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
+        FROM ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "oldRows"
+        FULL OUTER JOIN ${quoteSqlIdentifier(newFlatRowsTableName)} AS "newRows"
+          ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
+          AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
+        WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
+      `.toStatement(flatMapChangesTableName),
+      ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(flatMapChangesTableName))),
+    ];
+  });
+
+  return {
+    tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "flatMap",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      mapperSql: options.mapper.sql,
+    },
+    compareGroupKeys: options.fromTable.compareGroupKeys,
+    compareSortKeys: (a, b) => sqlExpression` 0 `,
+    init: () => {
+      const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`;
+      const fromRowsTableName = `from_rows_${generateSecureRandomString()}`;
+      const mappedRowsTableName = `mapped_rows_${generateSecureRandomString()}`;
+      const flatRowsTableName = `flat_rows_${generateSecureRandomString()}`;
+
+      return [
+        sqlStatement`
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          VALUES
+          (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
+        `,
+        options.fromTable.listGroups({
+          start: "start",
+          end: "end",
+          startInclusive: true,
+          endInclusive: true,
+        }).toStatement(fromGroupsTableName),
+        sqlQuery`
+          SELECT
+            "groups"."groupkey" AS "groupKey",
+            "rows"."rowidentifier" AS "rowIdentifier",
+            "rows"."rowdata" AS "rowData"
+          FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups"
+          CROSS JOIN LATERAL (
+            ${options.fromTable.listRowsInGroup({
+              groupKey: sqlExpression`"groups"."groupkey"`,
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            })}
+          ) AS "rows"
+        `.toStatement(fromRowsTableName),
+        sqlQuery`
+          SELECT
+            "rows"."groupKey" AS "groupKey",
+            "rows"."rowIdentifier" AS "sourceRowIdentifier",
+            "mapped"."rows" AS "mappedRows"
+          FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows"
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."rows" AS "rows"
+            FROM (
+              SELECT ${options.mapper}
+              FROM (
+                SELECT
+                  "rows"."rowIdentifier" AS "rowIdentifier",
+                  "rows"."rowData" AS "rowData"
+              ) AS "mapperInput"
+            ) AS "mapped"
+          ) AS "mapped" ON true
+        `.toStatement(mappedRowsTableName),
+        sqlQuery`
+          SELECT
+            "rows"."groupKey" AS "groupKey",
+            ("rows"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            "flatRow"."rowData" AS "rowData"
+          FROM ${quoteSqlIdentifier(mappedRowsTableName)} AS "rows"
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN jsonb_typeof("rows"."mappedRows") = 'array' THEN "rows"."mappedRows"
+              ELSE '[]'::jsonb
+            END
+          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+        `.toStatement(flatRowsTableName),
+        sqlStatement`
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(flatRowsTableName)}
+            UNION
+            SELECT DISTINCT
+              ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(flatRowsTableName)}
+            UNION
+            SELECT
+              ${getGroupRowPath(
+                sqlExpression`"groupKey"`,
+                sqlExpression`to_jsonb("rowIdentifier"::text)`,
+              )}::jsonb[] AS "keyPath",
+              jsonb_build_object('rowData', "rowData") AS "value"
+            FROM ${quoteSqlIdentifier(flatRowsTableName)}
+          ) AS "insertRows"
+        `,
+      ];
+    },
+    delete: () => [sqlStatement`
+      WITH RECURSIVE "pathsToDelete" AS (
+        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
+        UNION ALL
+        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
+        FROM "BulldozerStorageEngine"
+        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
+      )
+      DELETE FROM "BulldozerStorageEngine"
+      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
+    `],
+    isInitialized: () => isInitializedExpression,
+    listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
+      SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey
+      FROM "BulldozerStorageEngine" AS "groupPath"
+      WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND EXISTS (
+          SELECT 1
+          FROM "BulldozerStorageEngine" AS "groupRowsPath"
+          INNER JOIN "BulldozerStorageEngine" AS "groupRow"
+            ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath"
+          WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath"
+            AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text)
+        )
+        AND ${
+          start === "start"
+            ? sqlExpression`1 = 1`
+            : startInclusive
+              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0`
+              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0`
+        }
+        AND ${
+          end === "end"
+            ? sqlExpression`1 = 1`
+            : endInclusive
+              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0`
+              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0`
+        }
+    `,
+    listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery`
+      SELECT
+        ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier,
+        'null'::jsonb AS rowSortKey,
+        "value"->'rowData' AS rowData
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[]
+        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
+    ` : sqlQuery`
+      SELECT
+        "groupRows"."keyPath"[cardinality("groupRows"."keyPath") - 1] AS groupKey,
+        ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier,
+        'null'::jsonb AS rowSortKey,
+        "rows"."value"->'rowData' AS rowData
+      FROM "BulldozerStorageEngine" AS "groupRows"
+      INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath"
+      WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND cardinality("groupRows"."keyPath") = cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]) + 2
+        AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text)
+        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
+    `,
+    registerRowChangeTrigger: (trigger) => {
+      const id = generateSecureRandomString();
+      triggers.set(id, trigger);
+      return { deregister: () => triggers.delete(id) };
+    },
+  };
+}
 
 export declare function declareFilterTable<
   GK extends Json,

From 49dc922a09dea1ab651b6e1b30c636ff3748b205 Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 10:26:52 -0700
Subject: [PATCH 12/40] Flat map fuzz tests

---
 .../src/lib/bulldozer/db/index.fuzz.test.ts   | 328 +++++++++++++++++-
 1 file changed, 327 insertions(+), 1 deletion(-)

diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index 9a7241637b..b0eff95b44 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -33,6 +33,8 @@ type QueryableTable = {
 type SourceRow = { team: string | null, value: number };
 type TeamMappedRow = { team: string | null, valuePlusTen: number };
 type TeamBucketRow = { team: string | null, valueScaled: number, bucket: string };
+type TeamFlatMappedRow = { team: string | null, kind: string, mappedValue: number };
+type TeamFlatMappedPlusRow = { team: string | null, kind: string, mappedValuePlusOne: number };
 type GroupedRows> = Map }>;
 
 function expr(sql: string): SqlExpression {
@@ -124,6 +126,26 @@ function regroupByField>(
   }
   return regrouped;
 }
+function flatMapGroups, NewRow extends Record>(
+  groups: GroupedRows,
+  mapperFn: (row: OldRow) => NewRow[],
+): GroupedRows {
+  const mapped: GroupedRows = new Map();
+  for (const [groupKey, group] of groups) {
+    const rows = new Map();
+    for (const [rowIdentifier, rowData] of group.rows) {
+      const expandedRows = mapperFn(rowData);
+      for (let i = 0; i < expandedRows.length; i++) {
+        const expandedRow = expandedRows[i] ?? (() => {
+          throw new Error("flatMapGroups mapper returned undefined row");
+        })();
+        rows.set(`${rowIdentifier}:${i + 1}`, expandedRow);
+      }
+    }
+    mapped.set(groupKey, { groupKey: group.groupKey, rows });
+  }
+  return mapped;
+}
 
 describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   const dbUrls = getTestDbUrls();
@@ -360,6 +382,155 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
     }
   });
 
+  test("fuzz: flatMap/map/group pipelines preserve invariants under random mutations and re-inits", async () => {
+    const identifiers = ["f1", "f2", "f3", "f4", "f:5", "f 6", "f/7", "f'8"] as const;
+    const teams = ["alpha", "beta", "gamma", null] as const;
+
+    for (const seed of [501, 502, 503]) {
+      const rng = createRng(seed);
+      const sourceRows = new Map();
+      let pipelineInitialized = true;
+
+      const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `flat-fuzz-users-${seed}` });
+      const groupedTable = declareGroupByTable({
+        tableId: `flat-fuzz-users-by-team-${seed}`,
+        fromTable,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const flatMapTable = declareFlatMapTable({
+        tableId: `flat-fuzz-users-expanded-${seed}`,
+        fromTable: groupedTable,
+        mapper: mapper(`
+          CASE
+            WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb
+            ELSE jsonb_build_array(
+              jsonb_build_object(
+                'team', "rowData"->'team',
+                'kind', 'base',
+                'mappedValue', (("rowData"->>'value')::int + 100)
+              ),
+              jsonb_build_object(
+                'team', "rowData"->'team',
+                'kind', 'double',
+                'mappedValue', (("rowData"->>'value')::int * 2)
+              )
+            )
+          END AS "rows"
+        `),
+      });
+      const mapAfterFlat = declareMapTable({
+        tableId: `flat-fuzz-users-expanded-plus-${seed}`,
+        fromTable: flatMapTable,
+        mapper: mapper(`
+          ("rowData"->'team') AS "team",
+          ("rowData"->'kind') AS "kind",
+          (("rowData"->>'mappedValue')::int + 1) AS "mappedValuePlusOne"
+        `),
+      });
+      const groupedByKind = declareGroupByTable({
+        tableId: `flat-fuzz-users-by-kind-${seed}`,
+        fromTable: mapAfterFlat,
+        groupBy: mapper(`"rowData"->'kind' AS "groupKey"`),
+      });
+
+      await runStatements(fromTable.init());
+      await runStatements(groupedTable.init());
+      await runStatements(flatMapTable.init());
+      await runStatements(mapAfterFlat.init());
+      await runStatements(groupedByKind.init());
+
+      for (let step = 0; step < 60; step++) {
+        const roll = rng();
+        if (roll < 0.6) {
+          const rowIdentifier = choose(rng, identifiers);
+          const rowData: SourceRow = {
+            team: choose(rng, teams),
+            value: Math.floor(rng() * 80) - 20,
+          };
+          sourceRows.set(rowIdentifier, rowData);
+          await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+        } else if (roll < 0.84) {
+          const rowIdentifier = choose(rng, identifiers);
+          sourceRows.delete(rowIdentifier);
+          await runStatements(fromTable.deleteRow(rowIdentifier));
+        } else if (roll < 0.92) {
+          if (pipelineInitialized) {
+            await runStatements(groupedByKind.delete());
+            await runStatements(mapAfterFlat.delete());
+            await runStatements(flatMapTable.delete());
+            pipelineInitialized = false;
+          }
+        } else {
+          if (!pipelineInitialized) {
+            await runStatements(flatMapTable.init());
+            await runStatements(mapAfterFlat.init());
+            await runStatements(groupedByKind.init());
+            pipelineInitialized = true;
+          }
+        }
+
+        const expectedGrouped = computeTeamGroups(sourceRows);
+        const expectedFlat = flatMapGroups(expectedGrouped, (row): TeamFlatMappedRow[] => {
+          if ((row.value as number) < 0) return [];
+          return [
+            {
+              team: row.team as string | null,
+              kind: "base",
+              mappedValue: (row.value as number) + 100,
+            },
+            {
+              team: row.team as string | null,
+              kind: "double",
+              mappedValue: (row.value as number) * 2,
+            },
+          ];
+        });
+        const expectedMapped = mapGroups(expectedFlat, (row): TeamFlatMappedPlusRow => ({
+          team: row.team as string | null,
+          kind: row.kind as string,
+          mappedValuePlusOne: (row.mappedValue as number) + 1,
+        }));
+        const expectedKind = regroupByField(expectedMapped, (row) => row.kind as string);
+
+        await assertTableMatches(groupedTable, expectedGrouped);
+        if (pipelineInitialized) {
+          expect(await readBoolean(flatMapTable.isInitialized())).toBe(true);
+          expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(true);
+          expect(await readBoolean(groupedByKind.isInitialized())).toBe(true);
+          await assertTableMatches(flatMapTable, expectedFlat);
+          await assertTableMatches(mapAfterFlat, expectedMapped);
+          await assertTableMatches(groupedByKind, expectedKind);
+        } else {
+          expect(await readBoolean(flatMapTable.isInitialized())).toBe(false);
+          expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(false);
+          expect(await readBoolean(groupedByKind.isInitialized())).toBe(false);
+
+          const flatGroups = await readRows(flatMapTable.listGroups({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }));
+          const mappedGroups = await readRows(mapAfterFlat.listGroups({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }));
+          const kindGroups = await readRows(groupedByKind.listGroups({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }));
+          expect(flatGroups).toEqual([]);
+          expect(mappedGroups).toEqual([]);
+          expect(kindGroups).toEqual([]);
+        }
+      }
+    }
+  });
+
   test("fuzz: parallel map tables remain isolated with independent re-inits", async () => {
     const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const;
     const teams = ["alpha", "beta", null] as const;
@@ -476,4 +647,159 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       }
     }
   });
+
+  test("fuzz: parallel flatMap tables remain isolated with independent re-inits", async () => {
+    const identifiers = ["pf1", "pf2", "pf3", "pf 4", "pf:5"] as const;
+    const teams = ["alpha", "beta", null] as const;
+
+    for (const seed of [601, 602]) {
+      const rng = createRng(seed);
+      const sourceRows = new Map();
+      let flatAInitialized = true;
+      let flatBInitialized = true;
+
+      const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `parallel-flat-users-${seed}` });
+      const groupedTable = declareGroupByTable({
+        tableId: `parallel-flat-users-by-team-${seed}`,
+        fromTable,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const flatMapA = declareFlatMapTable({
+        tableId: `parallel-flat-users-a-${seed}`,
+        fromTable: groupedTable,
+        mapper: mapper(`
+          CASE
+            WHEN (("rowData"->>'value')::int) % 2 = 0 THEN jsonb_build_array(
+              jsonb_build_object(
+                'team', "rowData"->'team',
+                'lane', 'even',
+                'metricA', (("rowData"->>'value')::int + 1000)
+              )
+            )
+            ELSE '[]'::jsonb
+          END AS "rows"
+        `),
+      });
+      const flatMapB = declareFlatMapTable({
+        tableId: `parallel-flat-users-b-${seed}`,
+        fromTable: groupedTable,
+        mapper: mapper(`
+          CASE
+            WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb
+            ELSE jsonb_build_array(
+              jsonb_build_object(
+                'team', "rowData"->'team',
+                'lane', 'base',
+                'metricB', (("rowData"->>'value')::int)
+              ),
+              jsonb_build_object(
+                'team', "rowData"->'team',
+                'lane', 'triple',
+                'metricB', (("rowData"->>'value')::int * 3)
+              )
+            )
+          END AS "rows"
+        `),
+      });
+
+      await runStatements(fromTable.init());
+      await runStatements(groupedTable.init());
+      await runStatements(flatMapA.init());
+      await runStatements(flatMapB.init());
+
+      for (let step = 0; step < 55; step++) {
+        const roll = rng();
+        if (roll < 0.6) {
+          const rowIdentifier = choose(rng, identifiers);
+          const rowData: SourceRow = {
+            team: choose(rng, teams),
+            value: Math.floor(rng() * 50) - 10,
+          };
+          sourceRows.set(rowIdentifier, rowData);
+          await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+        } else if (roll < 0.82) {
+          const rowIdentifier = choose(rng, identifiers);
+          sourceRows.delete(rowIdentifier);
+          await runStatements(fromTable.deleteRow(rowIdentifier));
+        } else if (roll < 0.9) {
+          if (flatAInitialized) {
+            await runStatements(flatMapA.delete());
+            flatAInitialized = false;
+          }
+        } else if (roll < 0.94) {
+          if (!flatAInitialized) {
+            await runStatements(flatMapA.init());
+            flatAInitialized = true;
+          }
+        } else if (roll < 0.98) {
+          if (flatBInitialized) {
+            await runStatements(flatMapB.delete());
+            flatBInitialized = false;
+          }
+        } else {
+          if (!flatBInitialized) {
+            await runStatements(flatMapB.init());
+            flatBInitialized = true;
+          }
+        }
+
+        const expectedGrouped = computeTeamGroups(sourceRows);
+        await assertTableMatches(groupedTable, expectedGrouped);
+
+        const expectedFlatA = flatMapGroups(expectedGrouped, (row) => {
+          const value = row.value as number;
+          if (value % 2 !== 0) return [];
+          return [{
+            team: row.team as string | null,
+            lane: "even",
+            metricA: value + 1000,
+          }];
+        });
+        const expectedFlatB = flatMapGroups(expectedGrouped, (row) => {
+          const value = row.value as number;
+          if (value < 0) return [];
+          return [
+            {
+              team: row.team as string | null,
+              lane: "base",
+              metricB: value,
+            },
+            {
+              team: row.team as string | null,
+              lane: "triple",
+              metricB: value * 3,
+            },
+          ];
+        });
+
+        if (flatAInitialized) {
+          expect(await readBoolean(flatMapA.isInitialized())).toBe(true);
+          await assertTableMatches(flatMapA, expectedFlatA);
+        } else {
+          expect(await readBoolean(flatMapA.isInitialized())).toBe(false);
+          const groups = await readRows(flatMapA.listGroups({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }));
+          expect(groups).toEqual([]);
+        }
+
+        if (flatBInitialized) {
+          expect(await readBoolean(flatMapB.isInitialized())).toBe(true);
+          await assertTableMatches(flatMapB, expectedFlatB);
+        } else {
+          expect(await readBoolean(flatMapB.isInitialized())).toBe(false);
+          const groups = await readRows(flatMapB.listGroups({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }));
+          expect(groups).toEqual([]);
+        }
+      }
+    }
+  });
 });

From 3eccc22f80ab11ba4ba2e671921bc4e888f86a46 Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 11:22:11 -0700
Subject: [PATCH 13/40] Build MapTable from FlatMapTable

---
 .../src/lib/bulldozer/db/index.fuzz.test.ts   | 170 ++++----
 .../src/lib/bulldozer/db/index.perf.test.ts   |   2 +-
 .../src/lib/bulldozer/db/index.test.ts        | 222 ++++++++--
 apps/backend/src/lib/bulldozer/db/index.ts    | 403 ++++--------------
 4 files changed, 366 insertions(+), 431 deletions(-)

diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index b0eff95b44..27322afc77 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -99,7 +99,7 @@ function mapGroups, NewRow extends Record
   for (const [groupKey, group] of groups) {
     mapped.set(groupKey, {
       groupKey: group.groupKey,
-      rows: new Map([...group.rows.entries()].map(([rowIdentifier, rowData]) => [rowIdentifier, mapperFn(rowData)])),
+      rows: new Map([...group.rows.entries()].map(([rowIdentifier, rowData]) => [`${rowIdentifier}:1`, mapperFn(rowData)])),
     });
   }
   return mapped;
@@ -284,7 +284,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
     const identifiers = ["u1", "u2", "u3", "u4", "u:5", "u 6", "u/7", "u'8"] as const;
     const teams = ["alpha", "beta", "gamma", null] as const;
 
-    for (const seed of [101, 202, 303]) {
+    for (const seed of [101]) {
       const rng = createRng(seed);
       const sourceRows = new Map();
 
@@ -328,7 +328,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       await runStatements(mapTable2.init());
       await runStatements(groupedByBucket.init());
 
-      for (let step = 0; step < 60; step++) {
+      for (let step = 0; step < 24; step++) {
         const roll = rng();
         if (roll < 0.62) {
           const rowIdentifier = choose(rng, identifiers);
@@ -359,34 +359,36 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
           }
         }
 
-        const expectedGrouped = computeTeamGroups(sourceRows);
-        const expectedMap1 = mapGroups(expectedGrouped, (row): TeamMappedRow => ({
-          team: (row.team as string | null),
-          valuePlusTen: (row.value as number) + 10,
-        }));
-        const expectedMap2 = mapGroups(expectedMap1, (row): TeamBucketRow => {
-          const valueScaled = (row.valuePlusTen as number) * 2;
-          return {
+        if (step % 3 === 0 || step === 23) {
+          const expectedGrouped = computeTeamGroups(sourceRows);
+          const expectedMap1 = mapGroups(expectedGrouped, (row): TeamMappedRow => ({
             team: (row.team as string | null),
-            valueScaled,
-            bucket: valueScaled >= 30 ? "high" : "low",
-          };
-        });
-        const expectedBucket = regroupByField(expectedMap2, (row) => row.bucket as string);
-
-        await assertTableMatches(groupedTable, expectedGrouped);
-        await assertTableMatches(mapTable1, expectedMap1);
-        await assertTableMatches(mapTable2, expectedMap2);
-        await assertTableMatches(groupedByBucket, expectedBucket);
+            valuePlusTen: (row.value as number) + 10,
+          }));
+          const expectedMap2 = mapGroups(expectedMap1, (row): TeamBucketRow => {
+            const valueScaled = (row.valuePlusTen as number) * 2;
+            return {
+              team: (row.team as string | null),
+              valueScaled,
+              bucket: valueScaled >= 30 ? "high" : "low",
+            };
+          });
+          const expectedBucket = regroupByField(expectedMap2, (row) => row.bucket as string);
+
+          await assertTableMatches(groupedTable, expectedGrouped);
+          await assertTableMatches(mapTable1, expectedMap1);
+          await assertTableMatches(mapTable2, expectedMap2);
+          await assertTableMatches(groupedByBucket, expectedBucket);
+        }
       }
     }
-  });
+  }, 120_000);
 
   test("fuzz: flatMap/map/group pipelines preserve invariants under random mutations and re-inits", async () => {
     const identifiers = ["f1", "f2", "f3", "f4", "f:5", "f 6", "f/7", "f'8"] as const;
     const teams = ["alpha", "beta", "gamma", null] as const;
 
-    for (const seed of [501, 502, 503]) {
+    for (const seed of [501]) {
       const rng = createRng(seed);
       const sourceRows = new Map();
       let pipelineInitialized = true;
@@ -439,7 +441,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       await runStatements(mapAfterFlat.init());
       await runStatements(groupedByKind.init());
 
-      for (let step = 0; step < 60; step++) {
+      for (let step = 0; step < 24; step++) {
         const roll = rng();
         if (roll < 0.6) {
           const rowIdentifier = choose(rng, identifiers);
@@ -469,67 +471,69 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
           }
         }
 
-        const expectedGrouped = computeTeamGroups(sourceRows);
-        const expectedFlat = flatMapGroups(expectedGrouped, (row): TeamFlatMappedRow[] => {
-          if ((row.value as number) < 0) return [];
-          return [
-            {
-              team: row.team as string | null,
-              kind: "base",
-              mappedValue: (row.value as number) + 100,
-            },
-            {
-              team: row.team as string | null,
-              kind: "double",
-              mappedValue: (row.value as number) * 2,
-            },
-          ];
-        });
-        const expectedMapped = mapGroups(expectedFlat, (row): TeamFlatMappedPlusRow => ({
-          team: row.team as string | null,
-          kind: row.kind as string,
-          mappedValuePlusOne: (row.mappedValue as number) + 1,
-        }));
-        const expectedKind = regroupByField(expectedMapped, (row) => row.kind as string);
-
-        await assertTableMatches(groupedTable, expectedGrouped);
-        if (pipelineInitialized) {
-          expect(await readBoolean(flatMapTable.isInitialized())).toBe(true);
-          expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(true);
-          expect(await readBoolean(groupedByKind.isInitialized())).toBe(true);
-          await assertTableMatches(flatMapTable, expectedFlat);
-          await assertTableMatches(mapAfterFlat, expectedMapped);
-          await assertTableMatches(groupedByKind, expectedKind);
-        } else {
-          expect(await readBoolean(flatMapTable.isInitialized())).toBe(false);
-          expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(false);
-          expect(await readBoolean(groupedByKind.isInitialized())).toBe(false);
-
-          const flatGroups = await readRows(flatMapTable.listGroups({
-            start: "start",
-            end: "end",
-            startInclusive: true,
-            endInclusive: true,
-          }));
-          const mappedGroups = await readRows(mapAfterFlat.listGroups({
-            start: "start",
-            end: "end",
-            startInclusive: true,
-            endInclusive: true,
-          }));
-          const kindGroups = await readRows(groupedByKind.listGroups({
-            start: "start",
-            end: "end",
-            startInclusive: true,
-            endInclusive: true,
+        if (step % 3 === 0 || step === 23) {
+          const expectedGrouped = computeTeamGroups(sourceRows);
+          const expectedFlat = flatMapGroups(expectedGrouped, (row): TeamFlatMappedRow[] => {
+            if ((row.value as number) < 0) return [];
+            return [
+              {
+                team: row.team as string | null,
+                kind: "base",
+                mappedValue: (row.value as number) + 100,
+              },
+              {
+                team: row.team as string | null,
+                kind: "double",
+                mappedValue: (row.value as number) * 2,
+              },
+            ];
+          });
+          const expectedMapped = mapGroups(expectedFlat, (row): TeamFlatMappedPlusRow => ({
+            team: row.team as string | null,
+            kind: row.kind as string,
+            mappedValuePlusOne: (row.mappedValue as number) + 1,
           }));
-          expect(flatGroups).toEqual([]);
-          expect(mappedGroups).toEqual([]);
-          expect(kindGroups).toEqual([]);
+          const expectedKind = regroupByField(expectedMapped, (row) => row.kind as string);
+
+          await assertTableMatches(groupedTable, expectedGrouped);
+          if (pipelineInitialized) {
+            expect(await readBoolean(flatMapTable.isInitialized())).toBe(true);
+            expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(true);
+            expect(await readBoolean(groupedByKind.isInitialized())).toBe(true);
+            await assertTableMatches(flatMapTable, expectedFlat);
+            await assertTableMatches(mapAfterFlat, expectedMapped);
+            await assertTableMatches(groupedByKind, expectedKind);
+          } else {
+            expect(await readBoolean(flatMapTable.isInitialized())).toBe(false);
+            expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(false);
+            expect(await readBoolean(groupedByKind.isInitialized())).toBe(false);
+
+            const flatGroups = await readRows(flatMapTable.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }));
+            const mappedGroups = await readRows(mapAfterFlat.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }));
+            const kindGroups = await readRows(groupedByKind.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }));
+            expect(flatGroups).toEqual([]);
+            expect(mappedGroups).toEqual([]);
+            expect(kindGroups).toEqual([]);
+          }
         }
       }
     }
-  });
+  }, 120_000);
 
   test("fuzz: parallel map tables remain isolated with independent re-inits", async () => {
     const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const;
@@ -646,7 +650,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
         }
       }
     }
-  });
+  }, 120_000);
 
   test("fuzz: parallel flatMap tables remain isolated with independent re-inits", async () => {
     const identifiers = ["pf1", "pf2", "pf3", "pf 4", "pf:5"] as const;
@@ -801,5 +805,5 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
         }
       }
     }
-  });
+  }, 120_000);
 });
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index ec075a4a29..9cb4e8ff7f 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -546,7 +546,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     }));
-    const highBucketRow = highBucketRows.find((row) => row.rowidentifier === "seed-100000");
+    const highBucketRow = highBucketRows.find((row) => row.rowidentifier === "seed-100000:1:1");
     expect(highBucketRow).toBeDefined();
     expect(highBucketRow?.rowdata).toEqual({
       team: "delta",
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index d7b5de5030..8c898cd400 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -1055,8 +1055,8 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([
-      { rowIdentifier: "u1", rowData: { team: "alpha", mappedValue: 101 } },
-      { rowIdentifier: "u3", rowData: { team: "alpha", mappedValue: 103 } },
+      { rowIdentifier: "u1:1", rowData: { team: "alpha", mappedValue: 101 } },
+      { rowIdentifier: "u3:1", rowData: { team: "alpha", mappedValue: 103 } },
     ]);
 
     const allRows = await readRows(mappedTable.listRowsInGroup({
@@ -1066,9 +1066,9 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
-      { groupKey: "alpha", rowIdentifier: "u1" },
-      { groupKey: "alpha", rowIdentifier: "u3" },
-      { groupKey: "beta", rowIdentifier: "u2" },
+      { groupKey: "alpha", rowIdentifier: "u1:1" },
+      { groupKey: "alpha", rowIdentifier: "u3:1" },
+      { groupKey: "beta", rowIdentifier: "u2:1" },
     ]);
   });
 
@@ -1088,41 +1088,71 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       {
         event: "map_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: null,
         newRowData: { team: "alpha", mappedValue: 101 },
       },
       {
         event: "map_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: { team: "alpha", mappedValue: 101 },
         newRowData: { team: "alpha", mappedValue: 102 },
       },
       {
         event: "map_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: { team: "alpha", mappedValue: 102 },
         newRowData: null,
       },
       {
         event: "map_change",
         groupKey: "beta",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: null,
         newRowData: { team: "beta", mappedValue: 103 },
       },
       {
         event: "map_change",
         groupKey: "beta",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: { team: "beta", mappedValue: 103 },
         newRowData: null,
       },
     ]);
   });
 
+  test("mapTable uses flatMap-style rowIdentifier and skips unchanged updates", async () => {
+    const { fromTable, groupedTable, mappedTable } = createMappedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(mappedTable.init());
+    registerMapAuditTrigger(mappedTable, "map_change");
+
+    await runStatements(fromTable.setRow("user:1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("user:1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+
+    expect(await readMapTriggerAuditRows()).toEqual([
+      {
+        event: "map_change",
+        groupKey: "alpha",
+        rowIdentifier: "user:1:1",
+        oldRowData: null,
+        newRowData: { team: "alpha", mappedValue: 101 },
+      },
+    ]);
+
+    const alphaRows = await readRows(mappedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["user:1:1"]);
+  });
+
   test("mapTable deregistered trigger no longer runs", async () => {
     const { fromTable, groupedTable, mappedTable } = createMappedTable();
     await runStatements(fromTable.init());
@@ -1138,7 +1168,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       {
         event: "map_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1",
         oldRowData: null,
         newRowData: { team: "alpha", mappedValue: 101 },
       },
@@ -1236,8 +1266,8 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`));
 
     expect(normalizedRows).toEqual([
-      { groupKey: "alpha", rowIdentifier: "rows", rowData: { team: "alpha", mappedValue: 102 } },
-      { groupKey: "rows", rowIdentifier: "u1", rowData: { team: "rows", mappedValue: 101 } },
+      { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", mappedValue: 102 } },
+      { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", mappedValue: 101 } },
     ]);
   });
 
@@ -1265,6 +1295,128 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(staleGroupPaths).toEqual([]);
   });
 
+  test("mapTable matches equivalent single-row flatMap for rows, groups, and trigger payloads", async () => {
+    const { fromTable, groupedTable, mappedTable } = createMappedTable();
+    const equivalentFlatMapTable = declareFlatMapTable({
+      tableId: "users-by-team-mapped-equivalent-flatmap",
+      fromTable: groupedTable,
+      mapper: mapper(`
+        jsonb_build_array(
+          COALESCE(
+            (
+              SELECT to_jsonb("mapped")
+              FROM (
+                SELECT
+                  ("rowData"->'team') AS "team",
+                  (("rowData"->>'value')::int + 100) AS "mappedValue"
+              ) AS "mapped"
+            ),
+            'null'::jsonb
+          )
+        ) AS "rows"
+      `),
+    });
+
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(mappedTable.init());
+    await runStatements(equivalentFlatMapTable.init());
+
+    mappedTable.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral("map"))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+    equivalentFlatMapTable.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral("flat"))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("u1"));
+
+    const mapGroups = await readRows(mappedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    const flatGroups = await readRows(equivalentFlatMapTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(mapGroups).toEqual(flatGroups);
+
+    const normalizeRows = (rows: Iterable>) => [...rows]
+      .map((row) => ({
+        groupKey: (Reflect.get(row, "groupkey") as string | null),
+        rowIdentifier: String(Reflect.get(row, "rowidentifier")),
+        rowData: Reflect.get(row, "rowdata"),
+      }))
+      .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`));
+    const mapRows = normalizeRows(await readRows(mappedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    })));
+    const flatRows = normalizeRows(await readRows(equivalentFlatMapTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    })));
+    expect(mapRows).toEqual(flatRows);
+
+    const normalizeAuditRows = (rows: Iterable>) => [...rows]
+      .map((row) => ({
+        groupKey: (Reflect.get(row, "groupKey") as string | null),
+        rowIdentifier: String(Reflect.get(row, "rowIdentifier")),
+        oldRowData: Reflect.get(row, "oldRowData"),
+        newRowData: Reflect.get(row, "newRowData"),
+      }))
+      .sort((a, b) => stringCompare(
+        `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`,
+        `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`,
+      ));
+    const allAuditRows = await readMapTriggerAuditRows();
+    const mapAudit = normalizeAuditRows(allAuditRows.filter((row) => row.event === "map"));
+    const flatAudit = normalizeAuditRows(allAuditRows.filter((row) => row.event === "flat"));
+    expect(mapAudit).toEqual(flatAudit);
+  });
+
   test("flatMapTable init backfills fan-out rows and skips empty expansions", async () => {
     const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
     await runStatements(fromTable.init());
@@ -1520,7 +1672,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(baseRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
-      { rowIdentifier: "u2:1", rowData: { team: "beta", kind: "base", mappedValuePlusOne: 103 } },
+      { rowIdentifier: "u2:1:1", rowData: { team: "beta", kind: "base", mappedValuePlusOne: 103 } },
     ]);
 
     const doubleRows = await readRows(groupedByKind.listRowsInGroup({
@@ -1531,7 +1683,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(doubleRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
-      { rowIdentifier: "u2:2", rowData: { team: "beta", kind: "double", mappedValuePlusOne: 5 } },
+      { rowIdentifier: "u2:2:1", rowData: { team: "beta", kind: "double", mappedValuePlusOne: 5 } },
     ]);
   });
 
@@ -1563,8 +1715,8 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([
-      { rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } },
-      { rowIdentifier: "u2", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } },
+      { rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } },
+      { rowIdentifier: "u2:1:1", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } },
     ]);
 
     await runStatements(fromTable.deleteRow("u1"));
@@ -1576,7 +1728,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(alphaRowsAfterDelete.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
-      { rowIdentifier: "u2", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } },
+      { rowIdentifier: "u2:1:1", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } },
     ]);
   });
 
@@ -1598,7 +1750,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(nullGroupRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
-      { rowIdentifier: specialIdentifier, rowData: { team: null, valueScaled: 26, bucket: "low" } },
+      { rowIdentifier: `${specialIdentifier}:1:1`, rowData: { team: null, valueScaled: 26, bucket: "low" } },
     ]);
 
     await runStatements(fromTable.setRow(specialIdentifier, expr(`'{"team":"alpha","value":3}'::jsonb`)));
@@ -1630,9 +1782,9 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(allRowsAfterInit.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
-      { groupKey: "alpha", rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 22, bucket: "low" } },
-      { groupKey: "alpha", rowIdentifier: "u3", rowData: { team: "alpha", valueScaled: 26, bucket: "low" } },
-      { groupKey: "beta", rowIdentifier: "u2", rowData: { team: "beta", valueScaled: 24, bucket: "low" } },
+      { groupKey: "alpha", rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 22, bucket: "low" } },
+      { groupKey: "alpha", rowIdentifier: "u3:1:1", rowData: { team: "alpha", valueScaled: 26, bucket: "low" } },
+      { groupKey: "beta", rowIdentifier: "u2:1:1", rowData: { team: "beta", valueScaled: 24, bucket: "low" } },
     ]);
 
     await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`)));
@@ -1644,7 +1796,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(betaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
-      { rowIdentifier: "u2", rowData: { team: "beta", valueScaled: 60, bucket: "high" } },
+      { rowIdentifier: "u2:1:1", rowData: { team: "beta", valueScaled: 60, bucket: "high" } },
     ]);
   });
 
@@ -1675,7 +1827,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     }));
-    expect(lowRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1", "u3"]);
+    expect(lowRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1:1:1", "u3:1:1"]);
 
     await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":30}'::jsonb`)));
     await runStatements(fromTable.deleteRow("u3"));
@@ -1695,7 +1847,7 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     }));
-    expect(highRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1", "u2"]);
+    expect(highRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1:1:1", "u2:1:1"]);
   });
 
   test("composed trigger fanout works for stacked map and downstream groupBy tables", async () => {
@@ -1751,21 +1903,21 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       {
         event: "map_level_2_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: null,
         newRowData: { team: "alpha", valueScaled: 22, bucket: "low" },
       },
       {
         event: "map_level_2_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" },
         newRowData: { team: "alpha", valueScaled: 80, bucket: "high" },
       },
       {
         event: "map_level_2_change",
         groupKey: "alpha",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" },
         newRowData: null,
       },
@@ -1774,28 +1926,28 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       {
         event: "bucket_group_change",
         groupKey: "low",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: null,
         newRowData: { team: "alpha", valueScaled: 22, bucket: "low" },
       },
       {
         event: "bucket_group_change",
         groupKey: "low",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" },
         newRowData: null,
       },
       {
         event: "bucket_group_change",
         groupKey: "high",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: null,
         newRowData: { team: "alpha", valueScaled: 80, bucket: "high" },
       },
       {
         event: "bucket_group_change",
         groupKey: "high",
-        rowIdentifier: "u1",
+        rowIdentifier: "u1:1:1",
         oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" },
         newRowData: null,
       },
@@ -1837,9 +1989,9 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       endInclusive: true,
     }));
     expect(allBucketRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
-      { groupKey: "high", rowIdentifier: "u1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } },
-      { groupKey: "low", rowIdentifier: "u3", rowData: { team: "gamma", valueScaled: 24, bucket: "low" } },
-      { groupKey: "low", rowIdentifier: "u4", rowData: { team: "delta", valueScaled: 20, bucket: "low" } },
+      { groupKey: "high", rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } },
+      { groupKey: "low", rowIdentifier: "u3:1:1", rowData: { team: "gamma", valueScaled: 24, bucket: "low" } },
+      { groupKey: "low", rowIdentifier: "u4:1:1", rowData: { team: "delta", valueScaled: 20, bucket: "low" } },
     ]);
   });
 
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index 1156919f59..e7261fa659 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -1,5 +1,6 @@
 import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto";
 import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
+import { pick } from "@stackframe/stack-shared/dist/utils/objects";
 import { deindent, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 
 export type Table = {
@@ -456,315 +457,6 @@ export function declareGroupByTable<
   };
 }
 
-export function declareMapTable<
-  GK extends Json,
-  OldRD extends RowData,
-  NewRD extends RowData,
->(options: {
-  tableId: TableId,
-  fromTable: Table,
-  mapper: SqlMapper,
-}): Table {
-  const triggers = new Map) => SqlStatement[]>();
-  const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]);
-  const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]);
-  const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]);
-  const isInitializedExpression = sqlExpression`
-    EXISTS (
-      SELECT 1 FROM "BulldozerStorageEngine"
-      WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
-    )
-  `;
-
-  options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
-    const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`;
-    const mapChangesTableName = `map_changes_${generateSecureRandomString()}`;
-    return [
-      sqlQuery`
-        SELECT
-          "changes"."groupKey" AS "groupKey",
-          "changes"."rowIdentifier" AS "rowIdentifier",
-          "changes"."oldRowData" AS "oldRowData",
-          "changes"."newRowData" AS "newRowData",
-          ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
-          ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
-          "oldMapped"."rowData" AS "oldMappedRowData",
-          "newMapped"."rowData" AS "newMappedRowData"
-        FROM ${fromChangesTable} AS "changes"
-        LEFT JOIN LATERAL (
-          SELECT to_jsonb("mapped") AS "rowData"
-          FROM (
-            SELECT ${options.mapper}
-            FROM (
-              SELECT
-                "changes"."rowIdentifier" AS "rowIdentifier",
-                "changes"."oldRowData" AS "rowData"
-            ) AS "mapperInput"
-          ) AS "mapped"
-        ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
-        LEFT JOIN LATERAL (
-          SELECT to_jsonb("mapped") AS "rowData"
-          FROM (
-            SELECT ${options.mapper}
-            FROM (
-              SELECT
-                "changes"."rowIdentifier" AS "rowIdentifier",
-                "changes"."newRowData" AS "rowData"
-            ) AS "mapperInput"
-          ) AS "mapped"
-        ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
-        WHERE ${isInitializedExpression}
-      `.toStatement(mappedChangesTableName),
-      sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-        SELECT
-          gen_random_uuid(),
-          "insertRows"."keyPath",
-          "insertRows"."value"
-        FROM (
-          SELECT DISTINCT
-            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
-            'null'::jsonb AS "value"
-          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-          WHERE "hasNewRow"
-          UNION
-          SELECT DISTINCT
-            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
-            'null'::jsonb AS "value"
-          FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-          WHERE "hasNewRow"
-        ) AS "insertRows"
-        ON CONFLICT ("keyPath") DO NOTHING
-      `,
-      sqlStatement`
-        DELETE FROM "BulldozerStorageEngine" AS "target"
-        USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
-        WHERE "changes"."hasOldRow"
-          AND "target"."keyPath" = ${getGroupRowPath(
-            sqlExpression`"changes"."groupKey"`,
-            sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`,
-          )}::jsonb[]
-      `,
-      sqlStatement`
-        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-        SELECT
-          gen_random_uuid(),
-          ${getGroupRowPath(
-            sqlExpression`"groupKey"`,
-            sqlExpression`to_jsonb("rowIdentifier"::text)`,
-          )}::jsonb[],
-          jsonb_build_object('rowData', "newMappedRowData")
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasNewRow"
-        ON CONFLICT ("keyPath") DO UPDATE
-        SET "value" = EXCLUDED."value"
-      `,
-      sqlStatement`
-        DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
-        USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
-        WHERE "changes"."hasOldRow"
-          AND "staleGroupPath"."keyPath" IN (
-            ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[],
-            ${getGroupKeyPath(sqlExpression`"changes"."groupKey"`)}::jsonb[]
-          )
-          AND NOT EXISTS (
-            SELECT 1
-            FROM "BulldozerStorageEngine" AS "groupRow"
-            WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[]
-              AND NOT EXISTS (
-                SELECT 1
-                FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "deletingRow"
-                WHERE "deletingRow"."hasOldRow"
-                  AND "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-                  AND "groupRow"."keyPath" = ${getGroupRowPath(
-                    sqlExpression`"deletingRow"."groupKey"`,
-                    sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`,
-                  )}::jsonb[]
-              )
-          )
-          AND NOT EXISTS (
-            SELECT 1
-            FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "insertingRow"
-            WHERE "insertingRow"."hasNewRow"
-              AND "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-          )
-      `,
-      sqlQuery`
-        SELECT
-          "groupKey" AS "groupKey",
-          "rowIdentifier" AS "rowIdentifier",
-          'null'::jsonb AS "oldRowSortKey",
-          'null'::jsonb AS "newRowSortKey",
-          CASE WHEN "hasOldRow" THEN "oldMappedRowData" ELSE 'null'::jsonb END AS "oldRowData",
-          CASE WHEN "hasNewRow" THEN "newMappedRowData" ELSE 'null'::jsonb END AS "newRowData"
-        FROM ${quoteSqlIdentifier(mappedChangesTableName)}
-        WHERE "hasOldRow" OR "hasNewRow"
-      `.toStatement(mapChangesTableName),
-      ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(mapChangesTableName))),
-    ];
-  });
-
-  return {
-    tableId: options.tableId,
-    inputTables: [options.fromTable],
-    debugArgs: {
-      operator: "map",
-      tableId: tableIdToDebugString(options.tableId),
-      fromTableId: tableIdToDebugString(options.fromTable.tableId),
-      mapperSql: options.mapper.sql,
-    },
-    compareGroupKeys: options.fromTable.compareGroupKeys,
-    compareSortKeys: (a, b) => sqlExpression` 0 `,
-    init: () => {
-      const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`;
-      const fromRowsTableName = `from_rows_${generateSecureRandomString()}`;
-      const mappedRowsTableName = `mapped_rows_${generateSecureRandomString()}`;
-
-      return [
-        sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          VALUES
-          (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
-          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
-          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
-          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
-        `,
-        options.fromTable.listGroups({
-          start: "start",
-          end: "end",
-          startInclusive: true,
-          endInclusive: true,
-        }).toStatement(fromGroupsTableName),
-        sqlQuery`
-          SELECT
-            "groups"."groupkey" AS "groupKey",
-            "rows"."rowidentifier" AS "rowIdentifier",
-            "rows"."rowdata" AS "rowData"
-          FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups"
-          CROSS JOIN LATERAL (
-            ${options.fromTable.listRowsInGroup({
-              groupKey: sqlExpression`"groups"."groupkey"`,
-              start: "start",
-              end: "end",
-              startInclusive: true,
-              endInclusive: true,
-            })}
-          ) AS "rows"
-        `.toStatement(fromRowsTableName),
-        sqlQuery`
-          SELECT
-            "rows"."groupKey" AS "groupKey",
-            "rows"."rowIdentifier" AS "rowIdentifier",
-            "mapped"."rowData" AS "rowData"
-          FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows"
-          LEFT JOIN LATERAL (
-            SELECT to_jsonb("mapped") AS "rowData"
-            FROM (
-              SELECT ${options.mapper}
-              FROM (
-                SELECT
-                  "rows"."rowIdentifier" AS "rowIdentifier",
-                  "rows"."rowData" AS "rowData"
-              ) AS "mapperInput"
-            ) AS "mapped"
-          ) AS "mapped" ON true
-        `.toStatement(mappedRowsTableName),
-        sqlStatement`
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            "insertRows"."keyPath",
-            "insertRows"."value"
-          FROM (
-            SELECT DISTINCT
-              ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
-            UNION
-            SELECT DISTINCT
-              ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
-            UNION
-            SELECT
-              ${getGroupRowPath(
-                sqlExpression`"groupKey"`,
-                sqlExpression`to_jsonb("rowIdentifier"::text)`,
-              )}::jsonb[] AS "keyPath",
-              jsonb_build_object('rowData', "rowData") AS "value"
-            FROM ${quoteSqlIdentifier(mappedRowsTableName)}
-          ) AS "insertRows"
-        `,
-      ];
-    },
-    delete: () => [sqlStatement`
-      WITH RECURSIVE "pathsToDelete" AS (
-        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
-        UNION ALL
-        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
-        FROM "BulldozerStorageEngine"
-        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
-      )
-      DELETE FROM "BulldozerStorageEngine"
-      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
-    `],
-    isInitialized: () => isInitializedExpression,
-    listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
-      SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey
-      FROM "BulldozerStorageEngine" AS "groupPath"
-      WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
-        AND EXISTS (
-          SELECT 1
-          FROM "BulldozerStorageEngine" AS "groupRowsPath"
-          INNER JOIN "BulldozerStorageEngine" AS "groupRow"
-            ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath"
-          WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath"
-            AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text)
-        )
-        AND ${
-          start === "start"
-            ? sqlExpression`1 = 1`
-            : startInclusive
-              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0`
-              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0`
-        }
-        AND ${
-          end === "end"
-            ? sqlExpression`1 = 1`
-            : endInclusive
-              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0`
-              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0`
-        }
-    `,
-    listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery`
-      SELECT
-        ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier,
-        'null'::jsonb AS rowSortKey,
-        "value"->'rowData' AS rowData
-      FROM "BulldozerStorageEngine"
-      WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[]
-        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
-    ` : sqlQuery`
-      SELECT
-        "groupRows"."keyPath"[cardinality("groupRows"."keyPath") - 1] AS groupKey,
-        ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier,
-        'null'::jsonb AS rowSortKey,
-        "rows"."value"->'rowData' AS rowData
-      FROM "BulldozerStorageEngine" AS "groupRows"
-      INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath"
-      WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
-        AND cardinality("groupRows"."keyPath") = cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]) + 2
-        AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text)
-        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
-    `,
-    registerRowChangeTrigger: (trigger) => {
-      const id = generateSecureRandomString();
-      triggers.set(id, trigger);
-      return { deregister: () => triggers.delete(id) };
-    },
-  };
-}
-
 export function declareFlatMapTable<
   GK extends Json,
   OldRD extends RowData,
@@ -778,6 +470,8 @@ export function declareFlatMapTable<
   const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]);
   const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]);
   const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]);
+  const createExpandedRowIdentifier = (sourceRowIdentifier: SqlExpression, flatIndex: SqlExpression): SqlExpression =>
+    sqlExpression`(${sourceRowIdentifier} || ':' || (${flatIndex}::text))`;
   const isInitializedExpression = sqlExpression`
     EXISTS (
       SELECT 1 FROM "BulldozerStorageEngine"
@@ -827,7 +521,10 @@ export function declareFlatMapTable<
       sqlQuery`
         SELECT
           "changes"."groupKey" AS "groupKey",
-          ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+          ${createExpandedRowIdentifier(
+            sqlExpression`"changes"."sourceRowIdentifier"`,
+            sqlExpression`"flatRow"."flatIndex"`,
+          )} AS "rowIdentifier",
           "flatRow"."rowData" AS "rowData"
         FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
         CROSS JOIN LATERAL jsonb_array_elements(
@@ -845,7 +542,10 @@ export function declareFlatMapTable<
       sqlQuery`
         SELECT
           "changes"."groupKey" AS "groupKey",
-          ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+          ${createExpandedRowIdentifier(
+            sqlExpression`"changes"."sourceRowIdentifier"`,
+            sqlExpression`"flatRow"."flatIndex"`,
+          )} AS "rowIdentifier",
           "flatRow"."rowData" AS "rowData"
         FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes"
         CROSS JOIN LATERAL jsonb_array_elements(
@@ -1014,7 +714,10 @@ export function declareFlatMapTable<
         sqlQuery`
           SELECT
             "rows"."groupKey" AS "groupKey",
-            ("rows"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            ${createExpandedRowIdentifier(
+              sqlExpression`"rows"."sourceRowIdentifier"`,
+              sqlExpression`"flatRow"."flatIndex"`,
+            )} AS "rowIdentifier",
             "flatRow"."rowData" AS "rowData"
           FROM ${quoteSqlIdentifier(mappedRowsTableName)} AS "rows"
           CROSS JOIN LATERAL jsonb_array_elements(
@@ -1120,6 +823,82 @@ export function declareFlatMapTable<
   };
 }
 
+export function declareMapTable<
+  GK extends Json,
+  OldRD extends RowData,
+  NewRD extends RowData,
+>(options: {
+  tableId: TableId,
+  fromTable: Table,
+  mapper: SqlMapper,
+}): Table {
+  const nestedFlatMapTable = declareFlatMapTable({
+    tableId: { tableType: "internal", internalId: "map", parent: options.tableId },
+    fromTable: options.fromTable,
+    mapper: sqlMapper`
+      jsonb_build_array(
+        COALESCE(
+          (
+            SELECT to_jsonb("mapped")
+            FROM (
+              SELECT ${options.mapper}
+            ) AS "mapped"
+          ),
+          'null'::jsonb
+        )
+      ) AS "rows"
+    `,
+  });
+
+
+  return {
+    tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "map",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      mapperSql: options.mapper.sql,
+    },
+    init: () => [
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        VALUES
+        (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+        (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb),
+        (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb),
+        (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb)
+      `,
+      ...nestedFlatMapTable.init(),
+    ],
+    delete: () => [sqlStatement`
+      WITH RECURSIVE "pathsToDelete" AS (
+        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
+        UNION ALL
+        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
+        FROM "BulldozerStorageEngine"
+        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
+      )
+      DELETE FROM "BulldozerStorageEngine"
+      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
+    `],
+    isInitialized: () => sqlExpression`
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
+      )
+    `,
+
+    ...pick(nestedFlatMapTable, [
+      "compareGroupKeys",
+      "compareSortKeys",
+      "listGroups",
+      "listRowsInGroup",
+      "registerRowChangeTrigger",
+    ]),
+  };
+}
+
 export declare function declareFilterTable<
   GK extends Json,
   RD extends RowData,

From e041e7d708df8bf8d08d58030285bd5b596afc5c Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 11:48:57 -0700
Subject: [PATCH 14/40] Filter tables

---
 apps/backend/scripts/run-bulldozer-studio.ts  |  34 +++-
 .../src/lib/bulldozer/db/example-schema.ts    |  27 ++-
 .../src/lib/bulldozer/db/index.fuzz.test.ts   | 119 +++++++++++-
 .../src/lib/bulldozer/db/index.perf.test.ts   |  45 ++++-
 .../src/lib/bulldozer/db/index.test.ts        | 181 +++++++++++++++++-
 apps/backend/src/lib/bulldozer/db/index.ts    |  70 ++++++-
 6 files changed, 464 insertions(+), 12 deletions(-)

diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts
index 3fc40c9b65..778afc1cb2 100644
--- a/apps/backend/scripts/run-bulldozer-studio.ts
+++ b/apps/backend/scripts/run-bulldozer-studio.ts
@@ -337,6 +337,7 @@ function getStudioPageHtml(): string {
       --text: #f2f2f2;
       --muted: #b0b0b0;
       --accent: #66a3ff;
+      --filter: #f7b955;
       --danger: #ff5f56;
       --ok: #35c769;
       --mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
@@ -350,6 +351,7 @@ function getStudioPageHtml(): string {
       --text: #111111;
       --muted: #555555;
       --accent: #245ee9;
+      --filter: #b06b00;
       --danger: #d72638;
       --ok: #118a3e;
     }
@@ -476,6 +478,18 @@ function getStudioPageHtml(): string {
     .node-type.derived {
       color: var(--accent);
     }
+    .node-type.filter {
+      color: var(--filter);
+    }
+    .node-type.map {
+      color: var(--accent);
+    }
+    .node-type.flatmap {
+      color: color-mix(in srgb, var(--accent) 70%, var(--ok));
+    }
+    .node-type.groupby {
+      color: color-mix(in srgb, var(--accent) 80%, white);
+    }
     .node-name {
       font-size: 13px;
       font-weight: 700;
@@ -489,6 +503,9 @@ function getStudioPageHtml(): string {
     .node-name.derived {
       color: var(--text);
     }
+    .node-name.filter {
+      color: var(--filter);
+    }
     .node-meta {
       font-size: 11px;
       color: var(--muted);
@@ -955,18 +972,24 @@ function getStudioPageHtml(): string {
       for (const table of tables) {
         const pos = positions.get(table.id);
         if (!pos) continue;
-        const isStoredTable = String(table.operator || "").toLowerCase() === "stored";
+        const operatorClass = (() => {
+          const normalized = String(table.operator || "unknown").toLowerCase();
+          if (normalized === "stored" || normalized === "map" || normalized === "flatmap" || normalized === "groupby" || normalized === "filter") {
+            return normalized;
+          }
+          return "derived";
+        })();
         const node = document.createElement("div");
         node.className = "node" + (state.selectedTableId === table.id ? " active" : "");
         node.style.left = pos.x + "px";
         node.style.top = pos.y + "px";
 
         const type = document.createElement("div");
-        type.className = "node-type " + (isStoredTable ? "stored" : "derived");
+        type.className = "node-type " + operatorClass;
         type.textContent = String(table.operator || "unknown");
 
         const name = document.createElement("div");
-        name.className = "node-name mono " + (isStoredTable ? "stored" : "derived");
+        name.className = "node-name mono " + operatorClass;
         name.textContent = table.name;
 
         const meta = document.createElement("div");
@@ -980,7 +1003,7 @@ function getStudioPageHtml(): string {
         left.className = "row";
         if (!table.initialized) {
           const initBtn = document.createElement("button");
-          initBtn.className = "btn good";
+          initBtn.className = "btn bad";
           initBtn.textContent = "🚀 init";
           initBtn.onclick = (event) => {
             event.stopPropagation();
@@ -989,6 +1012,7 @@ function getStudioPageHtml(): string {
             });
           };
           left.appendChild(initBtn);
+          node.style.borderColor = "red";
         }
         const focusBtn = document.createElement("button");
         focusBtn.className = "btn icon";
@@ -1021,7 +1045,7 @@ function getStudioPageHtml(): string {
     }
 
     function getRawInputDefault() {
-      return "{\\n  \\"accountId\\": \\"acct-demo\\",\\n  \\"asset\\": \\"USD\\",\\n  \\"amount\\": \\"10.00\\",\\n  \\"side\\": \\"credit\\",\\n  \\"txHash\\": \\"0xdemo\\",\\n  \\"blockNumber\\": 1,\\n  \\"timestamp\\": \\"2026-01-01T00:00:00Z\\",\\n  \\"counterparty\\": null,\\n  \\"memo\\": null\\n}";
+      return "{\\n  \\"accountId\\": \\"acct-demo\\",\\n  \\"asset\\": \\"USD\\",\\n  \\"amount\\": \\"1500.00\\",\\n  \\"side\\": \\"credit\\",\\n  \\"txHash\\": \\"0xdemo\\",\\n  \\"blockNumber\\": 1,\\n  \\"timestamp\\": \\"2026-01-01T00:00:00Z\\",\\n  \\"counterparty\\": \\"acct-peer\\",\\n  \\"memo\\": null\\n}";
     }
 
     async function loadSchema() {
diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts
index ad139118f7..2fc08d3f78 100644
--- a/apps/backend/src/lib/bulldozer/db/example-schema.ts
+++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts
@@ -1,6 +1,7 @@
-import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
 
 const mapper = (sql: string) => ({ type: "mapper" as const, sql });
+const predicate = (sql: string) => ({ type: "predicate" as const, sql });
 
 /**
  * Example fungible-asset ledger schema composed from Bulldozer table operators.
@@ -99,6 +100,27 @@ export const exampleFungibleLedgerSchema = (() => {
     `),
   });
 
+  // Keep only entries with a non-null counterparty for suspicious-flow style inspections.
+  const accountEntriesWithCounterparty = declareFilterTable({
+    tableId: "bulldozer-example-ledger-account-entries-with-counterparty",
+    fromTable: entriesByAccount,
+    filter: predicate(`("rowData"->'counterparty') IS NOT NULL`),
+  });
+
+  // Keep only large-value entries to model risk/alerting-style subsets.
+  const highValueEntriesByAsset = declareFilterTable({
+    tableId: "bulldozer-example-ledger-high-value-entries-by-asset",
+    fromTable: entriesByAsset,
+    filter: predicate(`(("rowData"->>'amount')::numeric) >= 1000`),
+  });
+
+  // Partition high-value entries by account for analyst-friendly slices.
+  const highValueEntriesByAssetAccount = declareGroupByTable({
+    tableId: "bulldozer-example-ledger-high-value-entries-by-asset-account",
+    fromTable: highValueEntriesByAsset,
+    groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`),
+  });
+
   // Enrich asset-grouped rows for downstream analytics views.
   const assetEntriesNormalized = declareMapTable({
     tableId: "bulldozer-example-ledger-asset-entries-normalized",
@@ -123,6 +145,9 @@ export const exampleFungibleLedgerSchema = (() => {
     accountEntriesNormalized,
     accountEntryLegs,
     accountAssetPartitions,
+    accountEntriesWithCounterparty,
+    highValueEntriesByAsset,
+    highValueEntriesByAssetAccount,
     assetEntriesNormalized,
   };
 })();
diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index 27322afc77..7dd1e9ef4a 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -146,6 +146,21 @@ function flatMapGroups, NewRow extends Re
   }
   return mapped;
 }
+function filterGroups>(
+  groups: GroupedRows,
+  predicateFn: (row: Row) => boolean,
+): GroupedRows {
+  const filtered: GroupedRows = new Map();
+  for (const [groupKey, group] of groups) {
+    const rows = new Map();
+    for (const [rowIdentifier, rowData] of group.rows) {
+      if (!predicateFn(rowData)) continue;
+      rows.set(`${rowIdentifier}:1`, rowData);
+    }
+    filtered.set(groupKey, { groupKey: group.groupKey, rows });
+  }
+  return filtered;
+}
 
 describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   const dbUrls = getTestDbUrls();
@@ -535,6 +550,108 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
     }
   }, 120_000);
 
+  test("fuzz: filter/map pipelines preserve invariants under random mutations and re-inits", async () => {
+    const identifiers = ["ff1", "ff2", "ff3", "ff:4", "ff 5"] as const;
+    const teams = ["alpha", "beta", "gamma", null] as const;
+
+    for (const seed of [701]) {
+      const rng = createRng(seed);
+      const sourceRows = new Map();
+      let filterPipelineInitialized = true;
+
+      const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `filter-fuzz-users-${seed}` });
+      const groupedTable = declareGroupByTable({
+        tableId: `filter-fuzz-users-by-team-${seed}`,
+        fromTable,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const filterTable = declareFilterTable({
+        tableId: `filter-fuzz-users-threshold-${seed}`,
+        fromTable: groupedTable,
+        filter: { type: "predicate", sql: `("rowData"->'team') IS NOT NULL AND (("rowData"->>'value')::int) >= 10` },
+      });
+      const mappedAfterFilter = declareMapTable({
+        tableId: `filter-fuzz-users-mapped-${seed}`,
+        fromTable: filterTable,
+        mapper: mapper(`
+          ("rowData"->'team') AS "team",
+          (("rowData"->>'value')::int * 10) AS "scaledValue"
+        `),
+      });
+
+      await runStatements(fromTable.init());
+      await runStatements(groupedTable.init());
+      await runStatements(filterTable.init());
+      await runStatements(mappedAfterFilter.init());
+
+      for (let step = 0; step < 28; step++) {
+        const roll = rng();
+        if (roll < 0.6) {
+          const rowIdentifier = choose(rng, identifiers);
+          const rowData: SourceRow = {
+            team: choose(rng, teams),
+            value: Math.floor(rng() * 35) - 5,
+          };
+          sourceRows.set(rowIdentifier, rowData);
+          await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+        } else if (roll < 0.82) {
+          const rowIdentifier = choose(rng, identifiers);
+          sourceRows.delete(rowIdentifier);
+          await runStatements(fromTable.deleteRow(rowIdentifier));
+        } else if (roll < 0.9) {
+          if (filterPipelineInitialized) {
+            await runStatements(mappedAfterFilter.delete());
+            await runStatements(filterTable.delete());
+            filterPipelineInitialized = false;
+          }
+        } else {
+          if (!filterPipelineInitialized) {
+            await runStatements(filterTable.init());
+            await runStatements(mappedAfterFilter.init());
+            filterPipelineInitialized = true;
+          }
+        }
+
+        if (step % 3 === 0 || step === 27) {
+          const expectedGrouped = computeTeamGroups(sourceRows);
+          const expectedFiltered = filterGroups(expectedGrouped, (row) => row.team != null && row.value >= 10);
+          const expectedMapped = mapGroups(expectedFiltered, (row) => {
+            if (row.team == null) {
+              throw new Error("expected non-null team after filter predicate");
+            }
+            return {
+              team: row.team,
+              scaledValue: row.value * 10,
+            };
+          });
+
+          await assertTableMatches(groupedTable, expectedGrouped);
+          if (filterPipelineInitialized) {
+            expect(await readBoolean(filterTable.isInitialized())).toBe(true);
+            expect(await readBoolean(mappedAfterFilter.isInitialized())).toBe(true);
+            await assertTableMatches(filterTable, expectedFiltered);
+            await assertTableMatches(mappedAfterFilter, expectedMapped);
+          } else {
+            expect(await readBoolean(filterTable.isInitialized())).toBe(false);
+            expect(await readBoolean(mappedAfterFilter.isInitialized())).toBe(false);
+            expect(await readRows(filterTable.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }))).toEqual([]);
+            expect(await readRows(mappedAfterFilter.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }))).toEqual([]);
+          }
+        }
+      }
+    }
+  }, 120_000);
+
   test("fuzz: parallel map tables remain isolated with independent re-inits", async () => {
     const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const;
     const teams = ["alpha", "beta", null] as const;
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index 9cb4e8ff7f..927ef9fe54 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 type SqlExpression = { type: "expression", sql: string };
@@ -32,6 +32,8 @@ const LOAD_DERIVED_COUNT_QUERY_MAX_MS = 10_000;
 const LOAD_EXPANDING_INIT_MAX_MS = 120_000;
 const LOAD_EXPANDING_COUNT_QUERY_MAX_MS = 15_000;
 const LOAD_FILTERED_QUERY_MAX_MS = 4_000;
+const LOAD_FILTER_TABLE_INIT_MAX_MS = 90_000;
+const LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS = 8_000;
 
 function getTestDbUrls(): TestDb {
   const env = Reflect.get(import.meta, "env");
@@ -422,6 +424,11 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       fromTable: mappedTwice,
       groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` },
     });
+    const filteredHighValue = declareFilterTable({
+      tableId: "load-prefilled-users-high-value",
+      fromTable: groupedByTeam,
+      filter: { type: "predicate", sql: `( ("rowData"->>'value')::int ) >= 700` },
+    });
     const expandedByTeam = declareFlatMapTable({
       tableId: "load-prefilled-users-expanded",
       fromTable: groupedByTeam,
@@ -457,6 +464,10 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       await runStatements(groupedByBucket.init());
     });
     expect(bucketInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS);
+    const filterInit = await measureMs("load init filteredHighValue", async () => {
+      await runStatements(filteredHighValue.init());
+    });
+    expect(filterInit.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_INIT_MAX_MS);
     const expandInit = await measureMs("load init expandedByTeam", async () => {
       await runStatements(expandedByTeam.init());
     });
@@ -486,11 +497,18 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     });
+    const filteredHighValueCountQuery = filteredHighValue.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    });
     const derivedCounts = await measureMs("load count derived tables", async () => {
       return await Promise.all([
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(groupedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`),
+        sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"`),
       ]);
     });
@@ -498,7 +516,18 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(Number(derivedCounts.result[0][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[1][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1);
-    expect(Number(derivedCounts.result[3][0].count)).toBe((loadRowCount - 1) * 2);
+    expect(Number(derivedCounts.result[3][0].count)).toBeGreaterThan(0);
+    expect(Number(derivedCounts.result[3][0].count)).toBeLessThan(loadRowCount);
+    expect(Number(derivedCounts.result[4][0].count)).toBe((loadRowCount - 1) * 2);
+
+    const filteredHighValueCountOnly = await measureMs("load count filteredHighValue table only", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows"
+      `);
+    });
+    expect(filteredHighValueCountOnly.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS);
+    expect(Number(filteredHighValueCountOnly.result[0].count)).toBeGreaterThan(0);
 
     const expandedCountOnly = await measureMs("load count expanded table only", async () => {
       return await sql.unsafe(`
@@ -564,6 +593,16 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       { rowIdentifier: "seed-100000:1", rowData: { team: "delta", kind: "base", mappedValue: 1009 } },
       { rowIdentifier: "seed-100000:2", rowData: { team: "delta", kind: "double", mappedValue: 1998 } },
     ]);
+    const filteredDeltaRows = await readRows(filteredHighValue.listRowsInGroup({
+      groupKey: expr(`to_jsonb('delta'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(filteredDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "seed-100000:1", rowData: { team: "delta", value: 999 } },
+    ]);
 
     const bulkDelete = await measureMs("load full table delete", async () => {
       await runStatements(table.delete());
@@ -583,7 +622,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     `;
     expect(isInitializedRows[0].initialized).toBe(false);
 
-    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
+    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
   }, 180_000);
 });
 
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index 8c898cd400..1f8d9409f4 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -26,6 +26,7 @@ type SqlExpression = { type: "expression", sql: string };
 type SqlStatement = { type: "statement", sql: string, outputName?: string };
 type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement };
 type SqlMapper = { type: "mapper", sql: string };
+type SqlPredicate = { type: "predicate", sql: string };
 
 function expr(sql: string): SqlExpression {
   return { type: "expression", sql };
@@ -33,6 +34,9 @@ function expr(sql: string): SqlExpression {
 function mapper(sql: string): SqlMapper {
   return { type: "mapper", sql };
 }
+function predicate(sql: string): SqlPredicate {
+  return { type: "predicate", sql };
+}
 
 const sqlStringLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`;
 const sqlStatement = (strings: TemplateStringsArray, ...values: { sql: string }[]): SqlStatement => ({
@@ -241,6 +245,15 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     });
     return { fromTable, groupedTable, flatMappedTable };
   }
+  function createFilteredTable() {
+    const { fromTable, groupedTable } = createGroupedTable();
+    const filteredTable = declareFilterTable({
+      tableId: "users-by-team-filtered",
+      fromTable: groupedTable,
+      filter: predicate(`(("rowData"->>'value')::int) >= 2`),
+    });
+    return { fromTable, groupedTable, filteredTable };
+  }
   function createFlatMapMapGroupPipeline() {
     const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
     const mappedAfterFlatMap = declareMapTable({
@@ -363,6 +376,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       `,
     ]);
   }
+  function registerFilterAuditTrigger(
+    table: ReturnType["filteredTable"],
+    event: string,
+  ) {
+    return table.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral(event))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+  }
 
   test("init/isInitialized/delete lifecycle", async () => {
     const table = declareStoredTable<{ value: number }>({ tableId: "users" });
@@ -1644,6 +1680,149 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(staleGroupPaths).toEqual([]);
   });
 
+  test("filterTable init backfills matching rows, keeps own metadata, and deletes cleanly", async () => {
+    const { fromTable, groupedTable, filteredTable } = createFilteredTable();
+    await runStatements(fromTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u3", expr(`'{"team":"beta","value":3}'::jsonb`)));
+    await runStatements(fromTable.setRow("u4", expr(`'{"team":"beta","value":0}'::jsonb`)));
+    await runStatements(groupedTable.init());
+    await runStatements(filteredTable.init());
+
+    expect(await readBoolean(filteredTable.isInitialized())).toBe(true);
+
+    const groups = await readRows(filteredTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]);
+
+    const allRows = await readRows(filteredTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
+      { groupKey: "alpha", rowIdentifier: "u2:1", rowData: { team: "alpha", value: 2 } },
+      { groupKey: "beta", rowIdentifier: "u3:1", rowData: { team: "beta", value: 3 } },
+    ]);
+
+    const metadataRows = await sql`
+      SELECT 1
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPath" = ARRAY[
+        to_jsonb('table'::text),
+        to_jsonb('external:users-by-team-filtered'::text),
+        to_jsonb('storage'::text),
+        to_jsonb('metadata'::text)
+      ]::jsonb[]
+    `;
+    expect(metadataRows).toHaveLength(1);
+
+    await runStatements(filteredTable.delete());
+    expect(await readBoolean(filteredTable.isInitialized())).toBe(false);
+    const groupsAfterDelete = await readRows(filteredTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groupsAfterDelete).toEqual([]);
+  });
+
+  test("filterTable registerRowChangeTrigger emits inserts, updates, deletes, and moves", async () => {
+    const { fromTable, groupedTable, filteredTable } = createFilteredTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(filteredTable.init());
+    registerFilterAuditTrigger(filteredTable, "filter_change");
+
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":5}'::jsonb`)));
+
+    const normalizedAuditRows = (await readMapTriggerAuditRows())
+      .map((row) => ({
+        groupKey: row.groupKey,
+        rowIdentifier: row.rowIdentifier,
+        oldRowData: row.oldRowData,
+        newRowData: row.newRowData,
+      }))
+      .sort((a, b) => stringCompare(
+        `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`,
+        `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`,
+      ));
+    expect(normalizedAuditRows).toEqual([
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: null,
+        newRowData: { team: "alpha", value: 2 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: { team: "alpha", value: 2 },
+        newRowData: { team: "alpha", value: 3 },
+      },
+      {
+        groupKey: "alpha",
+        rowIdentifier: "u1:1",
+        oldRowData: { team: "alpha", value: 3 },
+        newRowData: null,
+      },
+      {
+        groupKey: "beta",
+        rowIdentifier: "u1:1",
+        oldRowData: null,
+        newRowData: { team: "beta", value: 5 },
+      },
+    ]);
+  });
+
+  test("filterTable stays no-op while uninitialized", async () => {
+    const { fromTable, groupedTable, filteredTable } = createFilteredTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    registerFilterAuditTrigger(filteredTable, "filter_change");
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`)));
+
+    expect(await readBoolean(filteredTable.isInitialized())).toBe(false);
+    expect(await readMapTriggerAuditRows()).toEqual([]);
+    expect(await readRows(filteredTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }))).toEqual([]);
+  });
+
+  test("filterTable listRowsInGroup (all groups) handles 'rows' collisions in group key and source row identifier", async () => {
+    const { fromTable, groupedTable, filteredTable } = createFilteredTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(filteredTable.init());
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":5}'::jsonb`)));
+    await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":4}'::jsonb`)));
+
+    const allRows = await readRows(filteredTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
+      { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", value: 4 } },
+      { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", value: 5 } },
+    ]);
+  });
+
   test("flatMap -> map -> groupBy composition stays consistent across updates", async () => {
     const { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind } = createFlatMapMapGroupPipeline();
     await runStatements(fromTable.init());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index e7261fa659..cd572e41c0 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -899,13 +899,81 @@ export function declareMapTable<
   };
 }
 
-export declare function declareFilterTable<
+export function declareFilterTable<
   GK extends Json,
   RD extends RowData,
 >(options: {
   tableId: TableId,
   fromTable: Table,
   filter: SqlPredicate,
+}): Table {
+  const nestedFlatMapTable = declareFlatMapTable({
+    tableId: { tableType: "internal", internalId: "filter", parent: options.tableId },
+    fromTable: options.fromTable,
+    mapper: sqlMapper`
+      CASE
+        WHEN ${options.filter}
+          THEN jsonb_build_array("rowData")
+        ELSE '[]'::jsonb
+      END AS "rows"
+    `,
+  });
+
+  return {
+    tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "filter",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      filterSql: options.filter.sql,
+    },
+    init: () => [
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        VALUES
+        (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+        (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb),
+        (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb),
+        (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb)
+      `,
+      ...nestedFlatMapTable.init(),
+    ],
+    delete: () => [sqlStatement`
+      WITH RECURSIVE "pathsToDelete" AS (
+        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
+        UNION ALL
+        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
+        FROM "BulldozerStorageEngine"
+        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
+      )
+      DELETE FROM "BulldozerStorageEngine"
+      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
+    `],
+    isInitialized: () => sqlExpression`
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
+      )
+    `,
+    ...pick(nestedFlatMapTable, [
+      "compareGroupKeys",
+      "compareSortKeys",
+      "listGroups",
+      "listRowsInGroup",
+      "registerRowChangeTrigger",
+    ]),
+  };
+}
+
+export declare function declareLimitTable<
+  GK extends Json,
+  RD extends RowData,
+>(options: {
+  tableId: TableId,
+  fromTable: Table,
+  limit: SqlExpression,
+  virtual: true,
 }): Table;
 
 export declare function declareSortTable<

From e2b0d3db6adad022da6795f6c5cf38dfbbfb6d0a Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 15:58:43 -0700
Subject: [PATCH 15/40] Limit tables

---
 apps/backend/scripts/run-bulldozer-studio.ts  |   8 +-
 .../src/lib/bulldozer/db/example-schema.ts    |  14 +-
 .../src/lib/bulldozer/db/index.fuzz.test.ts   |  92 +++-
 .../src/lib/bulldozer/db/index.perf.test.ts   |  45 +-
 .../src/lib/bulldozer/db/index.test.ts        | 200 ++++++++-
 apps/backend/src/lib/bulldozer/db/index.ts    | 394 +++++++++++++++++-
 6 files changed, 742 insertions(+), 11 deletions(-)

diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts
index 778afc1cb2..d2dc17301c 100644
--- a/apps/backend/scripts/run-bulldozer-studio.ts
+++ b/apps/backend/scripts/run-bulldozer-studio.ts
@@ -490,6 +490,9 @@ function getStudioPageHtml(): string {
     .node-type.groupby {
       color: color-mix(in srgb, var(--accent) 80%, white);
     }
+    .node-type.limit {
+      color: color-mix(in srgb, var(--filter) 75%, var(--text));
+    }
     .node-name {
       font-size: 13px;
       font-weight: 700;
@@ -506,6 +509,9 @@ function getStudioPageHtml(): string {
     .node-name.filter {
       color: var(--filter);
     }
+    .node-name.limit {
+      color: color-mix(in srgb, var(--filter) 85%, var(--text));
+    }
     .node-meta {
       font-size: 11px;
       color: var(--muted);
@@ -974,7 +980,7 @@ function getStudioPageHtml(): string {
         if (!pos) continue;
         const operatorClass = (() => {
           const normalized = String(table.operator || "unknown").toLowerCase();
-          if (normalized === "stored" || normalized === "map" || normalized === "flatmap" || normalized === "groupby" || normalized === "filter") {
+          if (normalized === "stored" || normalized === "map" || normalized === "flatmap" || normalized === "groupby" || normalized === "filter" || normalized === "limit") {
             return normalized;
           }
           return "derived";
diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts
index 2fc08d3f78..24ab566bc7 100644
--- a/apps/backend/src/lib/bulldozer/db/example-schema.ts
+++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts
@@ -1,4 +1,4 @@
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable } from "./index";
 
 const mapper = (sql: string) => ({ type: "mapper" as const, sql });
 const predicate = (sql: string) => ({ type: "predicate" as const, sql });
@@ -106,6 +106,11 @@ export const exampleFungibleLedgerSchema = (() => {
     fromTable: entriesByAccount,
     filter: predicate(`("rowData"->'counterparty') IS NOT NULL`),
   });
+  const accountCounterpartySample = declareLimitTable({
+    tableId: "bulldozer-example-ledger-account-counterparty-sample",
+    fromTable: accountEntriesWithCounterparty,
+    limit: { type: "expression", sql: "1" },
+  });
 
   // Keep only large-value entries to model risk/alerting-style subsets.
   const highValueEntriesByAsset = declareFilterTable({
@@ -120,6 +125,11 @@ export const exampleFungibleLedgerSchema = (() => {
     fromTable: highValueEntriesByAsset,
     groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`),
   });
+  const highValueEntriesByAssetAccountTop = declareLimitTable({
+    tableId: "bulldozer-example-ledger-high-value-entries-by-asset-account-top",
+    fromTable: highValueEntriesByAssetAccount,
+    limit: { type: "expression", sql: "3" },
+  });
 
   // Enrich asset-grouped rows for downstream analytics views.
   const assetEntriesNormalized = declareMapTable({
@@ -146,8 +156,10 @@ export const exampleFungibleLedgerSchema = (() => {
     accountEntryLegs,
     accountAssetPartitions,
     accountEntriesWithCounterparty,
+    accountCounterpartySample,
     highValueEntriesByAsset,
     highValueEntriesByAssetAccount,
+    highValueEntriesByAssetAccountTop,
     assetEntriesNormalized,
   };
 })();
diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index 7dd1e9ef4a..a30af8c128 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -161,6 +161,24 @@ function filterGroups>(
   }
   return filtered;
 }
+function limitGroups>(
+  groups: GroupedRows,
+  limit: number,
+): GroupedRows {
+  const limited: GroupedRows = new Map();
+  for (const [groupKey, group] of groups) {
+    const rows = new Map();
+    const sortedRows = [...group.rows.entries()].sort((a, b) => stringCompare(a[0], b[0]));
+    for (let i = 0; i < Math.min(limit, sortedRows.length); i++) {
+      const entry = sortedRows[i] ?? (() => {
+        throw new Error("limitGroups expected sorted row entry to exist");
+      })();
+      rows.set(entry[0], entry[1]);
+    }
+    limited.set(groupKey, { groupKey: group.groupKey, rows });
+  }
+  return limited;
+}
 
 describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   const dbUrls = getTestDbUrls();
@@ -652,6 +670,78 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
     }
   }, 120_000);
 
+  test("fuzz: grouped limit table remains consistent under random mutations and re-inits", async () => {
+    const identifiers = ["l1", "l2", "l3", "l4", "l 5", "l:6"] as const;
+    const teams = ["alpha", "beta", "gamma", null] as const;
+
+    for (const seed of [801]) {
+      const rng = createRng(seed);
+      const sourceRows = new Map();
+      let limitInitialized = true;
+
+      const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `limit-fuzz-users-${seed}` });
+      const groupedTable = declareGroupByTable({
+        tableId: `limit-fuzz-users-by-team-${seed}`,
+        fromTable,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const limitedByTeam = declareLimitTable({
+        tableId: `limit-fuzz-users-top2-${seed}`,
+        fromTable: groupedTable,
+        limit: expr(`2`),
+      });
+
+      await runStatements(fromTable.init());
+      await runStatements(groupedTable.init());
+      await runStatements(limitedByTeam.init());
+
+      for (let step = 0; step < 36; step++) {
+        const roll = rng();
+        if (roll < 0.62) {
+          const rowIdentifier = choose(rng, identifiers);
+          const rowData: SourceRow = {
+            team: choose(rng, teams),
+            value: Math.floor(rng() * 100),
+          };
+          sourceRows.set(rowIdentifier, rowData);
+          await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+        } else if (roll < 0.86) {
+          const rowIdentifier = choose(rng, identifiers);
+          sourceRows.delete(rowIdentifier);
+          await runStatements(fromTable.deleteRow(rowIdentifier));
+        } else if (roll < 0.93) {
+          if (limitInitialized) {
+            await runStatements(limitedByTeam.delete());
+            limitInitialized = false;
+          }
+        } else {
+          if (!limitInitialized) {
+            await runStatements(limitedByTeam.init());
+            limitInitialized = true;
+          }
+        }
+
+        if (step % 3 === 0 || step === 35) {
+          const expectedGrouped = computeTeamGroups(sourceRows);
+          const expectedLimited = limitGroups(expectedGrouped, 2);
+          await assertTableMatches(groupedTable, expectedGrouped);
+          if (limitInitialized) {
+            expect(await readBoolean(limitedByTeam.isInitialized())).toBe(true);
+            await assertTableMatches(limitedByTeam, expectedLimited);
+          } else {
+            expect(await readBoolean(limitedByTeam.isInitialized())).toBe(false);
+            expect(await readRows(limitedByTeam.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }))).toEqual([]);
+          }
+        }
+      }
+    }
+  }, 120_000);
+
   test("fuzz: parallel map tables remain isolated with independent re-inits", async () => {
     const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const;
     const teams = ["alpha", "beta", null] as const;
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index 927ef9fe54..f8e41cbdec 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 type SqlExpression = { type: "expression", sql: string };
@@ -34,6 +34,8 @@ const LOAD_EXPANDING_COUNT_QUERY_MAX_MS = 15_000;
 const LOAD_FILTERED_QUERY_MAX_MS = 4_000;
 const LOAD_FILTER_TABLE_INIT_MAX_MS = 90_000;
 const LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS = 8_000;
+const LOAD_LIMIT_TABLE_INIT_MAX_MS = 90_000;
+const LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS = 8_000;
 
 function getTestDbUrls(): TestDb {
   const env = Reflect.get(import.meta, "env");
@@ -429,6 +431,11 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       fromTable: groupedByTeam,
       filter: { type: "predicate", sql: `( ("rowData"->>'value')::int ) >= 700` },
     });
+    const limitedByTeam = declareLimitTable({
+      tableId: "load-prefilled-users-top-team-rows",
+      fromTable: groupedByTeam,
+      limit: expr(`25`),
+    });
     const expandedByTeam = declareFlatMapTable({
       tableId: "load-prefilled-users-expanded",
       fromTable: groupedByTeam,
@@ -468,6 +475,10 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       await runStatements(filteredHighValue.init());
     });
     expect(filterInit.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_INIT_MAX_MS);
+    const limitInit = await measureMs("load init limitedByTeam", async () => {
+      await runStatements(limitedByTeam.init());
+    });
+    expect(limitInit.elapsedMs).toBeLessThan(LOAD_LIMIT_TABLE_INIT_MAX_MS);
     const expandInit = await measureMs("load init expandedByTeam", async () => {
       await runStatements(expandedByTeam.init());
     });
@@ -503,12 +514,19 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     });
+    const limitedByTeamCountQuery = limitedByTeam.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    });
     const derivedCounts = await measureMs("load count derived tables", async () => {
       return await Promise.all([
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(groupedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows"`),
+        sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(limitedByTeamCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"`),
       ]);
     });
@@ -518,7 +536,9 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[3][0].count)).toBeGreaterThan(0);
     expect(Number(derivedCounts.result[3][0].count)).toBeLessThan(loadRowCount);
-    expect(Number(derivedCounts.result[4][0].count)).toBe((loadRowCount - 1) * 2);
+    expect(Number(derivedCounts.result[4][0].count)).toBeGreaterThan(0);
+    expect(Number(derivedCounts.result[4][0].count)).toBeLessThanOrEqual(100);
+    expect(Number(derivedCounts.result[5][0].count)).toBe((loadRowCount - 1) * 2);
 
     const filteredHighValueCountOnly = await measureMs("load count filteredHighValue table only", async () => {
       return await sql.unsafe(`
@@ -529,6 +549,16 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(filteredHighValueCountOnly.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS);
     expect(Number(filteredHighValueCountOnly.result[0].count)).toBeGreaterThan(0);
 
+    const limitedByTeamCountOnly = await measureMs("load count limitedByTeam table only", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (${toQueryableSqlQuery(limitedByTeamCountQuery)}) AS "rows"
+      `);
+    });
+    expect(limitedByTeamCountOnly.elapsedMs).toBeLessThan(LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS);
+    expect(Number(limitedByTeamCountOnly.result[0].count)).toBeGreaterThan(0);
+    expect(Number(limitedByTeamCountOnly.result[0].count)).toBeLessThanOrEqual(100);
+
     const expandedCountOnly = await measureMs("load count expanded table only", async () => {
       return await sql.unsafe(`
         SELECT COUNT(*)::int AS "count"
@@ -603,6 +633,15 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(filteredDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
       { rowIdentifier: "seed-100000:1", rowData: { team: "delta", value: 999 } },
     ]);
+    const limitedDeltaRows = await readRows(limitedByTeam.listRowsInGroup({
+      groupKey: expr(`to_jsonb('delta'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(limitedDeltaRows).toHaveLength(1);
+    expect(limitedDeltaRows[0]?.rowidentifier).toBe("seed-100000");
 
     const bulkDelete = await measureMs("load full table delete", async () => {
       await runStatements(table.delete());
@@ -622,7 +661,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     `;
     expect(isInitializedRows[0].initialized).toBe(false);
 
-    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
+    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, limitInit<=${LOAD_LIMIT_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, limitCount<=${LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
   }, 180_000);
 });
 
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index 1f8d9409f4..3887c0077c 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -254,6 +254,15 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     });
     return { fromTable, groupedTable, filteredTable };
   }
+  function createLimitedTable() {
+    const { fromTable, groupedTable } = createGroupedTable();
+    const limitedTable = declareLimitTable({
+      tableId: "users-by-team-limited",
+      fromTable: groupedTable,
+      limit: expr(`2`),
+    });
+    return { fromTable, groupedTable, limitedTable };
+  }
   function createFlatMapMapGroupPipeline() {
     const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
     const mappedAfterFlatMap = declareMapTable({
@@ -399,6 +408,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       `,
     ]);
   }
+  function registerLimitAuditTrigger(
+    table: ReturnType["limitedTable"],
+    event: string,
+  ) {
+    return table.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral(event))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+  }
 
   test("init/isInitialized/delete lifecycle", async () => {
     const table = declareStoredTable<{ value: number }>({ tableId: "users" });
@@ -1823,6 +1855,172 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     ]);
   });
 
+  test("limitTable init keeps only first N rows per group and stores metadata", async () => {
+    const { fromTable, groupedTable, limitedTable } = createLimitedTable();
+    await runStatements(fromTable.init());
+    await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("b2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":1}'::jsonb`)));
+    await runStatements(groupedTable.init());
+    await runStatements(limitedTable.init());
+
+    const groups = await readRows(limitedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]);
+
+    const allRows = await readRows(limitedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([
+      { groupKey: "alpha", rowIdentifier: "a1", rowData: { team: "alpha", value: 1 } },
+      { groupKey: "alpha", rowIdentifier: "a2", rowData: { team: "alpha", value: 2 } },
+      { groupKey: "beta", rowIdentifier: "b1", rowData: { team: "beta", value: 1 } },
+      { groupKey: "beta", rowIdentifier: "b2", rowData: { team: "beta", value: 2 } },
+    ]);
+
+    const metadataRows = await sql`
+      SELECT 1
+      FROM "BulldozerStorageEngine"
+      WHERE "keyPath" = ARRAY[
+        to_jsonb('table'::text),
+        to_jsonb('external:users-by-team-limited'::text),
+        to_jsonb('storage'::text),
+        to_jsonb('metadata'::text)
+      ]::jsonb[]
+    `;
+    expect(metadataRows).toHaveLength(1);
+  });
+
+  test("limitTable membership shifts when boundary rows are inserted, updated, or deleted", async () => {
+    const { fromTable, groupedTable, limitedTable } = createLimitedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(limitedTable.init());
+
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    let alphaRows = await readRows(limitedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u2", "u3"]);
+
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    alphaRows = await readRows(limitedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u1", "u2"]);
+
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":22}'::jsonb`)));
+    alphaRows = await readRows(limitedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "u1", rowData: { team: "alpha", value: 1 } },
+      { rowIdentifier: "u2", rowData: { team: "alpha", value: 22 } },
+    ]);
+
+    await runStatements(fromTable.deleteRow("u1"));
+    alphaRows = await readRows(limitedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u2", "u3"]);
+  });
+
+  test("limitTable trigger stream reconstructs the same final state as listRowsInGroup", async () => {
+    const { fromTable, groupedTable, limitedTable } = createLimitedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    await runStatements(limitedTable.init());
+    registerLimitAuditTrigger(limitedTable, "limit_change");
+
+    await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+    await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    await runStatements(fromTable.setRow("a4", expr(`'{"team":"alpha","value":4}'::jsonb`)));
+    await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("a1"));
+    await runStatements(fromTable.setRow("a5", expr(`'{"team":"alpha","value":5}'::jsonb`)));
+    await runStatements(fromTable.setRow("a0", expr(`'{"team":"alpha","value":0}'::jsonb`)));
+    await runStatements(fromTable.deleteRow("a2"));
+    await runStatements(fromTable.setRow("a0", expr(`'{"team":"beta","value":100}'::jsonb`)));
+
+    const auditRows = (await readMapTriggerAuditRows())
+      .filter((row) => row.event === "limit_change");
+    const reconstructed = new Map();
+    for (const row of auditRows) {
+      const groupKey = row.groupKey as string | null;
+      const rowIdentifier = String(row.rowIdentifier);
+      const key = `${groupKey ?? "__NULL__"}:${rowIdentifier}`;
+      if (row.newRowData == null) {
+        reconstructed.delete(key);
+      } else {
+        reconstructed.set(key, { groupKey, rowIdentifier, rowData: row.newRowData });
+      }
+    }
+
+    const actualRows = (await readRows(limitedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }))).map((row) => ({
+      groupKey: row.groupkey as string | null,
+      rowIdentifier: String(row.rowidentifier),
+      rowData: row.rowdata,
+    }));
+    const reconstructedRows = [...reconstructed.values()];
+    const sortRows = (rows: Array<{ groupKey: string | null, rowIdentifier: string, rowData: unknown }>) => rows
+      .sort((a, b) => stringCompare(
+        `${a.groupKey ?? "__NULL__"}:${a.rowIdentifier}:${JSON.stringify(a.rowData)}`,
+        `${b.groupKey ?? "__NULL__"}:${b.rowIdentifier}:${JSON.stringify(b.rowData)}`,
+      ));
+    expect(sortRows(reconstructedRows)).toEqual(sortRows(actualRows));
+  });
+
+  test("limitTable stays no-op while uninitialized", async () => {
+    const { fromTable, groupedTable, limitedTable } = createLimitedTable();
+    await runStatements(fromTable.init());
+    await runStatements(groupedTable.init());
+    registerLimitAuditTrigger(limitedTable, "limit_change");
+    await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`)));
+
+    expect(await readBoolean(limitedTable.isInitialized())).toBe(false);
+    const groups = await readRows(limitedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups).toEqual([]);
+    const limitAuditRows = (await readMapTriggerAuditRows()).filter((row) => row.event === "limit_change");
+    expect(limitAuditRows).toEqual([]);
+  });
+
   test("flatMap -> map -> groupBy composition stays consistent across updates", async () => {
     const { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind } = createFlatMapMapGroupPipeline();
     await runStatements(fromTable.init());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index cd572e41c0..bf9b7640a3 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -966,15 +966,401 @@ export function declareFilterTable<
   };
 }
 
-export declare function declareLimitTable<
+export function declareLimitTable<
   GK extends Json,
+  SK extends Json,
   RD extends RowData,
 >(options: {
   tableId: TableId,
-  fromTable: Table,
+  fromTable: Table,
   limit: SqlExpression,
-  virtual: true,
-}): Table;
+}): Table {
+  const triggers = new Map) => SqlStatement[]>();
+  const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]);
+  const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]);
+  const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]);
+  const normalizedLimit = sqlExpression`GREATEST((${options.limit})::int, 0)`;
+  const isInitializedExpression = sqlExpression`
+    EXISTS (
+      SELECT 1 FROM "BulldozerStorageEngine"
+      WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
+    )
+  `;
+
+  // TODO: Currently, we recompute the entire limit table when a particular group changes. In the future, we should use an ordered tree to do this incrementally
+
+  options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
+    const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`;
+    const affectedGroupsTableName = `affected_groups_${generateSecureRandomString()}`;
+    const oldGroupRowsTableName = `old_group_rows_${generateSecureRandomString()}`;
+    const newGroupRowsTableName = `new_group_rows_${generateSecureRandomString()}`;
+    const oldLimitedRowsTableName = `old_limited_rows_${generateSecureRandomString()}`;
+    const newLimitedRowsTableName = `new_limited_rows_${generateSecureRandomString()}`;
+    const limitChangesTableName = `limit_changes_${generateSecureRandomString()}`;
+    return [
+      sqlQuery`
+        SELECT
+          "changes"."groupKey" AS "groupKey",
+          "changes"."rowIdentifier" AS "rowIdentifier",
+          "changes"."oldRowSortKey" AS "oldRowSortKey",
+          "changes"."newRowSortKey" AS "newRowSortKey",
+          "changes"."oldRowData" AS "oldRowData",
+          "changes"."newRowData" AS "newRowData",
+          ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+          ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow"
+        FROM ${fromChangesTable} AS "changes"
+        WHERE ${isInitializedExpression}
+      `.toStatement(normalizedChangesTableName),
+      sqlQuery`
+        SELECT DISTINCT "changes"."groupKey" AS "groupKey"
+        FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes"
+        WHERE "changes"."hasOldRow" OR "changes"."hasNewRow"
+      `.toStatement(affectedGroupsTableName),
+      sqlQuery`
+        SELECT
+          "groups"."groupKey" AS "groupKey",
+          "rows"."rowidentifier" AS "rowIdentifier",
+          "rows"."rowsortkey" AS "rowSortKey",
+          "rows"."rowdata" AS "rowData"
+        FROM ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups"
+        CROSS JOIN LATERAL (
+          ${options.fromTable.listRowsInGroup({
+            groupKey: sqlExpression`"groups"."groupKey"`,
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          })}
+        ) AS "rows"
+      `.toStatement(oldGroupRowsTableName),
+      sqlQuery`
+        SELECT
+          "rows"."groupKey" AS "groupKey",
+          "rows"."rowIdentifier" AS "rowIdentifier",
+          "rows"."rowSortKey" AS "rowSortKey",
+          "rows"."rowData" AS "rowData"
+        FROM ${quoteSqlIdentifier(oldGroupRowsTableName)} AS "rows"
+        WHERE NOT EXISTS (
+          SELECT 1
+          FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes"
+          WHERE "changes"."hasOldRow"
+            AND "changes"."groupKey" IS NOT DISTINCT FROM "rows"."groupKey"
+            AND "changes"."rowIdentifier" = "rows"."rowIdentifier"
+        )
+        UNION ALL
+        SELECT
+          "changes"."groupKey" AS "groupKey",
+          "changes"."rowIdentifier" AS "rowIdentifier",
+          "changes"."newRowSortKey" AS "rowSortKey",
+          "changes"."newRowData" AS "rowData"
+        FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes"
+        WHERE "changes"."hasNewRow"
+      `.toStatement(newGroupRowsTableName),
+      sqlQuery`
+        SELECT
+          "rankedRows"."groupKey" AS "groupKey",
+          "rankedRows"."rowIdentifier" AS "rowIdentifier",
+          "rankedRows"."rowSortKey" AS "rowSortKey",
+          "rankedRows"."rowData" AS "rowData"
+        FROM (
+          SELECT
+            "rows"."groupKey" AS "groupKey",
+            "rows"."rowIdentifier" AS "rowIdentifier",
+            "rows"."rowSortKey" AS "rowSortKey",
+            "rows"."rowData" AS "rowData",
+            row_number() OVER (
+              PARTITION BY "rows"."groupKey"
+              ORDER BY "rows"."rowSortKey" ASC, "rows"."rowIdentifier" ASC
+            ) AS "rank"
+          FROM ${quoteSqlIdentifier(oldGroupRowsTableName)} AS "rows"
+        ) AS "rankedRows"
+        WHERE "rankedRows"."rank" <= ${normalizedLimit}
+      `.toStatement(oldLimitedRowsTableName),
+      sqlQuery`
+        SELECT
+          "rankedRows"."groupKey" AS "groupKey",
+          "rankedRows"."rowIdentifier" AS "rowIdentifier",
+          "rankedRows"."rowSortKey" AS "rowSortKey",
+          "rankedRows"."rowData" AS "rowData"
+        FROM (
+          SELECT
+            "rows"."groupKey" AS "groupKey",
+            "rows"."rowIdentifier" AS "rowIdentifier",
+            "rows"."rowSortKey" AS "rowSortKey",
+            "rows"."rowData" AS "rowData",
+            row_number() OVER (
+              PARTITION BY "rows"."groupKey"
+              ORDER BY "rows"."rowSortKey" ASC, "rows"."rowIdentifier" ASC
+            ) AS "rank"
+          FROM ${quoteSqlIdentifier(newGroupRowsTableName)} AS "rows"
+        ) AS "rankedRows"
+        WHERE "rankedRows"."rank" <= ${normalizedLimit}
+      `.toStatement(newLimitedRowsTableName),
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          "insertRows"."keyPath",
+          "insertRows"."value"
+        FROM (
+          SELECT DISTINCT
+            ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(newLimitedRowsTableName)}
+          UNION
+          SELECT DISTINCT
+            ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+            'null'::jsonb AS "value"
+          FROM ${quoteSqlIdentifier(newLimitedRowsTableName)}
+        ) AS "insertRows"
+        ON CONFLICT ("keyPath") DO NOTHING
+      `,
+      sqlStatement`
+        DELETE FROM "BulldozerStorageEngine" AS "target"
+        USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups"
+        WHERE "target"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[]
+      `,
+      sqlStatement`
+        INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+        SELECT
+          gen_random_uuid(),
+          ${getGroupRowPath(
+            sqlExpression`"groupKey"`,
+            sqlExpression`to_jsonb("rowIdentifier"::text)`,
+          )}::jsonb[],
+          jsonb_build_object(
+            'rowSortKey', "rowSortKey",
+            'rowData', "rowData"
+          )
+        FROM ${quoteSqlIdentifier(newLimitedRowsTableName)}
+        ON CONFLICT ("keyPath") DO UPDATE
+        SET "value" = EXCLUDED."value"
+      `,
+      sqlStatement`
+        DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+        USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups"
+        WHERE "staleGroupPath"."keyPath" IN (
+          ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[],
+          ${getGroupKeyPath(sqlExpression`"groups"."groupKey"`)}::jsonb[]
+        )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${quoteSqlIdentifier(newLimitedRowsTableName)} AS "newRows"
+            WHERE "newRows"."groupKey" IS NOT DISTINCT FROM "groups"."groupKey"
+          )
+      `,
+      sqlQuery`
+        SELECT
+          COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
+          COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
+          CASE WHEN "oldRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowSortKey" END AS "oldRowSortKey",
+          CASE WHEN "newRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowSortKey" END AS "newRowSortKey",
+          CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
+          CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
+        FROM ${quoteSqlIdentifier(oldLimitedRowsTableName)} AS "oldRows"
+        FULL OUTER JOIN ${quoteSqlIdentifier(newLimitedRowsTableName)} AS "newRows"
+          ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
+          AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
+        WHERE "oldRows"."rowSortKey" IS DISTINCT FROM "newRows"."rowSortKey"
+          OR "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
+      `.toStatement(limitChangesTableName),
+      ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(limitChangesTableName))),
+    ];
+  });
+
+  return {
+    tableId: options.tableId,
+    inputTables: [options.fromTable],
+    debugArgs: {
+      operator: "limit",
+      tableId: tableIdToDebugString(options.tableId),
+      fromTableId: tableIdToDebugString(options.fromTable.tableId),
+      limitSql: options.limit.sql,
+    },
+    compareGroupKeys: options.fromTable.compareGroupKeys,
+    compareSortKeys: options.fromTable.compareSortKeys,
+    init: () => {
+      const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`;
+      const fromRowsTableName = `from_rows_${generateSecureRandomString()}`;
+      const limitedRowsTableName = `limited_rows_${generateSecureRandomString()}`;
+      return [
+        sqlStatement`
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          VALUES
+          (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb),
+          (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb)
+        `,
+        options.fromTable.listGroups({
+          start: "start",
+          end: "end",
+          startInclusive: true,
+          endInclusive: true,
+        }).toStatement(fromGroupsTableName),
+        sqlQuery`
+          SELECT
+            "groups"."groupkey" AS "groupKey",
+            "rows"."rowidentifier" AS "rowIdentifier",
+            "rows"."rowsortkey" AS "rowSortKey",
+            "rows"."rowdata" AS "rowData"
+          FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups"
+          CROSS JOIN LATERAL (
+            ${options.fromTable.listRowsInGroup({
+              groupKey: sqlExpression`"groups"."groupkey"`,
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            })}
+          ) AS "rows"
+        `.toStatement(fromRowsTableName),
+        sqlQuery`
+          SELECT
+            "rankedRows"."groupKey" AS "groupKey",
+            "rankedRows"."rowIdentifier" AS "rowIdentifier",
+            "rankedRows"."rowSortKey" AS "rowSortKey",
+            "rankedRows"."rowData" AS "rowData"
+          FROM (
+            SELECT
+              "rows"."groupKey" AS "groupKey",
+              "rows"."rowIdentifier" AS "rowIdentifier",
+              "rows"."rowSortKey" AS "rowSortKey",
+              "rows"."rowData" AS "rowData",
+              row_number() OVER (
+                PARTITION BY "rows"."groupKey"
+                ORDER BY "rows"."rowSortKey" ASC, "rows"."rowIdentifier" ASC
+              ) AS "rank"
+            FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows"
+          ) AS "rankedRows"
+          WHERE "rankedRows"."rank" <= ${normalizedLimit}
+        `.toStatement(limitedRowsTableName),
+        sqlStatement`
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(limitedRowsTableName)}
+            UNION
+            SELECT DISTINCT
+              ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM ${quoteSqlIdentifier(limitedRowsTableName)}
+            UNION
+            SELECT
+              ${getGroupRowPath(
+                sqlExpression`"groupKey"`,
+                sqlExpression`to_jsonb("rowIdentifier"::text)`,
+              )}::jsonb[] AS "keyPath",
+              jsonb_build_object(
+                'rowSortKey', "rowSortKey",
+                'rowData', "rowData"
+              ) AS "value"
+            FROM ${quoteSqlIdentifier(limitedRowsTableName)}
+          ) AS "insertRows"
+        `,
+      ];
+    },
+    delete: () => [sqlStatement`
+      WITH RECURSIVE "pathsToDelete" AS (
+        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
+        UNION ALL
+        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
+        FROM "BulldozerStorageEngine"
+        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
+      )
+      DELETE FROM "BulldozerStorageEngine"
+      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
+    `],
+    isInitialized: () => isInitializedExpression,
+    listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
+      SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey
+      FROM "BulldozerStorageEngine" AS "groupPath"
+      WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND EXISTS (
+          SELECT 1
+          FROM "BulldozerStorageEngine" AS "groupRowsPath"
+          INNER JOIN "BulldozerStorageEngine" AS "groupRow"
+            ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath"
+          WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath"
+            AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text)
+        )
+        AND ${
+          start === "start"
+            ? sqlExpression`1 = 1`
+            : startInclusive
+              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0`
+              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0`
+        }
+        AND ${
+          end === "end"
+            ? sqlExpression`1 = 1`
+            : endInclusive
+              ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0`
+              : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0`
+        }
+    `,
+    listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery`
+      SELECT
+        ("row"."keyPath"[cardinality("row"."keyPath")] #>> '{}') AS rowIdentifier,
+        "row"."value"->'rowSortKey' AS rowSortKey,
+        "row"."value"->'rowData' AS rowData
+      FROM "BulldozerStorageEngine" AS "row"
+      WHERE "row"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[]
+        AND ${
+          start === "start"
+            ? sqlExpression`1 = 1`
+            : startInclusive
+              ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"row"."value"->'rowSortKey'`, start)} >= 0`
+              : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"row"."value"->'rowSortKey'`, start)} > 0`
+        }
+        AND ${
+          end === "end"
+            ? sqlExpression`1 = 1`
+            : endInclusive
+              ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"row"."value"->'rowSortKey'`, end)} <= 0`
+              : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"row"."value"->'rowSortKey'`, end)} < 0`
+        }
+      ORDER BY rowSortKey ASC, rowIdentifier ASC
+    ` : sqlQuery`
+      SELECT
+        "groupRows"."keyPath"[cardinality("groupRows"."keyPath") - 1] AS groupKey,
+        ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier,
+        "rows"."value"->'rowSortKey' AS rowSortKey,
+        "rows"."value"->'rowData' AS rowData
+      FROM "BulldozerStorageEngine" AS "groupRows"
+      INNER JOIN "BulldozerStorageEngine" AS "rows" ON "rows"."keyPathParent" = "groupRows"."keyPath"
+      WHERE "groupRows"."keyPathParent"[1:cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[])] = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]
+        AND cardinality("groupRows"."keyPath") = cardinality(${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[]) + 2
+        AND "groupRows"."keyPath"[cardinality("groupRows"."keyPath")] = to_jsonb('rows'::text)
+        AND ${
+          start === "start"
+            ? sqlExpression`1 = 1`
+            : startInclusive
+              ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, start)} >= 0`
+              : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, start)} > 0`
+        }
+        AND ${
+          end === "end"
+            ? sqlExpression`1 = 1`
+            : endInclusive
+              ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, end)} <= 0`
+              : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, end)} < 0`
+        }
+      ORDER BY groupKey ASC, rowSortKey ASC, rowIdentifier ASC
+    `,
+    registerRowChangeTrigger: (trigger) => {
+      const id = generateSecureRandomString();
+      triggers.set(id, trigger);
+      return { deregister: () => triggers.delete(id) };
+    },
+  };
+}
 
 export declare function declareSortTable<
   GK extends Json,

From f7f21aa1abf2fa84f2f6e956608acd8486c19f3e Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 18:22:45 -0700
Subject: [PATCH 16/40] Speed up fuzzing

---
 .../src/lib/bulldozer/db/index.fuzz.test.ts   | 219 +++++-
 .../src/lib/bulldozer/db/index.perf.test.ts   |  72 +-
 apps/backend/src/lib/bulldozer/db/index.ts    |  12 +-
 .../db/slow-stacked-mutation.sql.txt          | 706 ++++++++++++++++++
 claude/CLAUDE-KNOWLEDGE.md                    |   3 +
 5 files changed, 998 insertions(+), 14 deletions(-)
 create mode 100644 apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt

diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index a30af8c128..ff24eff961 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -1,6 +1,6 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
-import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";
 import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
@@ -36,6 +36,86 @@ type TeamBucketRow = { team: string | null, valueScaled: number, bucket: string
 type TeamFlatMappedRow = { team: string | null, kind: string, mappedValue: number };
 type TeamFlatMappedPlusRow = { team: string | null, kind: string, mappedValuePlusOne: number };
 type GroupedRows> = Map }>;
+type TraceSectionStats = {
+  count: number,
+  totalMs: number,
+  maxMs: number,
+  slowestExample: string,
+};
+type TraceBucket = {
+  totalTrackedMs: number,
+  sections: Map,
+  slowOps: Array<{ opKind: string, ms: number, detail: string }>,
+};
+
+const FUZZ_TRACE_ENABLED = (() => {
+  const env = Reflect.get(import.meta, "env");
+  const value = Reflect.get(env, "STACK_BULLDOZER_FUZZ_TRACE") ?? Reflect.get(env, "BULLDOZER_FUZZ_TRACE");
+  return value === true || value === "true" || value === "1";
+})();
+const MAX_SLOW_OPS = 20;
+const tracesByTest = new Map();
+
+function getCurrentTestNameForTrace(): string {
+  return expect.getState().currentTestName ?? "__unknown_test__";
+}
+function getTraceBucket(testName: string): TraceBucket {
+  const existing = tracesByTest.get(testName);
+  if (existing != null) return existing;
+  const created: TraceBucket = { totalTrackedMs: 0, sections: new Map(), slowOps: [] };
+  tracesByTest.set(testName, created);
+  return created;
+}
+function trimSqlForTrace(input: string): string {
+  const trimmed = input.replaceAll(/\s+/g, " ").trim();
+  if (trimmed.length <= 180) return trimmed;
+  return `${trimmed.slice(0, 177)}...`;
+}
+function callerForTrace(fallback: string): string {
+  const stack = (new Error().stack ?? "").split("\n").map((line) => line.trim());
+  const preferred = stack.find((line) =>
+    line.includes("index.fuzz.test.ts")
+    && !line.includes("callerForTrace")
+    && !line.includes("traceOperation")
+    && !line.includes("runStatements")
+    && !line.includes("readRows")
+    && !line.includes("readBoolean"),
+  );
+  if (preferred != null) return preferred;
+  return stack[3] ?? fallback;
+}
+function traceOperation(options: { opKind: "tx" | "query" | "expr", section: string, ms: number, detail: string }) {
+  if (!FUZZ_TRACE_ENABLED) return;
+  const testName = getCurrentTestNameForTrace();
+  const bucket = getTraceBucket(testName);
+  bucket.totalTrackedMs += options.ms;
+  const key = `${options.opKind}:${options.section}`;
+  const existing = bucket.sections.get(key);
+  if (existing != null) {
+    existing.count += 1;
+    existing.totalMs += options.ms;
+    if (options.ms > existing.maxMs) {
+      existing.maxMs = options.ms;
+      existing.slowestExample = options.detail;
+    }
+  } else {
+    bucket.sections.set(key, {
+      count: 1,
+      totalMs: options.ms,
+      maxMs: options.ms,
+      slowestExample: options.detail,
+    });
+  }
+  bucket.slowOps.push({
+    opKind: options.opKind,
+    ms: options.ms,
+    detail: `${options.section} :: ${options.detail}`,
+  });
+  bucket.slowOps.sort((a, b) => b.ms - a.ms);
+  if (bucket.slowOps.length > MAX_SLOW_OPS) {
+    bucket.slowOps.length = MAX_SLOW_OPS;
+  }
+}
 
 function expr(sql: string): SqlExpression {
   return { type: "expression", sql };
@@ -74,6 +154,9 @@ function groupKeyExpression(groupKey: string | null): SqlExpression {
     ? expr(`'null'::jsonb`)
     : expr(`to_jsonb(${sqlStringLiteral(groupKey)}::text)`);
 }
+function isRecord(value: unknown): value is Record {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 
 function computeTeamGroups(rows: Map): GroupedRows<{ team: string | null, value: number }> {
   const groups: GroupedRows<{ team: string | null, value: number }> = new Map();
@@ -186,18 +269,68 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   const adminSql = postgres(dbUrls.base, { onnotice: () => undefined });
   const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 });
 
-  async function runStatements(statements: SqlStatement[]) {
-    await sql.unsafe(toExecutableSqlTransaction(statements));
+  async function runStatements(statements: SqlStatement[], traceSection?: string) {
+    const txSql = toExecutableSqlTransaction(statements);
+    const startedAt = performance.now();
+    await sql.unsafe(txSql);
+    const elapsedMs = performance.now() - startedAt;
+    let rowCountDetail = "";
+    if (FUZZ_TRACE_ENABLED) {
+      const countRows = await sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM "BulldozerStorageEngine"`);
+      if (countRows.length === 0) {
+        throw new Error("expected count row for BulldozerStorageEngine");
+      }
+      const firstCountRow = countRows[0];
+      rowCountDetail = ` storageRows=${Number(firstCountRow.count)}`;
+    }
+    const detail = `statements=${statements.length} txSqlChars=${txSql.length}${rowCountDetail} first=${trimSqlForTrace(statements[0]?.sql ?? "none")}`;
+    traceOperation({
+      opKind: "tx",
+      section: traceSection ?? callerForTrace("runStatements"),
+      ms: elapsedMs,
+      detail,
+    });
   }
-  async function readBoolean(expression: SqlExpression) {
+  async function readBoolean(expression: SqlExpression, traceSection?: string) {
+    const startedAt = performance.now();
     const rows = await sql.unsafe(`SELECT (${expression.sql}) AS "value"`);
+    const elapsedMs = performance.now() - startedAt;
+    traceOperation({
+      opKind: "expr",
+      section: traceSection ?? callerForTrace("readBoolean"),
+      ms: elapsedMs,
+      detail: trimSqlForTrace(expression.sql),
+    });
     return rows[0].value === true;
   }
-  async function readRows(query: SqlQuery) {
-    return await sql.unsafe(toQueryableSqlQuery(query));
+  async function readRows(query: SqlQuery, traceSection?: string) {
+    const startedAt = performance.now();
+    const rows = await sql.unsafe(toQueryableSqlQuery(query));
+    const elapsedMs = performance.now() - startedAt;
+    traceOperation({
+      opKind: "query",
+      section: traceSection ?? callerForTrace("readRows"),
+      ms: elapsedMs,
+      detail: trimSqlForTrace(toQueryableSqlQuery(query)),
+    });
+    return rows;
   }
 
   async function assertTableMatches>(table: QueryableTable, expected: GroupedRows) {
+    const tableLabel = (() => {
+      const maybeRecord = table as unknown;
+      if (isRecord(maybeRecord)) {
+        const debugArgs = Reflect.get(maybeRecord, "debugArgs");
+        if (isRecord(debugArgs)) {
+          const tableId = Reflect.get(debugArgs, "tableId");
+          const operator = Reflect.get(debugArgs, "operator");
+          if (typeof tableId === "string" && typeof operator === "string") {
+            return `${operator}:${tableId}`;
+          }
+        }
+      }
+      return "table";
+    })();
     const expectedGroups = [...expected.values()]
       .filter((group) => group.rows.size > 0)
       .map((group) => group.groupKey)
@@ -208,7 +341,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       end: "end",
       startInclusive: true,
       endInclusive: true,
-    })))
+    }), `${tableLabel}.listGroups`))
       .map((row) => row.groupkey as string | null)
       .sort(nullableStringCompare);
 
@@ -226,7 +359,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       end: "end",
       startInclusive: true,
       endInclusive: true,
-    })))
+    }), `${tableLabel}.listRowsInGroup(all)`))
       .map((row) => ({
         groupKey: row.groupkey as string | null,
         rowIdentifier: row.rowidentifier as string,
@@ -250,7 +383,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
         end: "end",
         startInclusive: true,
         endInclusive: true,
-      })))
+      }), `${tableLabel}.listRowsInGroup(group)`))
         .map((row) => ({ rowIdentifier: row.rowidentifier as string, rowData: row.rowdata as Record }))
         .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier));
       expect(actualRows).toEqual(expectedRows);
@@ -262,7 +395,7 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
       end: "end",
       startInclusive: true,
       endInclusive: true,
-    }));
+    }), `${tableLabel}.listRowsInGroup(missing)`);
     expect(missingRows).toEqual([]);
   }
 
@@ -271,8 +404,25 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   });
 
   beforeEach(async () => {
+    const createExtensionStartedAt = performance.now();
     await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`;
+    traceOperation({
+      opKind: "query",
+      section: "beforeEach.createExtension",
+      ms: performance.now() - createExtensionStartedAt,
+      detail: "CREATE EXTENSION IF NOT EXISTS pgcrypto",
+    });
+
+    const dropTableStartedAt = performance.now();
     await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`;
+    traceOperation({
+      opKind: "query",
+      section: "beforeEach.dropTable",
+      ms: performance.now() - dropTableStartedAt,
+      detail: `DROP TABLE IF EXISTS "BulldozerStorageEngine"`,
+    });
+
+    const createTableStartedAt = performance.now();
     await sql`
       CREATE TABLE "BulldozerStorageEngine" (
         "id" UUID NOT NULL DEFAULT gen_random_uuid(),
@@ -292,13 +442,62 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
           ON DELETE CASCADE
       )
     `;
+    traceOperation({
+      opKind: "query",
+      section: "beforeEach.createTable",
+      ms: performance.now() - createTableStartedAt,
+      detail: `CREATE TABLE "BulldozerStorageEngine"`,
+    });
+
+    const createIndexStartedAt = performance.now();
     await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`;
+    traceOperation({
+      opKind: "query",
+      section: "beforeEach.createIndex",
+      ms: performance.now() - createIndexStartedAt,
+      detail: `CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx"`,
+    });
+
+    const seedRootsStartedAt = performance.now();
     await sql`
       INSERT INTO "BulldozerStorageEngine" ("keyPath", "value")
       VALUES
         (ARRAY[]::jsonb[], 'null'::jsonb),
         (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb)
     `;
+    traceOperation({
+      opKind: "query",
+      section: "beforeEach.seedRoots",
+      ms: performance.now() - seedRootsStartedAt,
+      detail: `INSERT root key paths`,
+    });
+  });
+
+  afterEach(() => {
+    if (!FUZZ_TRACE_ENABLED) return;
+    const testName = getCurrentTestNameForTrace();
+    const bucket = tracesByTest.get(testName);
+    if (bucket == null) return;
+
+    const topSections = [...bucket.sections.entries()]
+      .sort((a, b) => b[1].totalMs - a[1].totalMs)
+      .slice(0, 12);
+    const topOps = bucket.slowOps.slice(0, 12);
+
+    console.log(`\n[bulldozer-fuzz-trace] ${testName}`);
+    console.log(`[bulldozer-fuzz-trace] tracked_total_ms=${bucket.totalTrackedMs.toFixed(1)} sections=${bucket.sections.size}`);
+    for (const [sectionName, stats] of topSections) {
+      console.log(
+        `[bulldozer-fuzz-trace] section=${sectionName} count=${stats.count} total_ms=${stats.totalMs.toFixed(1)} avg_ms=${(stats.totalMs / stats.count).toFixed(2)} max_ms=${stats.maxMs.toFixed(1)} slowest="${stats.slowestExample}"`,
+      );
+    }
+    for (const op of topOps) {
+      console.log(
+        `[bulldozer-fuzz-trace] slow_op kind=${op.opKind} ms=${op.ms.toFixed(1)} detail="${op.detail}"`,
+      );
+    }
+
+    tracesByTest.delete(testName);
   });
 
   afterAll(async () => {
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index f8e41cbdec..0f76fdb3bd 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -18,7 +18,8 @@ const DEFAULT_MEASURED_OPS = 500;
 const IS_CI = (() => {
   const env = Reflect.get(import.meta, "env");
   const ci = Reflect.get(env, "CI");
-  return ci === true || ci === "true" || ci === "1";
+  const cursorAgent = Reflect.get(env, "CURSOR_AGENT");
+  return (ci === true || ci === "true" || ci === "1") && (cursorAgent !== true && cursorAgent !== 'true' && cursorAgent !== "1");
 })();
 const DEFAULT_LOAD_ROW_COUNT = IS_CI ? 200_000 : 20_000;
 const LOAD_PREFILL_MAX_MS = 30_000;
@@ -36,6 +37,7 @@ const LOAD_FILTER_TABLE_INIT_MAX_MS = 90_000;
 const LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS = 8_000;
 const LOAD_LIMIT_TABLE_INIT_MAX_MS = 90_000;
 const LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS = 8_000;
+const STACKED_MAP_PIPELINE_MUTATION_MAX_MS = 400;
 
 function getTestDbUrls(): TestDb {
   const env = Reflect.get(import.meta, "env");
@@ -330,6 +332,72 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(composed.operationsPerSecond).toBeGreaterThan(0);
   });
 
+  it("regression: stacked group-map-group mutations avoid the postgres JIT cliff", async () => {
+    const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-regression-users" });
+    const groupedByTeam = declareGroupByTable({
+      tableId: "perf-regression-users-by-team",
+      fromTable,
+      groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` },
+    });
+    const mappedLevel1 = declareMapTable({
+      tableId: "perf-regression-users-map-level-1",
+      fromTable: groupedByTeam,
+      mapper: { type: "mapper", sql: `
+        ("rowData"->'team') AS "team",
+        (("rowData"->>'value')::int + 1) AS "value",
+        (
+          CASE
+            WHEN ((("rowData"->>'value')::int + 1) % 2) = 0 THEN 'even'
+            ELSE 'odd'
+          END
+        ) AS "bucket"
+      ` },
+    });
+    const mappedLevel2 = declareMapTable({
+      tableId: "perf-regression-users-map-level-2",
+      fromTable: mappedLevel1,
+      mapper: { type: "mapper", sql: `
+        ("rowData"->'team') AS "team",
+        ("rowData"->'bucket') AS "bucket",
+        (("rowData"->>'value')::int * 3) AS "score"
+      ` },
+    });
+    const groupedByBucket = declareGroupByTable({
+      tableId: "perf-regression-users-by-bucket",
+      fromTable: mappedLevel2,
+      groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` },
+    });
+
+    await runStatements(fromTable.init());
+    await runStatements(groupedByTeam.init());
+    await runStatements(mappedLevel1.init());
+    await runStatements(mappedLevel2.init());
+    await runStatements(groupedByBucket.init());
+
+    const seedRows = [
+      ["u1", { team: "alpha", value: 5 }],
+      ["u2", { team: "beta", value: 7 }],
+      ["u3", { team: "gamma", value: 9 }],
+      ["u:4", { team: "alpha", value: 11 }],
+      ["u 5", { team: null, value: 13 }],
+    ] as const;
+    for (const [rowIdentifier, rowData] of seedRows) {
+      await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+    }
+
+    await runStatements(fromTable.setRow("u1", expr(jsonbLiteral({ team: "alpha", value: 15 }))));
+
+    const setRowMutation = await measureMs("regression stacked pipeline setRow", async () => {
+      await runStatements(fromTable.setRow("u2", expr(jsonbLiteral({ team: "beta", value: 19 }))));
+    });
+    expect(setRowMutation.elapsedMs).toBeLessThan(STACKED_MAP_PIPELINE_MUTATION_MAX_MS);
+
+    const deleteMutation = await measureMs("regression stacked pipeline deleteRow", async () => {
+      await runStatements(fromTable.deleteRow("u3"));
+    });
+    expect(deleteMutation.elapsedMs).toBeLessThan(STACKED_MAP_PIPELINE_MUTATION_MAX_MS);
+  });
+
   it("load test: prefilled stored table with hundreds of thousands of rows stays functional and fast", async () => {
     const loadRowCount = DEFAULT_LOAD_ROW_COUNT;
     const tableId = "load-prefilled-users";
@@ -641,7 +709,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       endInclusive: true,
     }));
     expect(limitedDeltaRows).toHaveLength(1);
-    expect(limitedDeltaRows[0]?.rowidentifier).toBe("seed-100000");
+    expect(limitedDeltaRows[0].rowidentifier).toBe("seed-100000");
 
     const bulkDelete = await measureMs("load full table delete", async () => {
       await runStatements(table.delete());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index bf9b7640a3..cc2b950160 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -850,7 +850,6 @@ export function declareMapTable<
     `,
   });
 
-
   return {
     tableId: options.tableId,
     inputTables: [options.fromTable],
@@ -888,7 +887,6 @@ export function declareMapTable<
         WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
       )
     `,
-
     ...pick(nestedFlatMapTable, [
       "compareGroupKeys",
       "compareSortKeys",
@@ -1362,6 +1360,14 @@ export function declareLimitTable<
   };
 }
 
+export declare function declareConcatTable<
+  GK extends Json,
+  RD extends RowData,
+>(options: {
+  tableId: TableId,
+  tables: Table[],
+}): Table;
+
 export declare function declareSortTable<
   GK extends Json,
   OldSK extends Json,
@@ -1395,6 +1401,8 @@ export function toExecutableSqlTransaction(statements: SqlStatement[]): string {
   return deindent`
     BEGIN;
 
+    SET LOCAL jit = off;
+
     SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID});
 
     WITH __dummy_statement_1__ AS (SELECT 1),
diff --git a/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt b/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt
new file mode 100644
index 0000000000..c9f7ada6a4
--- /dev/null
+++ b/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt
@@ -0,0 +1,706 @@
+BEGIN;
+
+SET LOCAL jit = off;
+
+SELECT pg_advisory_xact_lock(7857391);
+
+WITH __dummy_statement_1__ AS (SELECT 1),
+"old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer" AS (
+  
+            SELECT "value"->'rowData' AS "oldRowData"
+            FROM "BulldozerStorageEngine"
+            WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-101"'::jsonb, '"storage"'::jsonb, '"rows"'::jsonb, '"u2"'::jsonb]::jsonb[]
+          
+),
+"upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr" AS (
+  
+            INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+            VALUES (
+              gen_random_uuid(),
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-101"'::jsonb, '"storage"'::jsonb, '"rows"'::jsonb, '"u2"'::jsonb]::jsonb[],
+              
+          jsonb_build_object(
+            'rowData', '{"team":"beta","value":21}'::jsonb::jsonb
+          )
+        ::jsonb
+            )
+            ON CONFLICT ("keyPath") DO UPDATE
+            SET "value" = 
+          jsonb_build_object(
+            'rowData', '{"team":"beta","value":21}'::jsonb::jsonb
+          )
+        ::jsonb
+            RETURNING "value"->'rowData' AS "newRowData"
+          
+),
+"changes_rpbs2y486ebg9ae3n089fya0018rngx9228gz5yjpwjdg" AS (
+  
+            SELECT
+              'null'::jsonb AS "groupKey",
+              'u2'::text AS "rowIdentifier",
+              'null'::jsonb AS "oldRowSortKey",
+              'null'::jsonb AS "newRowSortKey",
+              "old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer"."oldRowData" AS "oldRowData",
+              "upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr"."newRowData" AS "newRowData"
+            FROM "upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr"
+            LEFT JOIN "old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer" ON true
+          
+),
+"mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS (
+  
+          SELECT
+            "changes"."rowIdentifier" AS "rowIdentifier",
+            "changes"."oldRowData" AS "oldRowData",
+            "changes"."newRowData" AS "newRowData",
+            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
+            "oldGroup"."groupKey" AS "oldGroupKey",
+            "newGroup"."groupKey" AS "newGroupKey"
+          FROM "changes_rpbs2y486ebg9ae3n089fya0018rngx9228gz5yjpwjdg" AS "changes"
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."groupKey"
+            FROM (
+              SELECT "rowData"->'team' AS "groupKey"
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."oldRowData" AS "rowData"
+              ) AS "groupByInput"
+            ) AS "mapped"
+          ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."groupKey"
+            FROM (
+              SELECT "rowData"->'team' AS "groupKey"
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."newRowData" AS "rowData"
+              ) AS "groupByInput"
+            ) AS "mapped"
+          ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
+          WHERE 
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
+      )
+    
+        
+),
+"unnamed_statement_6bw6129p" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey"]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
+            WHERE "hasNewRow"
+            UNION
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
+            WHERE "hasNewRow"
+          ) AS "insertRows"
+          ON CONFLICT ("keyPath") DO NOTHING
+        
+),
+"unnamed_statement_3bp4fqz7" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "target"
+          USING "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "changes"
+          WHERE "changes"."hasOldRow"
+            AND "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
+        
+),
+"unnamed_statement_je3f1c7g" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
+            jsonb_build_object('rowData', "newRowData")
+          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
+          WHERE "hasNewRow"
+          ON CONFLICT ("keyPath") DO UPDATE
+          SET "value" = EXCLUDED."value"
+        
+),
+"unnamed_statement_rxdy9p42" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+          USING "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "changes"
+          WHERE "changes"."hasOldRow"
+            AND "staleGroupPath"."keyPath" IN (
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[],
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey"]::jsonb[]
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "BulldozerStorageEngine" AS "groupRow"
+              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[]
+                AND NOT EXISTS (
+                  SELECT 1
+                  FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "deletingRow"
+                  WHERE "deletingRow"."hasOldRow"
+                    AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
+                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."oldGroupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
+                )
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "insertingRow"
+              WHERE "insertingRow"."hasNewRow"
+                AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
+            )
+        
+),
+"grouped_changes_w9bytqkqjvr4kyczj0nahg3v1qn45y4mjrcgzky0d1zxr" AS (
+  
+          SELECT
+            "oldGroupKey" AS "groupKey",
+            "rowIdentifier" AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            "oldRowData" AS "oldRowData",
+            CASE
+              WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData"
+              ELSE 'null'::jsonb
+            END AS "newRowData"
+          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
+          WHERE "hasOldRow"
+          UNION ALL
+          SELECT
+            "newGroupKey" AS "groupKey",
+            "rowIdentifier" AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            'null'::jsonb AS "oldRowData",
+            "newRowData" AS "newRowData"
+          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
+          WHERE "hasNewRow"
+            AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey")
+        
+),
+"mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            "changes"."rowIdentifier" AS "sourceRowIdentifier",
+            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
+            "oldMapped"."rows" AS "oldMappedRows",
+            "newMapped"."rows" AS "newMappedRows"
+          FROM "grouped_changes_w9bytqkqjvr4kyczj0nahg3v1qn45y4mjrcgzky0d1zxr" AS "changes"
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."rows" AS "rows"
+            FROM (
+              SELECT 
+        jsonb_build_array(
+          COALESCE(
+            (
+              SELECT to_jsonb("mapped")
+              FROM (
+                SELECT ("rowData"->'team') AS "team", (("rowData"->>'value')::int + 10) AS "valuePlusTen"
+              ) AS "mapped"
+            ),
+            'null'::jsonb
+          )
+        ) AS "rows"
+      
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."oldRowData" AS "rowData"
+              ) AS "mapperInput"
+            ) AS "mapped"
+          ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."rows" AS "rows"
+            FROM (
+              SELECT 
+        jsonb_build_array(
+          COALESCE(
+            (
+              SELECT to_jsonb("mapped")
+              FROM (
+                SELECT ("rowData"->'team') AS "team", (("rowData"->>'value')::int + 10) AS "valuePlusTen"
+              ) AS "mapped"
+            ),
+            'null'::jsonb
+          )
+        ) AS "rows"
+      
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."newRowData" AS "rowData"
+              ) AS "mapperInput"
+            ) AS "mapped"
+          ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
+          WHERE 
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
+      )
+    
+        
+),
+"old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            "flatRow"."rowData" AS "rowData"
+          FROM "mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS "changes"
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN "changes"."hasOldRow" THEN (
+                CASE
+                  WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows"
+                  ELSE '[]'::jsonb
+                END
+              )
+              ELSE '[]'::jsonb
+            END
+          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+        
+),
+"new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            "flatRow"."rowData" AS "rowData"
+          FROM "mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS "changes"
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN "changes"."hasNewRow" THEN (
+                CASE
+                  WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows"
+                  ELSE '[]'::jsonb
+                END
+              )
+              ELSE '[]'::jsonb
+            END
+          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+        
+),
+"unnamed_statement_733m8521" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey"]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
+            UNION
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
+          ) AS "insertRows"
+          ON CONFLICT ("keyPath") DO NOTHING
+        
+),
+"unnamed_statement_5fsnnkka" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "target"
+          USING "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "changes"
+          WHERE "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
+        
+),
+"unnamed_statement_93gncc42" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
+            jsonb_build_object('rowData', "rowData")
+          FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
+          ON CONFLICT ("keyPath") DO UPDATE
+          SET "value" = EXCLUDED."value"
+        
+),
+"unnamed_statement_fpzbmw3j" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+          USING "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "changes"
+          WHERE "staleGroupPath"."keyPath" IN (
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[],
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey"]::jsonb[]
+          )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "BulldozerStorageEngine" AS "groupRow"
+              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[]
+                AND NOT EXISTS (
+                  SELECT 1
+                  FROM "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "deletingRow"
+                  WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."groupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
+                )
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS "insertingRow"
+              WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+            )
+        
+),
+"flat_map_changes_ag6q3tne5fy9c7mbbr325v6hy6g60scj9rrntqx598s30" AS (
+  
+          SELECT
+            COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
+            COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
+            CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
+          FROM "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "oldRows"
+          FULL OUTER JOIN "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS "newRows"
+            ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
+            AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
+          WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
+        
+),
+"mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            "changes"."rowIdentifier" AS "sourceRowIdentifier",
+            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
+            "oldMapped"."rows" AS "oldMappedRows",
+            "newMapped"."rows" AS "newMappedRows"
+          FROM "flat_map_changes_ag6q3tne5fy9c7mbbr325v6hy6g60scj9rrntqx598s30" AS "changes"
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."rows" AS "rows"
+            FROM (
+              SELECT 
+        jsonb_build_array(
+          COALESCE(
+            (
+              SELECT to_jsonb("mapped")
+              FROM (
+                SELECT ("rowData"->'team') AS "team", (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", (CASE WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' ELSE 'low' END) AS "bucket"
+              ) AS "mapped"
+            ),
+            'null'::jsonb
+          )
+        ) AS "rows"
+      
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."oldRowData" AS "rowData"
+              ) AS "mapperInput"
+            ) AS "mapped"
+          ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."rows" AS "rows"
+            FROM (
+              SELECT 
+        jsonb_build_array(
+          COALESCE(
+            (
+              SELECT to_jsonb("mapped")
+              FROM (
+                SELECT ("rowData"->'team') AS "team", (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", (CASE WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' ELSE 'low' END) AS "bucket"
+              ) AS "mapped"
+            ),
+            'null'::jsonb
+          )
+        ) AS "rows"
+      
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."newRowData" AS "rowData"
+              ) AS "mapperInput"
+            ) AS "mapped"
+          ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
+          WHERE 
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
+      )
+    
+        
+),
+"old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            "flatRow"."rowData" AS "rowData"
+          FROM "mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS "changes"
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN "changes"."hasOldRow" THEN (
+                CASE
+                  WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows"
+                  ELSE '[]'::jsonb
+                END
+              )
+              ELSE '[]'::jsonb
+            END
+          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+        
+),
+"new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS (
+  
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
+            "flatRow"."rowData" AS "rowData"
+          FROM "mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS "changes"
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN "changes"."hasNewRow" THEN (
+                CASE
+                  WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows"
+                  ELSE '[]'::jsonb
+                END
+              )
+              ELSE '[]'::jsonb
+            END
+          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
+        
+),
+"unnamed_statement_6wwsxhrt" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey"]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
+            UNION
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
+          ) AS "insertRows"
+          ON CONFLICT ("keyPath") DO NOTHING
+        
+),
+"unnamed_statement_rmr146nf" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "target"
+          USING "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "changes"
+          WHERE "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
+        
+),
+"unnamed_statement_d4mggz1p" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
+            jsonb_build_object('rowData', "rowData")
+          FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
+          ON CONFLICT ("keyPath") DO UPDATE
+          SET "value" = EXCLUDED."value"
+        
+),
+"unnamed_statement_q6c1gdbq" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+          USING "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "changes"
+          WHERE "staleGroupPath"."keyPath" IN (
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[],
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey"]::jsonb[]
+          )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "BulldozerStorageEngine" AS "groupRow"
+              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[]
+                AND NOT EXISTS (
+                  SELECT 1
+                  FROM "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "deletingRow"
+                  WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."groupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
+                )
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS "insertingRow"
+              WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
+            )
+        
+),
+"flat_map_changes_hfe1g0gz05jxzx6j8ma5dv2n1fegvheyr3hrpp5rtgx78" AS (
+  
+          SELECT
+            COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
+            COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
+            CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
+          FROM "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "oldRows"
+          FULL OUTER JOIN "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS "newRows"
+            ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
+            AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
+          WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
+        
+),
+"mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS (
+  
+          SELECT
+            "changes"."rowIdentifier" AS "rowIdentifier",
+            "changes"."oldRowData" AS "oldRowData",
+            "changes"."newRowData" AS "newRowData",
+            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
+            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
+            "oldGroup"."groupKey" AS "oldGroupKey",
+            "newGroup"."groupKey" AS "newGroupKey"
+          FROM "flat_map_changes_hfe1g0gz05jxzx6j8ma5dv2n1fegvheyr3hrpp5rtgx78" AS "changes"
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."groupKey"
+            FROM (
+              SELECT "rowData"->'bucket' AS "groupKey"
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."oldRowData" AS "rowData"
+              ) AS "groupByInput"
+            ) AS "mapped"
+          ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
+          LEFT JOIN LATERAL (
+            SELECT "mapped"."groupKey"
+            FROM (
+              SELECT "rowData"->'bucket' AS "groupKey"
+              FROM (
+                SELECT
+                  "changes"."rowIdentifier" AS "rowIdentifier",
+                  "changes"."newRowData" AS "rowData"
+              ) AS "groupByInput"
+            ) AS "mapped"
+          ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
+          WHERE 
+      EXISTS (
+        SELECT 1 FROM "BulldozerStorageEngine"
+        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
+      )
+    
+        
+),
+"unnamed_statement_br2pqz8w" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            "insertRows"."keyPath",
+            "insertRows"."value"
+          FROM (
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey"]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
+            WHERE "hasNewRow"
+            UNION
+            SELECT DISTINCT
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
+              'null'::jsonb AS "value"
+            FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
+            WHERE "hasNewRow"
+          ) AS "insertRows"
+          ON CONFLICT ("keyPath") DO NOTHING
+        
+),
+"unnamed_statement_zvcwftv3" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "target"
+          USING "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "changes"
+          WHERE "changes"."hasOldRow"
+            AND "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
+        
+),
+"unnamed_statement_e5kztjcm" AS (
+  
+          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+          SELECT
+            gen_random_uuid(),
+            ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
+            jsonb_build_object('rowData', "newRowData")
+          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
+          WHERE "hasNewRow"
+          ON CONFLICT ("keyPath") DO UPDATE
+          SET "value" = EXCLUDED."value"
+        
+),
+"unnamed_statement_fmaqky9q" AS (
+  
+          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
+          USING "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "changes"
+          WHERE "changes"."hasOldRow"
+            AND "staleGroupPath"."keyPath" IN (
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[],
+              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey"]::jsonb[]
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "BulldozerStorageEngine" AS "groupRow"
+              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[]
+                AND NOT EXISTS (
+                  SELECT 1
+                  FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "deletingRow"
+                  WHERE "deletingRow"."hasOldRow"
+                    AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
+                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."oldGroupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
+                )
+            )
+            AND NOT EXISTS (
+              SELECT 1
+              FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "insertingRow"
+              WHERE "insertingRow"."hasNewRow"
+                AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
+            )
+        
+),
+"grouped_changes_e4p857w4kzj7vda37xcp61a730wq7472960nn1p1xz03g" AS (
+  
+          SELECT
+            "oldGroupKey" AS "groupKey",
+            "rowIdentifier" AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            "oldRowData" AS "oldRowData",
+            CASE
+              WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData"
+              ELSE 'null'::jsonb
+            END AS "newRowData"
+          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
+          WHERE "hasOldRow"
+          UNION ALL
+          SELECT
+            "newGroupKey" AS "groupKey",
+            "rowIdentifier" AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            'null'::jsonb AS "oldRowData",
+            "newRowData" AS "newRowData"
+          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
+          WHERE "hasNewRow"
+            AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey")
+        
+),
+__dummy_statement_2__ AS (SELECT 1)
+SELECT 1;
+
+COMMIT;
\ No newline at end of file
diff --git a/claude/CLAUDE-KNOWLEDGE.md b/claude/CLAUDE-KNOWLEDGE.md
index f682b9ef69..29d2cafdb5 100644
--- a/claude/CLAUDE-KNOWLEDGE.md
+++ b/claude/CLAUDE-KNOWLEDGE.md
@@ -79,6 +79,9 @@ A: Run lint from `apps/dashboard` directly (for example `pnpm lint -- "src/app/(
 Q: How should unsubscribe-link e2e tests avoid breakage from email theme/layout changes?
 A: In `apps/e2e/tests/backend/endpoints/api/v1/unsubscribe-link.test.ts`, avoid snapshotting the entire rendered HTML for transactional emails; assert stable behavior instead (email content present and `/api/v1/emails/unsubscribe-link` absent) so cosmetic wrapper/style changes do not fail the test.
 
+Q: Why is the JIT disabled for Bulldozer DB mutations with only a few rows?
+A: PostgreSQL JIT can dominate runtime for Bulldozer's giant single-statement CTE transactions. In a `group -> map -> map -> group` mutation with only 31 SQL statements and ~8 source rows, `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, VERBOSE)` showed ~1.4ms planning, ~1598.9ms execution, and ~1597.5ms of JIT time (`Optimization` ~836ms, `Emission` ~740ms) while the actual plan nodes were sub-millisecond. Disabling JIT locally for Bulldozer transactions with `SET LOCAL jit = off;` in `toExecutableSqlTransaction()` dropped the same query to ~0.63ms execution and brought the stacked fuzz case from ~41s to ~0.34s.
+
 Q: How should dashboard pages update project config values?
 A: Do not call `project.updateConfig(...)` directly from dashboard pages; lint enforces using `useUpdateConfig()` from `apps/dashboard/src/lib/config-update.tsx` so pushable-config confirmation flows are handled consistently.
 

From 90f61ecf576ee1ee34c44281d4ab4b79a9c56051 Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 19:23:27 -0700
Subject: [PATCH 17/40] Concat table

---
 apps/backend/scripts/run-bulldozer-studio.ts  | 412 ++++++++--
 .../src/lib/bulldozer/db/example-schema.ts    |   7 +-
 .../src/lib/bulldozer/db/index.fuzz.test.ts   | 121 ++-
 .../src/lib/bulldozer/db/index.perf.test.ts   | 107 ++-
 .../src/lib/bulldozer/db/index.test.ts        | 201 ++++-
 apps/backend/src/lib/bulldozer/db/index.ts    | 173 ++++-
 .../db/slow-stacked-mutation.sql.txt          | 706 ------------------
 7 files changed, 949 insertions(+), 778 deletions(-)
 delete mode 100644 apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt

diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts
index d2dc17301c..cbca3516dc 100644
--- a/apps/backend/scripts/run-bulldozer-studio.ts
+++ b/apps/backend/scripts/run-bulldozer-studio.ts
@@ -456,6 +456,8 @@ function getStudioPageHtml(): string {
       grid-template-rows: auto auto 1fr auto;
       gap: 6px;
       transition: border-color 0.15s ease;
+      cursor: grab;
+      user-select: none;
     }
     .node:hover {
       border-color: var(--accent);
@@ -464,6 +466,11 @@ function getStudioPageHtml(): string {
       border-color: var(--accent);
       box-shadow: inset 0 0 0 1px var(--accent);
     }
+    .node.dragging {
+      cursor: grabbing;
+      z-index: 4;
+      box-shadow: 0 10px 28px rgba(0, 0, 0, 0.22), inset 0 0 0 1px var(--accent);
+    }
     .node-type {
       font-size: 28px;
       font-weight: 800;
@@ -493,6 +500,9 @@ function getStudioPageHtml(): string {
     .node-type.limit {
       color: color-mix(in srgb, var(--filter) 75%, var(--text));
     }
+    .node-type.concat {
+      color: color-mix(in srgb, var(--accent) 60%, var(--filter));
+    }
     .node-name {
       font-size: 13px;
       font-weight: 700;
@@ -512,6 +522,9 @@ function getStudioPageHtml(): string {
     .node-name.limit {
       color: color-mix(in srgb, var(--filter) 85%, var(--text));
     }
+    .node-name.concat {
+      color: color-mix(in srgb, var(--accent) 55%, var(--filter));
+    }
     .node-meta {
       font-size: 11px;
       color: var(--muted);
@@ -729,8 +742,32 @@ function getStudioPageHtml(): string {
     const COLUMN_GAP_X = 320;
     const SCENE_MARGIN = 40;
     const THEME_STORAGE_KEY = "bulldozer-studio-theme";
+    const NODE_POSITIONS_STORAGE_KEY = "bulldozer-studio-node-positions-v1";
     const VERSION_POLL_INTERVAL_MS = 1200;
 
+    function loadStoredNodePositions() {
+      try {
+        const raw = window.localStorage.getItem(NODE_POSITIONS_STORAGE_KEY);
+        if (!raw) return {};
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+        const result = {};
+        for (const [key, value] of Object.entries(parsed)) {
+          if (!value || typeof value !== "object" || Array.isArray(value)) continue;
+          const x = Number(value.x);
+          const y = Number(value.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
+          result[key] = { x, y };
+        }
+        return result;
+      } catch (error) {
+        return {};
+      }
+    }
+    function persistNodePositions() {
+      window.localStorage.setItem(NODE_POSITIONS_STORAGE_KEY, JSON.stringify(state.manualNodePositions));
+    }
+
     const state = {
       mode: "table",
       schema: null,
@@ -747,12 +784,19 @@ function getStudioPageHtml(): string {
         y: 24,
         scale: 1,
       },
+      manualNodePositions: loadStoredNodePositions(),
       dragging: {
         active: false,
+        kind: null,
         startX: 0,
         startY: 0,
         startOffsetX: 0,
         startOffsetY: 0,
+        nodeId: null,
+        nodeStartX: 0,
+        nodeStartY: 0,
+        moved: false,
+        suppressClickTableId: null,
       },
     };
 
@@ -820,6 +864,10 @@ function getStudioPageHtml(): string {
       if (a === b) return 0;
       return a < b ? -1 : 1;
     }
+    function compareNumbers(a, b) {
+      if (a === b) return 0;
+      return a < b ? -1 : 1;
+    }
 
     async function fetchJson(path, init) {
       const response = await fetch(path, init);
@@ -856,12 +904,58 @@ function getStudioPageHtml(): string {
       visiting.delete(tableId);
       return depth;
     }
+    function getAverageOrderValue(ids, orderMap, fallback) {
+      const values = ids
+        .map((id) => orderMap.get(id))
+        .filter((value) => typeof value === "number");
+      if (values.length === 0) return fallback;
+      return values.reduce((sum, value) => sum + value, 0) / values.length;
+    }
+    function getNodePosition(tableId) {
+      const stored = state.manualNodePositions[tableId];
+      if (!stored || typeof stored !== "object") return null;
+      const x = Number(stored.x);
+      const y = Number(stored.y);
+      if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
+      return {
+        x: Math.max(SCENE_MARGIN / 2, x),
+        y: Math.max(SCENE_MARGIN / 2, y),
+      };
+    }
+    function pruneNodePositions(tables) {
+      const validIds = new Set(tables.map((table) => String(table.id)));
+      let changed = false;
+      const next = {};
+      for (const [tableId, value] of Object.entries(state.manualNodePositions)) {
+        if (!validIds.has(tableId)) {
+          changed = true;
+          continue;
+        }
+        next[tableId] = value;
+      }
+      if (!changed) return;
+      state.manualNodePositions = next;
+      persistNodePositions();
+    }
 
     function layoutGraph(tables) {
       const tableMap = new Map(tables.map((table) => [table.id, table]));
+      const reverseDependencies = new Map();
       const depthCache = new Map();
       const byDepth = new Map();
 
+      for (const table of tables) {
+        reverseDependencies.set(table.id, []);
+      }
+      for (const table of tables) {
+        const dependencies = Array.isArray(table.dependencies) ? table.dependencies : [];
+        for (const dependencyId of dependencies) {
+          const existing = reverseDependencies.get(dependencyId) ?? [];
+          existing.push(table.id);
+          reverseDependencies.set(dependencyId, existing);
+        }
+      }
+
       for (const table of tables) {
         const depth = computeDepth(table.id, tableMap, depthCache, new Set());
         if (!byDepth.has(depth)) byDepth.set(depth, []);
@@ -873,18 +967,66 @@ function getStudioPageHtml(): string {
       let sceneWidth = 600;
       let sceneHeight = 600;
 
+      for (const depth of depths) {
+        const row = byDepth.get(depth);
+        row.sort((a, b) => compareStrings(String(a.name), String(b.name)));
+      }
+
+      for (let iteration = 0; iteration < 6; iteration++) {
+        const orderMap = new Map();
+        for (const depth of depths) {
+          const row = byDepth.get(depth) ?? [];
+          for (let i = 0; i < row.length; i++) {
+            orderMap.set(row[i].id, i);
+          }
+        }
+        for (let depthIndex = 1; depthIndex < depths.length; depthIndex++) {
+          const depth = depths[depthIndex];
+          const row = byDepth.get(depth) ?? [];
+          row.sort((a, b) => {
+            const aDeps = Array.isArray(a.dependencies) ? a.dependencies : [];
+            const bDeps = Array.isArray(b.dependencies) ? b.dependencies : [];
+            const aScore = getAverageOrderValue(aDeps, orderMap, Number.MAX_SAFE_INTEGER / 4);
+            const bScore = getAverageOrderValue(bDeps, orderMap, Number.MAX_SAFE_INTEGER / 4);
+            return compareNumbers(aScore, bScore) || compareStrings(String(a.name), String(b.name));
+          });
+        }
+
+        orderMap.clear();
+        for (const depth of depths) {
+          const row = byDepth.get(depth) ?? [];
+          for (let i = 0; i < row.length; i++) {
+            orderMap.set(row[i].id, i);
+          }
+        }
+        for (let depthIndex = depths.length - 2; depthIndex >= 0; depthIndex--) {
+          const depth = depths[depthIndex];
+          const row = byDepth.get(depth) ?? [];
+          row.sort((a, b) => {
+            const aDependents = reverseDependencies.get(a.id) ?? [];
+            const bDependents = reverseDependencies.get(b.id) ?? [];
+            const aScore = getAverageOrderValue(aDependents, orderMap, Number.MAX_SAFE_INTEGER / 4);
+            const bScore = getAverageOrderValue(bDependents, orderMap, Number.MAX_SAFE_INTEGER / 4);
+            return compareNumbers(aScore, bScore) || compareStrings(String(a.name), String(b.name));
+          });
+        }
+      }
+
       for (let depthIndex = 0; depthIndex < depths.length; depthIndex++) {
         const depth = depths[depthIndex];
         const row = byDepth.get(depth);
-        row.sort((a, b) => compareStrings(String(a.name), String(b.name)));
         const totalWidth = row.length * NODE_WIDTH + (row.length - 1) * COLUMN_GAP_X;
         const startX = SCENE_MARGIN + Math.max(0, (900 - totalWidth) / 2);
         const y = SCENE_MARGIN + depthIndex * LEVEL_GAP_Y;
         for (let i = 0; i < row.length; i++) {
-          const x = startX + i * (NODE_WIDTH + COLUMN_GAP_X);
-          positions.set(row[i].id, { x, y });
+          const defaultX = startX + i * (NODE_WIDTH + COLUMN_GAP_X);
+          const defaultY = y;
+          const manualPosition = getNodePosition(row[i].id);
+          const x = manualPosition ? manualPosition.x : defaultX;
+          const finalY = manualPosition ? manualPosition.y : defaultY;
+          positions.set(row[i].id, { x, y: finalY });
           sceneWidth = Math.max(sceneWidth, x + NODE_WIDTH + SCENE_MARGIN);
-          sceneHeight = Math.max(sceneHeight, y + NODE_HEIGHT + SCENE_MARGIN);
+          sceneHeight = Math.max(sceneHeight, finalY + NODE_HEIGHT + SCENE_MARGIN);
         }
       }
 
@@ -892,8 +1034,154 @@ function getStudioPageHtml(): string {
         positions,
         sceneWidth,
         sceneHeight,
+        depthById: depthCache,
       };
     }
+    function syncSceneDimensions() {
+      if (!state.graphLayout) return;
+      graphScene.style.width = state.graphLayout.sceneWidth + "px";
+      graphScene.style.height = state.graphLayout.sceneHeight + "px";
+      graphEdges.setAttribute("width", String(state.graphLayout.sceneWidth));
+      graphEdges.setAttribute("height", String(state.graphLayout.sceneHeight));
+      graphEdges.setAttribute("viewBox", "0 0 " + state.graphLayout.sceneWidth + " " + state.graphLayout.sceneHeight);
+      graphNodes.style.width = state.graphLayout.sceneWidth + "px";
+      graphNodes.style.height = state.graphLayout.sceneHeight + "px";
+    }
+    function buildGraphEdges(tables, positions, depthById) {
+      const edges = [];
+      const outgoingByNode = new Map();
+      const incomingByNode = new Map();
+      for (const table of tables) {
+        const to = positions.get(table.id);
+        if (!to) continue;
+        const dependencies = Array.isArray(table.dependencies) ? table.dependencies : [];
+        for (const dependencyId of dependencies) {
+          const from = positions.get(dependencyId);
+          if (!from) continue;
+          const edge = {
+            id: dependencyId + "->" + table.id,
+            fromId: dependencyId,
+            toId: table.id,
+            from,
+            to,
+            depthFrom: depthById.get(dependencyId) ?? 0,
+            depthTo: depthById.get(table.id) ?? 0,
+            sourceSlotIndex: 0,
+            sourceSlotCount: 1,
+            targetSlotIndex: 0,
+            targetSlotCount: 1,
+            laneOffset: 0,
+          };
+          edges.push(edge);
+          const outgoing = outgoingByNode.get(dependencyId) ?? [];
+          outgoing.push(edge);
+          outgoingByNode.set(dependencyId, outgoing);
+          const incoming = incomingByNode.get(table.id) ?? [];
+          incoming.push(edge);
+          incomingByNode.set(table.id, incoming);
+        }
+      }
+
+      for (const [nodeId, nodeEdges] of outgoingByNode.entries()) {
+        nodeEdges.sort((a, b) => compareNumbers(a.to.x, b.to.x) || compareStrings(a.toId, b.toId));
+        for (let i = 0; i < nodeEdges.length; i++) {
+          nodeEdges[i].sourceSlotIndex = i;
+          nodeEdges[i].sourceSlotCount = nodeEdges.length;
+        }
+      }
+      for (const [nodeId, nodeEdges] of incomingByNode.entries()) {
+        nodeEdges.sort((a, b) => compareNumbers(a.from.x, b.from.x) || compareStrings(a.fromId, b.fromId));
+        for (let i = 0; i < nodeEdges.length; i++) {
+          nodeEdges[i].targetSlotIndex = i;
+          nodeEdges[i].targetSlotCount = nodeEdges.length;
+        }
+      }
+
+      const edgesByDepthSpan = new Map();
+      for (const edge of edges) {
+        const bucketKey = edge.depthFrom + "->" + edge.depthTo;
+        const bucket = edgesByDepthSpan.get(bucketKey) ?? [];
+        bucket.push(edge);
+        edgesByDepthSpan.set(bucketKey, bucket);
+      }
+      for (const bucket of edgesByDepthSpan.values()) {
+        bucket.sort((a, b) => {
+          const aCenter = (a.from.x + a.to.x) / 2;
+          const bCenter = (b.from.x + b.to.x) / 2;
+          return compareNumbers(aCenter, bCenter) || compareStrings(a.id, b.id);
+        });
+        for (let i = 0; i < bucket.length; i++) {
+          const centeredIndex = i - (bucket.length - 1) / 2;
+          bucket[i].laneOffset = Math.max(-4, Math.min(4, centeredIndex)) * 18;
+        }
+      }
+
+      return edges;
+    }
+    function getEdgeAnchorX(position, slotIndex, slotCount) {
+      if (slotCount <= 1) return position.x + NODE_WIDTH / 2;
+      return position.x + ((slotIndex + 1) / (slotCount + 1)) * NODE_WIDTH;
+    }
+    function renderGraphEdges(tables) {
+      graphEdges.innerHTML = "";
+      if (!state.graphLayout) return;
+      const positions = state.graphLayout.positions;
+
+      const defs = document.createElementNS("http://www.w3.org/2000/svg", "defs");
+      const marker = document.createElementNS("http://www.w3.org/2000/svg", "marker");
+      marker.setAttribute("id", "arrow");
+      marker.setAttribute("viewBox", "0 0 10 10");
+      marker.setAttribute("refX", "9");
+      marker.setAttribute("refY", "5");
+      marker.setAttribute("markerWidth", "7");
+      marker.setAttribute("markerHeight", "7");
+      marker.setAttribute("orient", "auto");
+      const markerPath = document.createElementNS("http://www.w3.org/2000/svg", "path");
+      markerPath.setAttribute("d", "M 0 0 L 10 5 L 0 10 z");
+      markerPath.setAttribute("fill", "var(--accent)");
+      marker.appendChild(markerPath);
+      defs.appendChild(marker);
+      graphEdges.appendChild(defs);
+
+      const edges = buildGraphEdges(tables, positions, state.graphLayout.depthById ?? new Map());
+      for (const edge of edges) {
+        const startX = getEdgeAnchorX(edge.from, edge.sourceSlotIndex, edge.sourceSlotCount);
+        const startY = edge.from.y + NODE_HEIGHT;
+        const endX = getEdgeAnchorX(edge.to, edge.targetSlotIndex, edge.targetSlotCount);
+        const endY = edge.to.y;
+        const laneY = Math.min(endY - 20, Math.max(startY + 20, (startY + endY) / 2 + edge.laneOffset));
+        const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
+        path.setAttribute("d", "M " + startX + " " + startY + " C " + startX + " " + laneY + ", " + endX + " " + laneY + ", " + endX + " " + endY);
+        path.setAttribute("fill", "none");
+        path.setAttribute("stroke", "var(--accent)");
+        path.setAttribute("stroke-width", "1.7");
+        path.setAttribute("stroke-linecap", "round");
+        path.setAttribute("stroke-linejoin", "round");
+        path.setAttribute("marker-end", "url(#arrow)");
+        graphEdges.appendChild(path);
+      }
+    }
+    function syncNodePositions() {
+      if (!state.graphLayout) return;
+      const positions = state.graphLayout.positions;
+      for (const node of graphNodes.querySelectorAll(".node")) {
+        const tableId = node.getAttribute("data-table-id");
+        if (!tableId) continue;
+        const position = positions.get(tableId);
+        if (!position) continue;
+        node.style.left = position.x + "px";
+        node.style.top = position.y + "px";
+      }
+    }
+    function relayoutGraph() {
+      if (!state.schema || !Array.isArray(state.schema.tables)) return;
+      pruneNodePositions(state.schema.tables);
+      state.graphLayout = layoutGraph(state.schema.tables);
+      syncSceneDimensions();
+      renderGraphEdges(state.schema.tables);
+      syncNodePositions();
+      updateSceneTransform();
+    }
 
     function updateSceneTransform() {
       graphScene.style.transform = "translate(" + state.viewport.x + "px, " + state.viewport.y + "px) scale(" + state.viewport.scale + ")";
@@ -927,68 +1215,17 @@ function getStudioPageHtml(): string {
       if (!state.schema || !Array.isArray(state.schema.tables)) return;
 
       const tables = state.schema.tables;
-      state.graphLayout = layoutGraph(tables);
-      const positions = state.graphLayout.positions;
-      graphScene.style.width = state.graphLayout.sceneWidth + "px";
-      graphScene.style.height = state.graphLayout.sceneHeight + "px";
-      graphEdges.setAttribute("width", String(state.graphLayout.sceneWidth));
-      graphEdges.setAttribute("height", String(state.graphLayout.sceneHeight));
-      graphEdges.setAttribute("viewBox", "0 0 " + state.graphLayout.sceneWidth + " " + state.graphLayout.sceneHeight);
-      graphNodes.style.width = state.graphLayout.sceneWidth + "px";
-      graphNodes.style.height = state.graphLayout.sceneHeight + "px";
-
-      const defs = document.createElementNS("http://www.w3.org/2000/svg", "defs");
-      const marker = document.createElementNS("http://www.w3.org/2000/svg", "marker");
-      marker.setAttribute("id", "arrow");
-      marker.setAttribute("viewBox", "0 0 10 10");
-      marker.setAttribute("refX", "9");
-      marker.setAttribute("refY", "5");
-      marker.setAttribute("markerWidth", "7");
-      marker.setAttribute("markerHeight", "7");
-      marker.setAttribute("orient", "auto");
-      const markerPath = document.createElementNS("http://www.w3.org/2000/svg", "path");
-      markerPath.setAttribute("d", "M 0 0 L 10 5 L 0 10 z");
-      markerPath.setAttribute("fill", "var(--accent)");
-      marker.appendChild(markerPath);
-      defs.appendChild(marker);
-      graphEdges.appendChild(defs);
-
-      for (const table of tables) {
-        const to = positions.get(table.id);
-        if (!to) continue;
-        const dependencies = Array.isArray(table.dependencies) ? table.dependencies : [];
-        for (const dependencyId of dependencies) {
-          const from = positions.get(dependencyId);
-          if (!from) continue;
-          const startX = from.x + NODE_WIDTH / 2;
-          const startY = from.y + NODE_HEIGHT;
-          const endX = to.x + NODE_WIDTH / 2;
-          const endY = to.y;
-          const midY = startY + (endY - startY) / 2;
-          const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
-          path.setAttribute("d", "M " + startX + " " + startY + " V " + midY + " H " + endX + " V " + endY);
-          path.setAttribute("fill", "none");
-          path.setAttribute("stroke", "var(--accent)");
-          path.setAttribute("stroke-width", "1.7");
-          path.setAttribute("marker-end", "url(#arrow)");
-          graphEdges.appendChild(path);
-        }
-      }
-
       for (const table of tables) {
-        const pos = positions.get(table.id);
-        if (!pos) continue;
         const operatorClass = (() => {
           const normalized = String(table.operator || "unknown").toLowerCase();
-          if (normalized === "stored" || normalized === "map" || normalized === "flatmap" || normalized === "groupby" || normalized === "filter" || normalized === "limit") {
+          if (normalized === "stored" || normalized === "map" || normalized === "flatmap" || normalized === "groupby" || normalized === "filter" || normalized === "limit" || normalized === "concat") {
             return normalized;
           }
           return "derived";
         })();
         const node = document.createElement("div");
         node.className = "node" + (state.selectedTableId === table.id ? " active" : "");
-        node.style.left = pos.x + "px";
-        node.style.top = pos.y + "px";
+        node.setAttribute("data-table-id", String(table.id));
 
         const type = document.createElement("div");
         type.className = "node-type " + operatorClass;
@@ -1038,7 +1275,30 @@ function getStudioPageHtml(): string {
         node.appendChild(name);
         node.appendChild(meta);
         node.appendChild(actions);
+        node.addEventListener("mousedown", (event) => {
+          const target = event.target;
+          if (!(target instanceof HTMLElement)) return;
+          if (target.closest("button")) return;
+          const position = state.graphLayout?.positions.get(table.id);
+          if (!position) return;
+          event.preventDefault();
+          event.stopPropagation();
+          state.dragging.active = true;
+          state.dragging.kind = "node";
+          state.dragging.nodeId = String(table.id);
+          state.dragging.startX = event.clientX;
+          state.dragging.startY = event.clientY;
+          state.dragging.nodeStartX = position.x;
+          state.dragging.nodeStartY = position.y;
+          state.dragging.moved = false;
+          node.classList.add("dragging");
+          graphShell.classList.add("dragging");
+        });
         node.onclick = () => {
+          if (state.dragging.suppressClickTableId === table.id) {
+            state.dragging.suppressClickTableId = null;
+            return;
+          }
           runUiAction("load table details", async () => {
             setMode("table");
             await selectTable(table.id);
@@ -1047,7 +1307,7 @@ function getStudioPageHtml(): string {
         graphNodes.appendChild(node);
       }
 
-      updateSceneTransform();
+      relayoutGraph();
     }
 
     function getRawInputDefault() {
@@ -1479,22 +1739,52 @@ function getStudioPageHtml(): string {
         if (!(target instanceof HTMLElement)) return;
         if (target.closest(".node")) return;
         state.dragging.active = true;
+        state.dragging.kind = "pan";
         state.dragging.startX = event.clientX;
         state.dragging.startY = event.clientY;
         state.dragging.startOffsetX = state.viewport.x;
         state.dragging.startOffsetY = state.viewport.y;
+        state.dragging.moved = false;
         graphShell.classList.add("dragging");
       });
 
       window.addEventListener("mousemove", (event) => {
         if (!state.dragging.active) return;
-        state.viewport.x = state.dragging.startOffsetX + (event.clientX - state.dragging.startX);
-        state.viewport.y = state.dragging.startOffsetY + (event.clientY - state.dragging.startY);
-        updateSceneTransform();
+        const deltaX = event.clientX - state.dragging.startX;
+        const deltaY = event.clientY - state.dragging.startY;
+        if (Math.abs(deltaX) > 3 || Math.abs(deltaY) > 3) {
+          state.dragging.moved = true;
+        }
+        if (state.dragging.kind === "pan") {
+          state.viewport.x = state.dragging.startOffsetX + deltaX;
+          state.viewport.y = state.dragging.startOffsetY + deltaY;
+          updateSceneTransform();
+          return;
+        }
+        if (state.dragging.kind === "node" && state.dragging.nodeId) {
+          const nextX = Math.max(SCENE_MARGIN / 2, state.dragging.nodeStartX + deltaX / state.viewport.scale);
+          const nextY = Math.max(SCENE_MARGIN / 2, state.dragging.nodeStartY + deltaY / state.viewport.scale);
+          state.manualNodePositions[state.dragging.nodeId] = { x: nextX, y: nextY };
+          persistNodePositions();
+          relayoutGraph();
+        }
       });
 
       window.addEventListener("mouseup", () => {
+        if (state.dragging.kind === "node" && state.dragging.nodeId && state.dragging.moved) {
+          state.dragging.suppressClickTableId = state.dragging.nodeId;
+        }
+        const draggingNodeId = state.dragging.nodeId;
+        if (draggingNodeId) {
+          const node = graphNodes.querySelector('[data-table-id="' + draggingNodeId.replaceAll('"', '\\"') + '"]');
+          if (node instanceof HTMLElement) {
+            node.classList.remove("dragging");
+          }
+        }
         state.dragging.active = false;
+        state.dragging.kind = null;
+        state.dragging.nodeId = null;
+        state.dragging.moved = false;
         graphShell.classList.remove("dragging");
       });
     }
diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts
index 24ab566bc7..18abe0e285 100644
--- a/apps/backend/src/lib/bulldozer/db/example-schema.ts
+++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts
@@ -1,4 +1,4 @@
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable } from "./index";
+import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable } from "./index";
 
 const mapper = (sql: string) => ({ type: "mapper" as const, sql });
 const predicate = (sql: string) => ({ type: "predicate" as const, sql });
@@ -125,6 +125,10 @@ export const exampleFungibleLedgerSchema = (() => {
     fromTable: highValueEntriesByAsset,
     groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`),
   });
+  const accountPriorityEntries = declareConcatTable({
+    tableId: "bulldozer-example-ledger-account-priority-entries",
+    tables: [accountEntriesWithCounterparty, highValueEntriesByAssetAccount],
+  });
   const highValueEntriesByAssetAccountTop = declareLimitTable({
     tableId: "bulldozer-example-ledger-high-value-entries-by-asset-account-top",
     fromTable: highValueEntriesByAssetAccount,
@@ -159,6 +163,7 @@ export const exampleFungibleLedgerSchema = (() => {
     accountCounterpartySample,
     highValueEntriesByAsset,
     highValueEntriesByAssetAccount,
+    accountPriorityEntries,
     highValueEntriesByAssetAccountTop,
     assetEntriesNormalized,
   };
diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
index ff24eff961..d7a376b9a9 100644
--- a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts
@@ -1,7 +1,7 @@
-import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
+import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -262,6 +262,24 @@ function limitGroups>(
   }
   return limited;
 }
+function concatGroups>(
+  groupsList: GroupedRows[],
+): GroupedRows {
+  const concatenated: GroupedRows = new Map();
+  for (let tableIndex = 0; tableIndex < groupsList.length; tableIndex++) {
+    const groups = groupsList[tableIndex] ?? (() => {
+      throw new Error("concatGroups expected grouped rows for table index");
+    })();
+    for (const [groupKey, group] of groups) {
+      const existing = concatenated.get(groupKey) ?? { groupKey: group.groupKey, rows: new Map() };
+      for (const [rowIdentifier, rowData] of group.rows) {
+        existing.rows.set(`${tableIndex}:${rowIdentifier}`, rowData);
+      }
+      concatenated.set(groupKey, existing);
+    }
+  }
+  return concatenated;
+}
 
 describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
   const dbUrls = getTestDbUrls();
@@ -941,6 +959,105 @@ describe.sequential("bulldozer db fuzz composition (real postgres)", () => {
     }
   }, 120_000);
 
+  test("fuzz: virtual concat table preserves prefixed rows across parallel source mutations", async () => {
+    const identifiers = ["c1", "c2", "c3", "c:4", "c 5", "c/6", "c'7"] as const;
+    const teams = ["alpha", "beta", "gamma"] as const;
+
+    for (const seed of [1801]) {
+      const rng = createRng(seed);
+      const sourceRowsA = new Map();
+      const sourceRowsB = new Map();
+      let secondInputInitialized = true;
+      let concatInitialized = true;
+
+      const fromTableA = declareStoredTable<{ value: number, team: string | null }>({ tableId: `concat-fuzz-users-a-${seed}` });
+      const fromTableB = declareStoredTable<{ value: number, team: string | null }>({ tableId: `concat-fuzz-users-b-${seed}` });
+      const groupedTableA = declareGroupByTable({
+        tableId: `concat-fuzz-users-a-by-team-${seed}`,
+        fromTable: fromTableA,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const groupedTableB = declareGroupByTable({
+        tableId: `concat-fuzz-users-b-by-team-${seed}`,
+        fromTable: fromTableB,
+        groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+      });
+      const concatenatedTable = declareConcatTable({
+        tableId: `concat-fuzz-users-by-team-${seed}`,
+        tables: [groupedTableA, groupedTableB],
+      });
+
+      await runStatements(fromTableA.init());
+      await runStatements(fromTableB.init());
+      await runStatements(groupedTableA.init());
+      await runStatements(groupedTableB.init());
+      await runStatements(concatenatedTable.init());
+
+      for (let step = 0; step < 24; step++) {
+        const roll = rng();
+        const mutateTableA = roll < 0.42;
+        const mutateTableB = roll >= 0.42 && roll < 0.84;
+        const targetRows = mutateTableA ? sourceRowsA : sourceRowsB;
+        const targetTable = mutateTableA ? fromTableA : fromTableB;
+
+        if (mutateTableA || mutateTableB) {
+          if (rng() < 0.68) {
+            const rowIdentifier = choose(rng, identifiers);
+            const rowData: SourceRow = {
+              team: choose(rng, teams),
+              value: Math.floor(rng() * 60),
+            };
+            targetRows.set(rowIdentifier, rowData);
+            await runStatements(targetTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData))));
+          } else {
+            const rowIdentifier = choose(rng, identifiers);
+            targetRows.delete(rowIdentifier);
+            await runStatements(targetTable.deleteRow(rowIdentifier));
+          }
+        } else if (roll < 0.90) {
+          if (secondInputInitialized) {
+            await runStatements(groupedTableB.delete());
+            secondInputInitialized = false;
+          }
+        } else if (roll < 0.95) {
+          if (concatInitialized) {
+            await runStatements(concatenatedTable.delete());
+            concatInitialized = false;
+          }
+        } else {
+          if (!secondInputInitialized) {
+            await runStatements(groupedTableB.init());
+            secondInputInitialized = true;
+          } else if (!concatInitialized) {
+            await runStatements(concatenatedTable.init());
+            concatInitialized = true;
+          }
+        }
+
+        if (step % 3 === 0 || step === 23) {
+          const expectedA = computeTeamGroups(sourceRowsA);
+          const expectedB = computeTeamGroups(sourceRowsB);
+          if (!concatInitialized) {
+            expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false);
+            const groups = await readRows(concatenatedTable.listGroups({
+              start: "start",
+              end: "end",
+              startInclusive: true,
+              endInclusive: true,
+            }));
+            expect(groups).toEqual([]);
+          } else if (secondInputInitialized) {
+            expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+            await assertTableMatches(concatenatedTable, concatGroups([expectedA, expectedB]));
+          } else {
+            expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+            await assertTableMatches(concatenatedTable, concatGroups([expectedA]));
+          }
+        }
+      }
+    }
+  }, 120_000);
+
   test("fuzz: parallel map tables remain isolated with independent re-inits", async () => {
     const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const;
     const teams = ["alpha", "beta", null] as const;
diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
index 0f76fdb3bd..420bc80fc3 100644
--- a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 type SqlExpression = { type: "expression", sql: string };
@@ -37,7 +37,11 @@ const LOAD_FILTER_TABLE_INIT_MAX_MS = 90_000;
 const LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS = 8_000;
 const LOAD_LIMIT_TABLE_INIT_MAX_MS = 90_000;
 const LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS = 8_000;
+const LOAD_CONCAT_TABLE_INIT_MAX_MS = 10_000;
+const LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS = 8_000;
 const STACKED_MAP_PIPELINE_MUTATION_MAX_MS = 400;
+const VIRTUAL_CONCAT_COUNT_QUERY_MAX_MS = 500;
+const VIRTUAL_CONCAT_LOAD_ROW_COUNT = 5_000;
 
 function getTestDbUrls(): TestDb {
   const env = Reflect.get(import.meta, "env");
@@ -398,6 +402,61 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(deleteMutation.elapsedMs).toBeLessThan(STACKED_MAP_PIPELINE_MUTATION_MAX_MS);
   });
 
+  it("regression: virtual concat queries stay fast after metadata-only initialization", async () => {
+    const tableAId = "perf-concat-users-a";
+    const tableBId = "perf-concat-users-b";
+    const fromTableA = declareStoredTable<{ value: number, team: string | null }>({ tableId: tableAId });
+    const fromTableB = declareStoredTable<{ value: number, team: string | null }>({ tableId: tableBId });
+    const groupedByTeamA = declareGroupByTable({
+      tableId: "perf-concat-users-a-by-team",
+      fromTable: fromTableA,
+      groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` },
+    });
+    const groupedByTeamB = declareGroupByTable({
+      tableId: "perf-concat-users-b-by-team",
+      fromTable: fromTableB,
+      groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` },
+    });
+    const concatenatedByTeam = declareConcatTable({
+      tableId: "perf-concat-users-by-team",
+      tables: [groupedByTeamA, groupedByTeamB],
+    });
+
+    expect((await readRows(concatenatedByTeam.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    })))).toEqual([]);
+
+    await prefillStoredTableInSingleStatement(tableAId, VIRTUAL_CONCAT_LOAD_ROW_COUNT);
+    await prefillStoredTableInSingleStatement(tableBId, VIRTUAL_CONCAT_LOAD_ROW_COUNT);
+    await runStatements(groupedByTeamA.init());
+    await runStatements(groupedByTeamB.init());
+    await runStatements(concatenatedByTeam.init());
+    expect(await readRows(concatenatedByTeam.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }))).not.toEqual([]);
+
+    const concatenatedCountQuery = concatenatedByTeam.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    });
+    const countRows = await measureMs("virtual concat count query", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (${toQueryableSqlQuery(concatenatedCountQuery)}) AS "rows"
+      `);
+    });
+    expect(countRows.elapsedMs).toBeLessThan(VIRTUAL_CONCAT_COUNT_QUERY_MAX_MS);
+    expect(Number(countRows.result[0].count)).toBe(VIRTUAL_CONCAT_LOAD_ROW_COUNT * 2);
+  });
+
   it("load test: prefilled stored table with hundreds of thousands of rows stays functional and fast", async () => {
     const loadRowCount = DEFAULT_LOAD_ROW_COUNT;
     const tableId = "load-prefilled-users";
@@ -499,6 +558,10 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       fromTable: groupedByTeam,
       filter: { type: "predicate", sql: `( ("rowData"->>'value')::int ) >= 700` },
     });
+    const concatenatedByTeam = declareConcatTable({
+      tableId: "load-prefilled-users-concat",
+      tables: [groupedByTeam, filteredHighValue],
+    });
     const limitedByTeam = declareLimitTable({
       tableId: "load-prefilled-users-top-team-rows",
       fromTable: groupedByTeam,
@@ -543,6 +606,10 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       await runStatements(filteredHighValue.init());
     });
     expect(filterInit.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_INIT_MAX_MS);
+    const concatInit = await measureMs("load init concatenatedByTeam", async () => {
+      await runStatements(concatenatedByTeam.init());
+    });
+    expect(concatInit.elapsedMs).toBeLessThan(LOAD_CONCAT_TABLE_INIT_MAX_MS);
     const limitInit = await measureMs("load init limitedByTeam", async () => {
       await runStatements(limitedByTeam.init());
     });
@@ -582,6 +649,12 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
       startInclusive: true,
       endInclusive: true,
     });
+    const concatenatedByTeamCountQuery = concatenatedByTeam.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    });
     const limitedByTeamCountQuery = limitedByTeam.listRowsInGroup({
       start: "start",
       end: "end",
@@ -594,6 +667,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows"`),
+        sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(concatenatedByTeamCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(limitedByTeamCountQuery)}) AS "rows"`),
         sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"`),
       ]);
@@ -604,9 +678,11 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1);
     expect(Number(derivedCounts.result[3][0].count)).toBeGreaterThan(0);
     expect(Number(derivedCounts.result[3][0].count)).toBeLessThan(loadRowCount);
-    expect(Number(derivedCounts.result[4][0].count)).toBeGreaterThan(0);
-    expect(Number(derivedCounts.result[4][0].count)).toBeLessThanOrEqual(100);
-    expect(Number(derivedCounts.result[5][0].count)).toBe((loadRowCount - 1) * 2);
+    expect(Number(derivedCounts.result[4][0].count)).toBeGreaterThan(loadRowCount - 1);
+    expect(Number(derivedCounts.result[4][0].count)).toBeLessThan((loadRowCount - 1) * 2);
+    expect(Number(derivedCounts.result[5][0].count)).toBeGreaterThan(0);
+    expect(Number(derivedCounts.result[5][0].count)).toBeLessThanOrEqual(100);
+    expect(Number(derivedCounts.result[6][0].count)).toBe((loadRowCount - 1) * 2);
 
     const filteredHighValueCountOnly = await measureMs("load count filteredHighValue table only", async () => {
       return await sql.unsafe(`
@@ -617,6 +693,16 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(filteredHighValueCountOnly.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS);
     expect(Number(filteredHighValueCountOnly.result[0].count)).toBeGreaterThan(0);
 
+    const concatenatedByTeamCountOnly = await measureMs("load count concatenatedByTeam table only", async () => {
+      return await sql.unsafe(`
+        SELECT COUNT(*)::int AS "count"
+        FROM (${toQueryableSqlQuery(concatenatedByTeamCountQuery)}) AS "rows"
+      `);
+    });
+    expect(concatenatedByTeamCountOnly.elapsedMs).toBeLessThan(LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS);
+    expect(Number(concatenatedByTeamCountOnly.result[0].count)).toBeGreaterThan(loadRowCount - 1);
+    expect(Number(concatenatedByTeamCountOnly.result[0].count)).toBeLessThan((loadRowCount - 1) * 2);
+
     const limitedByTeamCountOnly = await measureMs("load count limitedByTeam table only", async () => {
       return await sql.unsafe(`
         SELECT COUNT(*)::int AS "count"
@@ -701,6 +787,17 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     expect(filteredDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
       { rowIdentifier: "seed-100000:1", rowData: { team: "delta", value: 999 } },
     ]);
+    const concatenatedDeltaRows = await readRows(concatenatedByTeam.listRowsInGroup({
+      groupKey: expr(`to_jsonb('delta'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(concatenatedDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "0:seed-100000", rowData: { team: "delta", value: 999 } },
+      { rowIdentifier: "1:seed-100000:1", rowData: { team: "delta", value: 999 } },
+    ]);
     const limitedDeltaRows = await readRows(limitedByTeam.listRowsInGroup({
       groupKey: expr(`to_jsonb('delta'::text)`),
       start: "start",
@@ -729,7 +826,7 @@ describe.sequential("bulldozer db performance (real postgres)", () => {
     `;
     expect(isInitializedRows[0].initialized).toBe(false);
 
-    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, limitInit<=${LOAD_LIMIT_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, limitCount<=${LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
+    logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, concatInit<=${LOAD_CONCAT_TABLE_INIT_MAX_MS}, limitInit<=${LOAD_LIMIT_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, concatCount<=${LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS}, limitCount<=${LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`);
   }, 180_000);
 });
 
diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts
index 3887c0077c..98dfae9f53 100644
--- a/apps/backend/src/lib/bulldozer/db/index.test.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.test.ts
@@ -1,7 +1,7 @@
 import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings";
 import postgres from "postgres";
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
-import { declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
+import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLimitTable, declareMapTable, declareStoredTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index";
 
 type TestDb = { full: string, base: string };
 
@@ -263,6 +263,25 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     });
     return { fromTable, groupedTable, limitedTable };
   }
+  function createConcatenatedTable() {
+    const fromTableA = declareStoredTable<{ value: number, team: string }>({ tableId: "users-a" });
+    const fromTableB = declareStoredTable<{ value: number, team: string }>({ tableId: "users-b" });
+    const groupedTableA = declareGroupByTable({
+      tableId: "users-a-by-team",
+      fromTable: fromTableA,
+      groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+    });
+    const groupedTableB = declareGroupByTable({
+      tableId: "users-b-by-team",
+      fromTable: fromTableB,
+      groupBy: mapper(`"rowData"->'team' AS "groupKey"`),
+    });
+    const concatenatedTable = declareConcatTable({
+      tableId: "users-by-team-concat",
+      tables: [groupedTableA, groupedTableB],
+    });
+    return { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable };
+  }
   function createFlatMapMapGroupPipeline() {
     const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable();
     const mappedAfterFlatMap = declareMapTable({
@@ -431,6 +450,29 @@ describe.sequential("declareStoredTable (real postgres)", () => {
       `,
     ]);
   }
+  function registerConcatAuditTrigger(
+    table: ReturnType["concatenatedTable"],
+    event: string,
+  ) {
+    return table.registerRowChangeTrigger((changesTable) => [
+      sqlStatement`
+        INSERT INTO "BulldozerMapTriggerAudit" (
+          "event",
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        )
+        SELECT
+          ${expr(sqlStringLiteral(event))},
+          "groupKey",
+          "rowIdentifier",
+          "oldRowData",
+          "newRowData"
+        FROM ${changesTable}
+      `,
+    ]);
+  }
 
   test("init/isInitialized/delete lifecycle", async () => {
     const table = declareStoredTable<{ value: number }>({ tableId: "users" });
@@ -2021,6 +2063,163 @@ describe.sequential("declareStoredTable (real postgres)", () => {
     expect(limitAuditRows).toEqual([]);
   });
 
+  test("concatTable virtually concatenates grouped inputs and prefixes row identifiers", async () => {
+    const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable();
+    await runStatements(fromTableA.init());
+    await runStatements(fromTableB.init());
+    await runStatements(groupedTableA.init());
+    await runStatements(groupedTableB.init());
+
+    await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTableA.setRow("a2", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTableB.setRow("b1", expr(`'{"team":"alpha","value":3}'::jsonb`)));
+    await runStatements(fromTableB.setRow("b2", expr(`'{"team":"gamma","value":4}'::jsonb`)));
+
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false);
+    expect(await readRows(concatenatedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }))).toEqual([]);
+    await runStatements(concatenatedTable.init());
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+
+    const groups = await readRows(concatenatedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta", "gamma"]);
+
+    const alphaRows = await readRows(concatenatedTable.listRowsInGroup({
+      groupKey: expr(`to_jsonb('alpha'::text)`),
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([
+      { rowIdentifier: "0:a1", rowData: { team: "alpha", value: 1 } },
+      { rowIdentifier: "1:b1", rowData: { team: "alpha", value: 3 } },
+    ]);
+
+    const allRows = await readRows(concatenatedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(allRows.map((row) => ({
+      groupKey: row.groupkey,
+      rowIdentifier: row.rowidentifier,
+      rowData: row.rowdata,
+    }))).toEqual([
+      { groupKey: "alpha", rowIdentifier: "0:a1", rowData: { team: "alpha", value: 1 } },
+      { groupKey: "beta", rowIdentifier: "0:a2", rowData: { team: "beta", value: 2 } },
+      { groupKey: "alpha", rowIdentifier: "1:b1", rowData: { team: "alpha", value: 3 } },
+      { groupKey: "gamma", rowIdentifier: "1:b2", rowData: { team: "gamma", value: 4 } },
+    ]);
+  });
+
+  test("concatTable forwards prefixed trigger changes from each input table", async () => {
+    const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable();
+    await runStatements(fromTableA.init());
+    await runStatements(fromTableB.init());
+    await runStatements(groupedTableA.init());
+    await runStatements(groupedTableB.init());
+    await runStatements(concatenatedTable.init());
+    registerConcatAuditTrigger(concatenatedTable, "concat_change");
+
+    await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    await runStatements(fromTableB.setRow("b1", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    await runStatements(fromTableB.setRow("b1", expr(`'{"team":"gamma","value":5}'::jsonb`)));
+    await runStatements(fromTableA.deleteRow("a1"));
+
+    const auditRows = (await readMapTriggerAuditRows())
+      .filter((row) => row.event === "concat_change")
+      .map((row) => ({
+        groupKey: row.groupKey,
+        rowIdentifier: row.rowIdentifier,
+        oldRowData: row.oldRowData,
+        newRowData: row.newRowData,
+      }));
+    expect(auditRows).toEqual([
+      { groupKey: "alpha", rowIdentifier: "0:a1", oldRowData: null, newRowData: { team: "alpha", value: 1 } },
+      { groupKey: "beta", rowIdentifier: "1:b1", oldRowData: null, newRowData: { team: "beta", value: 2 } },
+      { groupKey: "beta", rowIdentifier: "1:b1", oldRowData: { team: "beta", value: 2 }, newRowData: null },
+      { groupKey: "gamma", rowIdentifier: "1:b1", oldRowData: null, newRowData: { team: "gamma", value: 5 } },
+      { groupKey: "alpha", rowIdentifier: "0:a1", oldRowData: { team: "alpha", value: 1 }, newRowData: null },
+    ]);
+  });
+
+  test("concatTable stays virtual but requires its own metadata initialization", async () => {
+    const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable();
+
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false);
+
+    const beforeInitGroups = await readRows(concatenatedTable.listGroups({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(beforeInitGroups).toEqual([]);
+
+    await runStatements(fromTableA.init());
+    await runStatements(groupedTableA.init());
+    await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`)));
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false);
+
+    const oneSideOnlyRows = await readRows(concatenatedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(oneSideOnlyRows).toEqual([]);
+
+    await runStatements(concatenatedTable.init());
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+    const rowsAfterConcatInit = await readRows(concatenatedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(rowsAfterConcatInit.map((row) => row.rowidentifier)).toEqual(["0:a1"]);
+
+    await runStatements(fromTableB.init());
+    await runStatements(groupedTableB.init());
+    await runStatements(fromTableB.setRow("b1", expr(`'{"team":"beta","value":2}'::jsonb`)));
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+
+    await runStatements(concatenatedTable.delete());
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false);
+
+    const rowsAfterDelete = await readRows(concatenatedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(rowsAfterDelete).toEqual([]);
+
+    await runStatements(concatenatedTable.init());
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+
+    await runStatements(groupedTableB.delete());
+    expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true);
+    const rowsAfterInputDelete = await readRows(concatenatedTable.listRowsInGroup({
+      start: "start",
+      end: "end",
+      startInclusive: true,
+      endInclusive: true,
+    }));
+    expect(rowsAfterInputDelete.map((row) => row.rowidentifier)).toEqual(["0:a1"]);
+  });
+
   test("flatMap -> map -> groupBy composition stays consistent across updates", async () => {
     const { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind } = createFlatMapMapGroupPipeline();
     await runStatements(fromTable.init());
diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts
index cc2b950160..8382212d97 100644
--- a/apps/backend/src/lib/bulldozer/db/index.ts
+++ b/apps/backend/src/lib/bulldozer/db/index.ts
@@ -30,6 +30,10 @@ export type Table = {
   registerRowChangeTrigger(trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[]): { deregister: () => void },
 };
 
+
+// ====== Table implementations ======
+// IMPORTANT NOTE: For every new table implementation, we should also add tests (unit, fuzzing, & perf; including an entry in the "hundreds of thousands" perf test), an example in the example schema, and support in Bulldozer Studio.
+
 export function declareStoredTable(options: {
   tableId: TableId,
 }): Table & {
@@ -1360,13 +1364,178 @@ export function declareLimitTable<
   };
 }
 
-export declare function declareConcatTable<
+export function declareConcatTable<
   GK extends Json,
   RD extends RowData,
 >(options: {
   tableId: TableId,
   tables: Table[],
-}): Table;
+}): Table {
+  const firstTable = options.tables[0] ?? (() => {
+    throw new StackAssertionError("declareConcatTable requires at least one input table", { tableId: options.tableId });
+  })();
+  const triggers = new Map) => SqlStatement[]>();
+  const rawExpression = (sql: string): SqlExpression => ({ type: "expression", sql });
+  const isInitializedExpression = sqlExpression`
+    EXISTS (
+      SELECT 1 FROM "BulldozerStorageEngine"
+      WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
+    )
+  `;
+  const createConcatenatedRowIdentifierSql = (tableIndex: number, rowIdentifierSql: string) =>
+    `${quoteSqlStringLiteral(`${tableIndex}:`).sql} || ${rowIdentifierSql}`;
+  const getInputInitializedSql = (table: Table) => table.isInitialized().sql;
+  const getUnionedListGroupsSql = (queryOptions: Parameters[0]) => {
+    return options.tables
+      .map((table) => deindent`
+        SELECT "sourceGroups"."groupkey" AS "groupKey"
+        FROM (${table.listGroups(queryOptions).sql}) AS "sourceGroups"
+        WHERE ${getInputInitializedSql(table)}
+      `)
+      .join("\nUNION ALL\n");
+  };
+  const getUnionedListRowsSql = (queryOptions: Parameters[0] & { allGroups: boolean }) => {
+    return options.tables.map((table, tableIndex) => {
+      if (queryOptions.allGroups) {
+        return deindent`
+          SELECT
+            "sourceRows"."groupkey" AS "groupKey",
+            ${createConcatenatedRowIdentifierSql(tableIndex, `"sourceRows"."rowidentifier"`)} AS "rowIdentifier",
+            'null'::jsonb AS "rowSortKey",
+            "sourceRows"."rowdata" AS "rowData",
+            ${tableIndex}::int AS "sourceTableIndex",
+            row_number() OVER (
+              ORDER BY "sourceRows"."groupkey" ASC, "sourceRows"."rowsortkey" ASC, "sourceRows"."rowidentifier" ASC
+            ) AS "sourceRowIndex"
+          FROM (${table.listRowsInGroup({
+            start: "start",
+            end: "end",
+            startInclusive: true,
+            endInclusive: true,
+          }).sql}) AS "sourceRows"
+          WHERE ${getInputInitializedSql(table)}
+        `;
+      }
+      const groupKey = queryOptions.groupKey ?? (() => {
+        throw new StackAssertionError("declareConcatTable specific-group query requires a group key");
+      })();
+      return deindent`
+        SELECT
+          ${createConcatenatedRowIdentifierSql(tableIndex, `"sourceRows"."rowidentifier"`)} AS "rowIdentifier",
+          'null'::jsonb AS "rowSortKey",
+          "sourceRows"."rowdata" AS "rowData",
+          ${tableIndex}::int AS "sourceTableIndex",
+          row_number() OVER (
+            ORDER BY "sourceRows"."rowsortkey" ASC, "sourceRows"."rowidentifier" ASC
+          ) AS "sourceRowIndex"
+        FROM (${table.listRowsInGroup({
+          groupKey,
+          start: "start",
+          end: "end",
+          startInclusive: true,
+          endInclusive: true,
+        }).sql}) AS "sourceRows"
+        WHERE ${getInputInitializedSql(table)}
+      `;
+    }).join("\nUNION ALL\n");
+  };
+
+  options.tables.forEach((table, tableIndex) => {
+    table.registerRowChangeTrigger((changesTable) => {
+      const concatChangesTableName = `concat_changes_${generateSecureRandomString()}`;
+      return [
+        sqlQuery`
+          SELECT
+            "changes"."groupKey" AS "groupKey",
+            ${rawExpression(createConcatenatedRowIdentifierSql(tableIndex, `"changes"."rowIdentifier"`))} AS "rowIdentifier",
+            'null'::jsonb AS "oldRowSortKey",
+            'null'::jsonb AS "newRowSortKey",
+            "changes"."oldRowData" AS "oldRowData",
+            "changes"."newRowData" AS "newRowData"
+          FROM ${changesTable} AS "changes"
+          WHERE ${isInitializedExpression}
+            AND ${rawExpression(getInputInitializedSql(table))}
+        `.toStatement(concatChangesTableName),
+        ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(concatChangesTableName))),
+      ];
+    });
+  });
+
+  return {
+    tableId: options.tableId,
+    inputTables: options.tables,
+    debugArgs: {
+      operator: "concat",
+      tableId: tableIdToDebugString(options.tableId),
+      inputTableIds: options.tables.map((table) => tableIdToDebugString(table.tableId)),
+    },
+    listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
+      SELECT DISTINCT "concatGroups"."groupKey" AS groupKey
+      FROM (${rawExpression(getUnionedListGroupsSql({ start, end, startInclusive, endInclusive }))}) AS "concatGroups"
+      WHERE ${isInitializedExpression}
+    `,
+    listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey != null ? sqlQuery`
+      SELECT
+        "concatRows"."rowIdentifier" AS rowIdentifier,
+        "concatRows"."rowSortKey" AS rowSortKey,
+        "concatRows"."rowData" AS rowData
+      FROM (${rawExpression(getUnionedListRowsSql({
+        groupKey,
+        start,
+        end,
+        startInclusive,
+        endInclusive,
+        allGroups: false,
+      }))}) AS "concatRows"
+      WHERE ${isInitializedExpression}
+        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
+      ORDER BY "concatRows"."sourceTableIndex" ASC, "concatRows"."sourceRowIndex" ASC, "concatRows"."rowIdentifier" ASC
+    ` : sqlQuery`
+      SELECT
+        "concatRows"."groupKey" AS groupKey,
+        "concatRows"."rowIdentifier" AS rowIdentifier,
+        "concatRows"."rowSortKey" AS rowSortKey,
+        "concatRows"."rowData" AS rowData
+      FROM (${rawExpression(getUnionedListRowsSql({
+        start,
+        end,
+        startInclusive,
+        endInclusive,
+        allGroups: true,
+      }))}) AS "concatRows"
+      WHERE ${isInitializedExpression}
+        AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })}
+      ORDER BY "concatRows"."sourceTableIndex" ASC, "concatRows"."sourceRowIndex" ASC, "concatRows"."rowIdentifier" ASC
+    `,
+    compareGroupKeys: firstTable.compareGroupKeys,
+    compareSortKeys: () => sqlExpression`0`,
+    init: () => [sqlStatement`
+      INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
+      VALUES
+      (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
+      (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb),
+      (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb),
+      (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb)
+    `],
+    delete: () => [sqlStatement`
+      WITH RECURSIVE "pathsToDelete" AS (
+        SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
+        UNION ALL
+        SELECT "BulldozerStorageEngine"."keyPath" AS "path"
+        FROM "BulldozerStorageEngine"
+        INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
+      )
+      DELETE FROM "BulldozerStorageEngine"
+      WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
+    `],
+    isInitialized: () => isInitializedExpression,
+    registerRowChangeTrigger: (trigger) => {
+      const id = generateSecureRandomString();
+      triggers.set(id, trigger);
+      return { deregister: () => triggers.delete(id) };
+    },
+  };
+}
 
 export declare function declareSortTable<
   GK extends Json,
diff --git a/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt b/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt
deleted file mode 100644
index c9f7ada6a4..0000000000
--- a/apps/backend/src/lib/bulldozer/db/slow-stacked-mutation.sql.txt
+++ /dev/null
@@ -1,706 +0,0 @@
-BEGIN;
-
-SET LOCAL jit = off;
-
-SELECT pg_advisory_xact_lock(7857391);
-
-WITH __dummy_statement_1__ AS (SELECT 1),
-"old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer" AS (
-  
-            SELECT "value"->'rowData' AS "oldRowData"
-            FROM "BulldozerStorageEngine"
-            WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-101"'::jsonb, '"storage"'::jsonb, '"rows"'::jsonb, '"u2"'::jsonb]::jsonb[]
-          
-),
-"upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr" AS (
-  
-            INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-            VALUES (
-              gen_random_uuid(),
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-101"'::jsonb, '"storage"'::jsonb, '"rows"'::jsonb, '"u2"'::jsonb]::jsonb[],
-              
-          jsonb_build_object(
-            'rowData', '{"team":"beta","value":21}'::jsonb::jsonb
-          )
-        ::jsonb
-            )
-            ON CONFLICT ("keyPath") DO UPDATE
-            SET "value" = 
-          jsonb_build_object(
-            'rowData', '{"team":"beta","value":21}'::jsonb::jsonb
-          )
-        ::jsonb
-            RETURNING "value"->'rowData' AS "newRowData"
-          
-),
-"changes_rpbs2y486ebg9ae3n089fya0018rngx9228gz5yjpwjdg" AS (
-  
-            SELECT
-              'null'::jsonb AS "groupKey",
-              'u2'::text AS "rowIdentifier",
-              'null'::jsonb AS "oldRowSortKey",
-              'null'::jsonb AS "newRowSortKey",
-              "old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer"."oldRowData" AS "oldRowData",
-              "upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr"."newRowData" AS "newRowData"
-            FROM "upserted_rows_c2sk1nywpsygs51cfqnd7qg3jd8nk7zh6jjqry0j6bwzr"
-            LEFT JOIN "old_rows_qzm5g336vh1djv54sn9hwf7510yxxrzv5h4kwrgqgxrer" ON true
-          
-),
-"mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS (
-  
-          SELECT
-            "changes"."rowIdentifier" AS "rowIdentifier",
-            "changes"."oldRowData" AS "oldRowData",
-            "changes"."newRowData" AS "newRowData",
-            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
-            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
-            "oldGroup"."groupKey" AS "oldGroupKey",
-            "newGroup"."groupKey" AS "newGroupKey"
-          FROM "changes_rpbs2y486ebg9ae3n089fya0018rngx9228gz5yjpwjdg" AS "changes"
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."groupKey"
-            FROM (
-              SELECT "rowData"->'team' AS "groupKey"
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."oldRowData" AS "rowData"
-              ) AS "groupByInput"
-            ) AS "mapped"
-          ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."groupKey"
-            FROM (
-              SELECT "rowData"->'team' AS "groupKey"
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."newRowData" AS "rowData"
-              ) AS "groupByInput"
-            ) AS "mapped"
-          ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
-          WHERE 
-      EXISTS (
-        SELECT 1 FROM "BulldozerStorageEngine"
-        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
-      )
-    
-        
-),
-"unnamed_statement_6bw6129p" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            "insertRows"."keyPath",
-            "insertRows"."value"
-          FROM (
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey"]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
-            WHERE "hasNewRow"
-            UNION
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
-            WHERE "hasNewRow"
-          ) AS "insertRows"
-          ON CONFLICT ("keyPath") DO NOTHING
-        
-),
-"unnamed_statement_3bp4fqz7" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "target"
-          USING "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "changes"
-          WHERE "changes"."hasOldRow"
-            AND "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
-        
-),
-"unnamed_statement_je3f1c7g" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
-            jsonb_build_object('rowData', "newRowData")
-          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
-          WHERE "hasNewRow"
-          ON CONFLICT ("keyPath") DO UPDATE
-          SET "value" = EXCLUDED."value"
-        
-),
-"unnamed_statement_rxdy9p42" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
-          USING "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "changes"
-          WHERE "changes"."hasOldRow"
-            AND "staleGroupPath"."keyPath" IN (
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[],
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey"]::jsonb[]
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "BulldozerStorageEngine" AS "groupRow"
-              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[]
-                AND NOT EXISTS (
-                  SELECT 1
-                  FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "deletingRow"
-                  WHERE "deletingRow"."hasOldRow"
-                    AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
-                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-team-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."oldGroupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
-                )
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr" AS "insertingRow"
-              WHERE "insertingRow"."hasNewRow"
-                AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
-            )
-        
-),
-"grouped_changes_w9bytqkqjvr4kyczj0nahg3v1qn45y4mjrcgzky0d1zxr" AS (
-  
-          SELECT
-            "oldGroupKey" AS "groupKey",
-            "rowIdentifier" AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            "oldRowData" AS "oldRowData",
-            CASE
-              WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData"
-              ELSE 'null'::jsonb
-            END AS "newRowData"
-          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
-          WHERE "hasOldRow"
-          UNION ALL
-          SELECT
-            "newGroupKey" AS "groupKey",
-            "rowIdentifier" AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            'null'::jsonb AS "oldRowData",
-            "newRowData" AS "newRowData"
-          FROM "mapped_changes_v70kjg6fn4dna3s5krv29khdqyym6b0q4s6kt011wnzqr"
-          WHERE "hasNewRow"
-            AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey")
-        
-),
-"mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            "changes"."rowIdentifier" AS "sourceRowIdentifier",
-            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
-            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
-            "oldMapped"."rows" AS "oldMappedRows",
-            "newMapped"."rows" AS "newMappedRows"
-          FROM "grouped_changes_w9bytqkqjvr4kyczj0nahg3v1qn45y4mjrcgzky0d1zxr" AS "changes"
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."rows" AS "rows"
-            FROM (
-              SELECT 
-        jsonb_build_array(
-          COALESCE(
-            (
-              SELECT to_jsonb("mapped")
-              FROM (
-                SELECT ("rowData"->'team') AS "team", (("rowData"->>'value')::int + 10) AS "valuePlusTen"
-              ) AS "mapped"
-            ),
-            'null'::jsonb
-          )
-        ) AS "rows"
-      
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."oldRowData" AS "rowData"
-              ) AS "mapperInput"
-            ) AS "mapped"
-          ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."rows" AS "rows"
-            FROM (
-              SELECT 
-        jsonb_build_array(
-          COALESCE(
-            (
-              SELECT to_jsonb("mapped")
-              FROM (
-                SELECT ("rowData"->'team') AS "team", (("rowData"->>'value')::int + 10) AS "valuePlusTen"
-              ) AS "mapped"
-            ),
-            'null'::jsonb
-          )
-        ) AS "rows"
-      
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."newRowData" AS "rowData"
-              ) AS "mapperInput"
-            ) AS "mapped"
-          ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
-          WHERE 
-      EXISTS (
-        SELECT 1 FROM "BulldozerStorageEngine"
-        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
-      )
-    
-        
-),
-"old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
-            "flatRow"."rowData" AS "rowData"
-          FROM "mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS "changes"
-          CROSS JOIN LATERAL jsonb_array_elements(
-            CASE
-              WHEN "changes"."hasOldRow" THEN (
-                CASE
-                  WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows"
-                  ELSE '[]'::jsonb
-                END
-              )
-              ELSE '[]'::jsonb
-            END
-          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
-        
-),
-"new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
-            "flatRow"."rowData" AS "rowData"
-          FROM "mapped_changes_yjwm0xz9dar7fw7fzpr2r8qj3yv026rggya55xxv858fr" AS "changes"
-          CROSS JOIN LATERAL jsonb_array_elements(
-            CASE
-              WHEN "changes"."hasNewRow" THEN (
-                CASE
-                  WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows"
-                  ELSE '[]'::jsonb
-                END
-              )
-              ELSE '[]'::jsonb
-            END
-          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
-        
-),
-"unnamed_statement_733m8521" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            "insertRows"."keyPath",
-            "insertRows"."value"
-          FROM (
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey"]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
-            UNION
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
-          ) AS "insertRows"
-          ON CONFLICT ("keyPath") DO NOTHING
-        
-),
-"unnamed_statement_5fsnnkka" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "target"
-          USING "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "changes"
-          WHERE "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
-        
-),
-"unnamed_statement_93gncc42" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
-            jsonb_build_object('rowData', "rowData")
-          FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8"
-          ON CONFLICT ("keyPath") DO UPDATE
-          SET "value" = EXCLUDED."value"
-        
-),
-"unnamed_statement_fpzbmw3j" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
-          USING "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "changes"
-          WHERE "staleGroupPath"."keyPath" IN (
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[],
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey"]::jsonb[]
-          )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "BulldozerStorageEngine" AS "groupRow"
-              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[]
-                AND NOT EXISTS (
-                  SELECT 1
-                  FROM "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "deletingRow"
-                  WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-1-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."groupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
-                )
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS "insertingRow"
-              WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-            )
-        
-),
-"flat_map_changes_ag6q3tne5fy9c7mbbr325v6hy6g60scj9rrntqx598s30" AS (
-  
-          SELECT
-            COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
-            COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
-            CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
-          FROM "old_flat_rows_c387p2qzpg01std6py4vzn1y3j9y19wx690yeveywhspg" AS "oldRows"
-          FULL OUTER JOIN "new_flat_rows_b05fbprh41k3y6awcxh9tx83nbgzzfky5jkwfz18gveg8" AS "newRows"
-            ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
-            AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
-          WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
-        
-),
-"mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            "changes"."rowIdentifier" AS "sourceRowIdentifier",
-            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
-            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
-            "oldMapped"."rows" AS "oldMappedRows",
-            "newMapped"."rows" AS "newMappedRows"
-          FROM "flat_map_changes_ag6q3tne5fy9c7mbbr325v6hy6g60scj9rrntqx598s30" AS "changes"
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."rows" AS "rows"
-            FROM (
-              SELECT 
-        jsonb_build_array(
-          COALESCE(
-            (
-              SELECT to_jsonb("mapped")
-              FROM (
-                SELECT ("rowData"->'team') AS "team", (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", (CASE WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' ELSE 'low' END) AS "bucket"
-              ) AS "mapped"
-            ),
-            'null'::jsonb
-          )
-        ) AS "rows"
-      
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."oldRowData" AS "rowData"
-              ) AS "mapperInput"
-            ) AS "mapped"
-          ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."rows" AS "rows"
-            FROM (
-              SELECT 
-        jsonb_build_array(
-          COALESCE(
-            (
-              SELECT to_jsonb("mapped")
-              FROM (
-                SELECT ("rowData"->'team') AS "team", (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", (CASE WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' ELSE 'low' END) AS "bucket"
-              ) AS "mapped"
-            ),
-            'null'::jsonb
-          )
-        ) AS "rows"
-      
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."newRowData" AS "rowData"
-              ) AS "mapperInput"
-            ) AS "mapped"
-          ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
-          WHERE 
-      EXISTS (
-        SELECT 1 FROM "BulldozerStorageEngine"
-        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
-      )
-    
-        
-),
-"old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
-            "flatRow"."rowData" AS "rowData"
-          FROM "mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS "changes"
-          CROSS JOIN LATERAL jsonb_array_elements(
-            CASE
-              WHEN "changes"."hasOldRow" THEN (
-                CASE
-                  WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows"
-                  ELSE '[]'::jsonb
-                END
-              )
-              ELSE '[]'::jsonb
-            END
-          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
-        
-),
-"new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS (
-  
-          SELECT
-            "changes"."groupKey" AS "groupKey",
-            ("changes"."sourceRowIdentifier" || ':' || ("flatRow"."flatIndex"::text)) AS "rowIdentifier",
-            "flatRow"."rowData" AS "rowData"
-          FROM "mapped_changes_hpypfyd2pxhhmb2x1tpgwjfq3sydfhqg37vct76x43zrg" AS "changes"
-          CROSS JOIN LATERAL jsonb_array_elements(
-            CASE
-              WHEN "changes"."hasNewRow" THEN (
-                CASE
-                  WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows"
-                  ELSE '[]'::jsonb
-                END
-              )
-              ELSE '[]'::jsonb
-            END
-          ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex")
-        
-),
-"unnamed_statement_6wwsxhrt" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            "insertRows"."keyPath",
-            "insertRows"."value"
-          FROM (
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey"]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
-            UNION
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
-          ) AS "insertRows"
-          ON CONFLICT ("keyPath") DO NOTHING
-        
-),
-"unnamed_statement_rmr146nf" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "target"
-          USING "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "changes"
-          WHERE "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
-        
-),
-"unnamed_statement_d4mggz1p" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "groupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
-            jsonb_build_object('rowData', "rowData")
-          FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg"
-          ON CONFLICT ("keyPath") DO UPDATE
-          SET "value" = EXCLUDED."value"
-        
-),
-"unnamed_statement_q6c1gdbq" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
-          USING "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "changes"
-          WHERE "staleGroupPath"."keyPath" IN (
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[],
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey"]::jsonb[]
-          )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "BulldozerStorageEngine" AS "groupRow"
-              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."groupKey", '"rows"'::jsonb]::jsonb[]
-                AND NOT EXISTS (
-                  SELECT 1
-                  FROM "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "deletingRow"
-                  WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-map-level-2-101"'::jsonb, '"table"'::jsonb, '"internal:map"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."groupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
-                )
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS "insertingRow"
-              WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey"
-            )
-        
-),
-"flat_map_changes_hfe1g0gz05jxzx6j8ma5dv2n1fegvheyr3hrpp5rtgx78" AS (
-  
-          SELECT
-            COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey",
-            COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData",
-            CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData"
-          FROM "old_flat_rows_jqg9m7ygb2dprm9fkcjnd73w4wjyjr6m305c3nd7zrdtr" AS "oldRows"
-          FULL OUTER JOIN "new_flat_rows_81x457bywzj9cmmwrg0a75328rvf1aa0tqd4n9s1vtbqg" AS "newRows"
-            ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey"
-            AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier"
-          WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData"
-        
-),
-"mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS (
-  
-          SELECT
-            "changes"."rowIdentifier" AS "rowIdentifier",
-            "changes"."oldRowData" AS "oldRowData",
-            "changes"."newRowData" AS "newRowData",
-            ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow",
-            ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow",
-            "oldGroup"."groupKey" AS "oldGroupKey",
-            "newGroup"."groupKey" AS "newGroupKey"
-          FROM "flat_map_changes_hfe1g0gz05jxzx6j8ma5dv2n1fegvheyr3hrpp5rtgx78" AS "changes"
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."groupKey"
-            FROM (
-              SELECT "rowData"->'bucket' AS "groupKey"
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."oldRowData" AS "rowData"
-              ) AS "groupByInput"
-            ) AS "mapped"
-          ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object')
-          LEFT JOIN LATERAL (
-            SELECT "mapped"."groupKey"
-            FROM (
-              SELECT "rowData"->'bucket' AS "groupKey"
-              FROM (
-                SELECT
-                  "changes"."rowIdentifier" AS "rowIdentifier",
-                  "changes"."newRowData" AS "rowData"
-              ) AS "groupByInput"
-            ) AS "mapped"
-          ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object')
-          WHERE 
-      EXISTS (
-        SELECT 1 FROM "BulldozerStorageEngine"
-        WHERE "keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"metadata"'::jsonb]::jsonb[]
-      )
-    
-        
-),
-"unnamed_statement_br2pqz8w" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            "insertRows"."keyPath",
-            "insertRows"."value"
-          FROM (
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey"]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
-            WHERE "hasNewRow"
-            UNION
-            SELECT DISTINCT
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb]::jsonb[] AS "keyPath",
-              'null'::jsonb AS "value"
-            FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
-            WHERE "hasNewRow"
-          ) AS "insertRows"
-          ON CONFLICT ("keyPath") DO NOTHING
-        
-),
-"unnamed_statement_zvcwftv3" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "target"
-          USING "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "changes"
-          WHERE "changes"."hasOldRow"
-            AND "target"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb, to_jsonb("changes"."rowIdentifier"::text)]::jsonb[]
-        
-),
-"unnamed_statement_e5kztjcm" AS (
-  
-          INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
-          SELECT
-            gen_random_uuid(),
-            ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "newGroupKey", '"rows"'::jsonb, to_jsonb("rowIdentifier"::text)]::jsonb[],
-            jsonb_build_object('rowData', "newRowData")
-          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
-          WHERE "hasNewRow"
-          ON CONFLICT ("keyPath") DO UPDATE
-          SET "value" = EXCLUDED."value"
-        
-),
-"unnamed_statement_fmaqky9q" AS (
-  
-          DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath"
-          USING "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "changes"
-          WHERE "changes"."hasOldRow"
-            AND "staleGroupPath"."keyPath" IN (
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[],
-              ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey"]::jsonb[]
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "BulldozerStorageEngine" AS "groupRow"
-              WHERE "groupRow"."keyPathParent" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "changes"."oldGroupKey", '"rows"'::jsonb]::jsonb[]
-                AND NOT EXISTS (
-                  SELECT 1
-                  FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "deletingRow"
-                  WHERE "deletingRow"."hasOldRow"
-                    AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
-                    AND "groupRow"."keyPath" = ARRAY['"table"'::jsonb, '"external:fuzz-users-by-bucket-101"'::jsonb, '"storage"'::jsonb, '"groups"'::jsonb, "deletingRow"."oldGroupKey", '"rows"'::jsonb, to_jsonb("deletingRow"."rowIdentifier"::text)]::jsonb[]
-                )
-            )
-            AND NOT EXISTS (
-              SELECT 1
-              FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g" AS "insertingRow"
-              WHERE "insertingRow"."hasNewRow"
-                AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey"
-            )
-        
-),
-"grouped_changes_e4p857w4kzj7vda37xcp61a730wq7472960nn1p1xz03g" AS (
-  
-          SELECT
-            "oldGroupKey" AS "groupKey",
-            "rowIdentifier" AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            "oldRowData" AS "oldRowData",
-            CASE
-              WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData"
-              ELSE 'null'::jsonb
-            END AS "newRowData"
-          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
-          WHERE "hasOldRow"
-          UNION ALL
-          SELECT
-            "newGroupKey" AS "groupKey",
-            "rowIdentifier" AS "rowIdentifier",
-            'null'::jsonb AS "oldRowSortKey",
-            'null'::jsonb AS "newRowSortKey",
-            'null'::jsonb AS "oldRowData",
-            "newRowData" AS "newRowData"
-          FROM "mapped_changes_9hat6erjww49qyv9pavsrffg590njzr9z5xpdja4h7t9g"
-          WHERE "hasNewRow"
-            AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey")
-        
-),
-__dummy_statement_2__ AS (SELECT 1)
-SELECT 1;
-
-COMMIT;
\ No newline at end of file

From 69b3d4f9d286319bf96db4de0ddefafe638e521b Mon Sep 17 00:00:00 2001
From: Konstantin Wohlwend 
Date: Wed, 25 Mar 2026 19:30:11 -0700
Subject: [PATCH 18/40] Bulldozer Studio: Better node placing algorithm

---
 apps/backend/package.json                    |   1 +
 apps/backend/scripts/run-bulldozer-studio.ts | 114 +++++++++++++++++--
 pnpm-lock.yaml                               |  85 +++++++-------
 3 files changed, 152 insertions(+), 48 deletions(-)

diff --git a/apps/backend/package.json b/apps/backend/package.json
index 139951b88a..90da3f1b88 100644
--- a/apps/backend/package.json
+++ b/apps/backend/package.json
@@ -94,6 +94,7 @@
     "chokidar-cli": "^3.0.0",
     "dotenv": "^16.4.5",
     "dotenv-cli": "^7.3.0",
+    "elkjs": "^0.11.1",
     "emailable": "^3.1.1",
     "freestyle-sandboxes": "^0.1.6",
     "jiti": "^2.6.1",
diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts
index cbca3516dc..1d66c037f9 100644
--- a/apps/backend/scripts/run-bulldozer-studio.ts
+++ b/apps/backend/scripts/run-bulldozer-studio.ts
@@ -1,6 +1,7 @@
 import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
 import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
 import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings";
+import ELK from "elkjs/lib/elk.bundled.js";
 import http from "node:http";
 import { exampleFungibleLedgerSchema } from "../src/lib/bulldozer/db/example-schema";
 import { toQueryableSqlQuery } from "../src/lib/bulldozer/db/index";
@@ -38,6 +39,12 @@ type StudioTableRecord = {
 const STUDIO_PORT = Number(`${getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81")}39`);
 const BULLDOZER_LOCK_ID = 7857391;
 const STUDIO_INSTANCE_ID = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+const GRAPH_NODE_WIDTH = 260;
+const GRAPH_NODE_HEIGHT = 126;
+const GRAPH_LEVEL_GAP_Y = 230;
+const GRAPH_COLUMN_GAP_X = 320;
+const GRAPH_SCENE_MARGIN = 40;
+const elk = new ELK();
 
 function isRecord(value: unknown): value is Record {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -197,6 +204,58 @@ async function getTableSnapshot(record: StudioTableRecord): Promise<{
   };
 }
 
+async function computeStudioLayout(tables: Array>>): Promise,
+  sceneWidth: number,
+  sceneHeight: number,
+}> {
+  try {
+    const layout = await elk.layout({
+      id: "bulldozer-studio",
+      layoutOptions: {
+        "elk.algorithm": "layered",
+        "elk.direction": "DOWN",
+        "elk.padding": `[top=${GRAPH_SCENE_MARGIN},left=${GRAPH_SCENE_MARGIN},bottom=${GRAPH_SCENE_MARGIN},right=${GRAPH_SCENE_MARGIN}]`,
+        "elk.spacing.nodeNode": String(Math.floor(GRAPH_COLUMN_GAP_X / 2)),
+        "elk.layered.spacing.nodeNodeBetweenLayers": String(Math.floor(GRAPH_LEVEL_GAP_Y / 2)),
+        "elk.layered.crossingMinimization.strategy": "LAYER_SWEEP",
+        "elk.layered.nodePlacement.strategy": "NETWORK_SIMPLEX",
+        "elk.layered.considerModelOrder.strategy": "NODES_AND_EDGES",
+        "elk.layered.thoroughness": "40",
+      },
+      children: tables.map((table) => ({
+        id: table.id,
+        width: GRAPH_NODE_WIDTH,
+        height: GRAPH_NODE_HEIGHT,
+      })),
+      edges: tables.flatMap((table) => {
+        return table.dependencies.map((dependencyId, index) => ({
+          id: `${dependencyId}->${table.id}:${index}`,
+          sources: [dependencyId],
+          targets: [table.id],
+        }));
+      }),
+    });
+
+    const positions: Record = {};
+    for (const child of layout.children ?? []) {
+      if (typeof child.id !== "string") continue;
+      positions[child.id] = {
+        x: Number(child.x ?? 0),
+        y: Number(child.y ?? 0),
+      };
+    }
+
+    return {
+      positions,
+      sceneWidth: Number(Reflect.get(layout, "width") ?? 600),
+      sceneHeight: Number(Reflect.get(layout, "height") ?? 600),
+    };
+  } catch (error) {
+    return null;
+  }
+}
+
 async function getTableDetails(record: StudioTableRecord): Promise<{
   table: Awaited>,
   groups: Array<{ groupKey: unknown, rows: Array<{ rowIdentifier: unknown, rowSortKey: unknown, rowData: unknown }> }>,
@@ -736,11 +795,11 @@ function getStudioPageHtml(): string {