diff --git a/.vscode/settings.json b/.vscode/settings.json index 9937c7581f..72e3892e83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,10 +7,6 @@ "typescript.tsdk": "node_modules/typescript/lib", "editor.tabSize": 2, "cSpell.words": [ - "glassmorphic", - "sparkline", - "Clickhouse", - "pushable", "autoupdate", "backlinks", "Cancelation", @@ -19,14 +15,15 @@ "chinthakagodawita", "Ciphertext", "cjsx", + "Clickhouse", "clsx", - "dbgenerated", "cmdk", "codegen", "crockford", "Crudl", "ctsx", "datapoints", + "dbgenerated", "deindent", "Deindentable", "deindented", @@ -42,6 +39,7 @@ "fkey", "frontends", "geoip", + "glassmorphic", "healthcheck", "hookform", "hostable", @@ -51,6 +49,7 @@ "JWTs", "katex", "localstack", + "ltree", "lucide", "Luma", "midfix", @@ -64,6 +63,7 @@ "nicified", "nicify", "oidc", + "onnotice", "openapi", "opentelemetry", "otel", @@ -81,6 +81,7 @@ "Prefetchers", "Proxied", "psql", + "pushable", "qrcode", "QSTASH", "quetzallabs", @@ -89,6 +90,7 @@ "retryable", "RPID", "simplewebauthn", + "sparkline", "spoofable", "stackable", "stackauth", @@ -109,6 +111,7 @@ "Unmigrated", "unsubscribers", "upsert", + "upserted", "Upvotes", "upvoting", "webapi", diff --git a/AGENTS.md b/AGENTS.md index fde90ebf10..30a32c4458 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ This file provides guidance to coding agents when working with code in this repo ### Essential Commands - **Install dependencies**: `pnpm install` - **Run tests**: `pnpm test run` (uses Vitest). You can filter with `pnpm test run `. The `run` is important to not trigger watch mode -- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually. +- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually. Use `pnpm -C lint` to lint a specific package. - **Type check**: `pnpm typecheck` #### Extra commands @@ -108,6 +108,7 @@ To see all development ports, refer to the index.html of `apps/dev-launchpad/pub - Any design components you add or modify in the dashboard, update the Playground page accordingly to showcase the changes. - Unless very clearly equivalent from types, prefer explicit null/undefinedness checks over boolean checks, eg. `foo == null` instead of `!foo`. - Ensure **aggressively** that all code has low coupling and high cohesion. This is really important as it makes sure our code remains consistent and maintainable. Eagerly refactor things into better abstractions and look out for them actively. +- Always let me know about the tradeoffs and decisions you make while implementing a non-trivial change. ### Code-related - Use ES6 maps instead of records wherever you can. diff --git a/apps/backend/package.json b/apps/backend/package.json index 8d74706df2..374626ad13 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -11,7 +11,7 @@ "with-env:dev": "dotenv -c development --", "with-env:prod": "dotenv -c production --", "with-env:test": "dotenv -c test --", - "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"", + "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs,bulldozer-studio\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\" \"pnpm run run-bulldozer-studio\"", "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev", "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev", "build": "pnpm run codegen && next build", @@ -48,7 +48,8 @@ "run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts", "run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts", "verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts", - "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts" + "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts", + "run-bulldozer-studio": "pnpm run with-env:dev tsx watch --clear-screen=false scripts/run-bulldozer-studio.ts" }, "prisma": { "seed": "pnpm run db-seed-script" @@ -93,6 +94,7 @@ "chokidar-cli": "^3.0.0", "dotenv": "^16.4.5", "dotenv-cli": "^7.3.0", + "elkjs": "^0.11.1", "emailable": "^3.1.1", "freestyle-sandboxes": "^0.1.6", "jiti": "^2.6.1", diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql new file mode 100644 index 0000000000..388283cfcb --- /dev/null +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/migration.sql @@ -0,0 +1,31 @@ +-- CreateTable +CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE +); + +-- Seed root hierarchy rows used by all tables. +INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") +VALUES + ('00000000-0000-0000-0000-000000000100'::uuid, ARRAY[]::jsonb[], 'null'::jsonb); + +INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") +VALUES + ('00000000-0000-0000-0000-000000000101'::uuid, ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb); + +-- CreateIndex +CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent"); diff --git a/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts new file mode 100644 index 0000000000..bfcb577b1e --- /dev/null +++ b/apps/backend/prisma/migrations/20260323120000_add_bulldozer_data/tests/ltree-queries.ts @@ -0,0 +1,113 @@ +import type { Sql } from "postgres"; +import { expect } from "vitest"; + +export const postMigration = async (sql: Sql) => { + await sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + ('00000000-0000-0000-0000-000000000001'::uuid, ARRAY[to_jsonb('root'::text)]::jsonb[], '{"node":"root"}'::jsonb), + ('00000000-0000-0000-0000-000000000002'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[], '{"node":"branch"}'::jsonb), + ('00000000-0000-0000-0000-000000000003'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[], '{"node":"leaf"}'::jsonb), + ('00000000-0000-0000-0000-000000000004'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('other'::text)]::jsonb[], '{"node":"other"}'::jsonb) + `; + + const exactRows = await sql` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[] + `; + + expect(exactRows).toHaveLength(1); + expect(exactRows[0].value).toEqual({ node: "leaf" }); + + const nestedRows = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath"[1:cardinality(ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[])] = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[] + ORDER BY "keyPath" + `; + + expect(nestedRows.map((row) => row.keyPath)).toEqual([ + "root.branch", + "root.branch.leaf", + ]); + + const directChildrenRows = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ARRAY[to_jsonb('root'::text)]::jsonb[] + ORDER BY "keyPath" + `; + + expect(directChildrenRows.map((row) => row.keyPath)).toEqual([ + "root.branch", + "root.other", + ]); + + const indexRows = await sql` + SELECT "indexname" + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = 'BulldozerStorageEngine' + AND indexname IN ( + 'BulldozerStorageEngine_keyPath_key', + 'BulldozerStorageEngine_keyPathParent_idx' + ) + ORDER BY "indexname" + `; + + expect(indexRows.map((row) => row.indexname)).toEqual([ + "BulldozerStorageEngine_keyPathParent_idx", + "BulldozerStorageEngine_keyPath_key", + ]); + + const seededRootRows = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (ARRAY[]::jsonb[], ARRAY[to_jsonb('table'::text)]::jsonb[]) + ORDER BY cardinality("keyPath") + `; + + expect(seededRootRows.map((row) => row.keyPath)).toEqual([ + "", + "table", + ]); + + const generatedColumnRows = await sql` + SELECT attname + FROM pg_attribute + WHERE attrelid = 'public."BulldozerStorageEngine"'::regclass + AND attname = 'keyPathParent' + AND attgenerated = 's' + `; + + expect(generatedColumnRows).toHaveLength(1); + + const fkConstraintRows = await sql` + SELECT conname + FROM pg_constraint + WHERE conrelid = 'public."BulldozerStorageEngine"'::regclass + AND conname = 'BulldozerStorageEngine_keyPathParent_fkey' + `; + + expect(fkConstraintRows).toHaveLength(1); + + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "keyPathParent", "value") + VALUES ( + '00000000-0000-0000-0000-000000000005'::uuid, + ARRAY[to_jsonb('root'::text), to_jsonb('mismatch'::text)]::jsonb[], + ARRAY[]::jsonb[], + '{"node":"invalid"}'::jsonb + ) + `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); + + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + '00000000-0000-0000-0000-000000000006'::uuid, + ARRAY[to_jsonb('missing-parent'::text), to_jsonb('child'::text)]::jsonb[], + '{"node":"invalid-fk"}'::jsonb + ) + `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); +}; diff --git a/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/migration.sql b/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/migration.sql new file mode 100644 index 0000000000..bba06822a4 --- /dev/null +++ b/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/migration.sql @@ -0,0 +1,295 @@ +-- CreateTable +CREATE TABLE "BulldozerTimeFoldQueue" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "tableStoragePath" JSONB[] NOT NULL, + "groupKey" JSONB NOT NULL, + "rowIdentifier" TEXT NOT NULL, + "scheduledAt" TIMESTAMPTZ NOT NULL, + "stateAfter" JSONB NOT NULL, + "rowData" JSONB NOT NULL, + "reducerSql" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "BulldozerTimeFoldQueue_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "BulldozerTimeFoldMetadata" ( + "key" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastProcessedAt" TIMESTAMPTZ NOT NULL, + + CONSTRAINT "BulldozerTimeFoldMetadata_pkey" PRIMARY KEY ("key") +); + +-- Seed singleton metadata row. +INSERT INTO "BulldozerTimeFoldMetadata" ("key", "lastProcessedAt") +VALUES ('singleton', now()) +ON CONFLICT ("key") DO NOTHING; + +-- CreateIndex +CREATE UNIQUE INDEX "BulldozerTimeFoldQueue_table_group_row_key" + ON "BulldozerTimeFoldQueue"("tableStoragePath", "groupKey", "rowIdentifier"); + +-- CreateIndex +CREATE INDEX "BulldozerTimeFoldQueue_scheduledAt_idx" + ON "BulldozerTimeFoldQueue"("scheduledAt"); + +-- Worker function used by pg_cron and callable manually in tests. +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +CREATE OR REPLACE FUNCTION public.bulldozer_timefold_process_queue() +RETURNS void +LANGUAGE plpgsql +AS $function$ +DECLARE + cutoff_timestamp timestamptz; + queued_row "BulldozerTimeFoldQueue"%ROWTYPE; + group_path jsonb[]; + rows_path jsonb[]; + states_path jsonb[]; + state_row_path jsonb[]; + existing_state jsonb; + old_emitted_rows jsonb; + newly_emitted_rows jsonb; + accumulated_emitted_rows jsonb; + current_state jsonb; + current_timestamp_value timestamptz; + next_state jsonb; + next_rows_data jsonb; + normalized_next_rows_data jsonb; + next_timestamp timestamptz; + previous_emitted_row_count int; + reducer_iterations int; + new_row_record record; +BEGIN + PERFORM pg_advisory_xact_lock(7857391); + + INSERT INTO "BulldozerTimeFoldMetadata" ("key", "lastProcessedAt") + VALUES ('singleton', now()) + ON CONFLICT ("key") DO NOTHING; + + cutoff_timestamp := now(); + + UPDATE "BulldozerTimeFoldMetadata" + SET + "lastProcessedAt" = cutoff_timestamp, + "updatedAt" = CURRENT_TIMESTAMP + WHERE "key" = 'singleton'; + + LOOP + SELECT * + INTO queued_row + FROM "BulldozerTimeFoldQueue" + WHERE "scheduledAt" <= cutoff_timestamp + ORDER BY "scheduledAt" ASC, "id" ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED; + + EXIT WHEN NOT FOUND; + + DELETE FROM "BulldozerTimeFoldQueue" + WHERE "id" = queued_row."id"; + + group_path := queued_row."tableStoragePath" || ARRAY[to_jsonb('groups'::text), queued_row."groupKey"]::jsonb[]; + rows_path := group_path || ARRAY[to_jsonb('rows'::text)]::jsonb[]; + states_path := group_path || ARRAY[to_jsonb('states'::text)]::jsonb[]; + state_row_path := states_path || ARRAY[to_jsonb(queued_row."rowIdentifier")]::jsonb[]; + + SELECT "value" + INTO existing_state + FROM "BulldozerStorageEngine" + WHERE "keyPath" = state_row_path; + + IF existing_state IS NULL THEN + CONTINUE; + END IF; + + IF existing_state->'rowData' IS DISTINCT FROM queued_row."rowData" THEN + CONTINUE; + END IF; + + old_emitted_rows := CASE + WHEN jsonb_typeof(existing_state->'emittedRowsData') = 'array' THEN existing_state->'emittedRowsData' + ELSE '[]'::jsonb + END; + newly_emitted_rows := '[]'::jsonb; + accumulated_emitted_rows := old_emitted_rows; + previous_emitted_row_count := jsonb_array_length(old_emitted_rows); + + current_state := queued_row."stateAfter"; + current_timestamp_value := queued_row."scheduledAt"; + reducer_iterations := 0; + + LOOP + reducer_iterations := reducer_iterations + 1; + IF reducer_iterations > 10000 THEN + RAISE EXCEPTION 'bulldozer timefold reducer exceeded 10k iterations for row %', queued_row."rowIdentifier"; + END IF; + + EXECUTE format( + $reducer$ + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + to_jsonb("reducerRows"."newRowsData") AS "newRowsData", + CASE + WHEN "reducerRows"."nextTimestamp" IS NULL THEN NULL::timestamptz + ELSE ("reducerRows"."nextTimestamp")::timestamptz + END AS "nextTimestamp" + FROM ( + SELECT %s + FROM ( + SELECT + $1::jsonb AS "oldState", + $2::jsonb AS "oldRowData", + $3::timestamptz AS "timestamp" + ) AS "reducerInput" + ) AS "reducerRows" + $reducer$, + queued_row."reducerSql" + ) + INTO next_state, next_rows_data, next_timestamp + USING current_state, queued_row."rowData", current_timestamp_value; + + normalized_next_rows_data := CASE + WHEN jsonb_typeof(next_rows_data) = 'array' THEN next_rows_data + ELSE '[]'::jsonb + END; + newly_emitted_rows := newly_emitted_rows || normalized_next_rows_data; + accumulated_emitted_rows := accumulated_emitted_rows || normalized_next_rows_data; + current_state := next_state; + + EXIT WHEN next_timestamp IS NULL OR next_timestamp > cutoff_timestamp; + current_timestamp_value := next_timestamp; + END LOOP; + + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), group_path, 'null'::jsonb), + (gen_random_uuid(), rows_path, 'null'::jsonb), + (gen_random_uuid(), states_path, 'null'::jsonb) + ON CONFLICT ("keyPath") DO NOTHING; + + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + state_row_path, + jsonb_build_object( + 'rowData', queued_row."rowData", + 'stateAfter', current_state, + 'emittedRowsData', accumulated_emitted_rows, + 'nextTimestamp', + CASE + WHEN next_timestamp IS NULL THEN 'null'::jsonb + ELSE to_jsonb(next_timestamp) + END + ) + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value"; + + FOR new_row_record IN + SELECT + "rows"."rowData" AS "rowData", + "rows"."rowIndex" AS "rowIndex" + FROM jsonb_array_elements(newly_emitted_rows) WITH ORDINALITY AS "rows"("rowData", "rowIndex") + LOOP + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + rows_path || ARRAY[to_jsonb((queued_row."rowIdentifier" || ':' || (previous_emitted_row_count + new_row_record."rowIndex")::text)::text)]::jsonb[], + jsonb_build_object('rowData', new_row_record."rowData") + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value"; + END LOOP; + + IF next_timestamp IS NOT NULL AND next_timestamp > cutoff_timestamp THEN + INSERT INTO "BulldozerTimeFoldQueue" ( + "id", + "tableStoragePath", + "groupKey", + "rowIdentifier", + "scheduledAt", + "stateAfter", + "rowData", + "reducerSql" + ) + VALUES ( + gen_random_uuid(), + queued_row."tableStoragePath", + queued_row."groupKey", + queued_row."rowIdentifier", + next_timestamp, + current_state, + queued_row."rowData", + queued_row."reducerSql" + ) + ON CONFLICT ("tableStoragePath", "groupKey", "rowIdentifier") DO UPDATE + SET + "scheduledAt" = EXCLUDED."scheduledAt", + "stateAfter" = EXCLUDED."stateAfter", + "rowData" = EXCLUDED."rowData", + "reducerSql" = EXCLUDED."reducerSql", + "updatedAt" = CURRENT_TIMESTAMP; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = rows_path + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = states_path + ) + THEN + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (rows_path, states_path, group_path); + END IF; + END LOOP; +END; +$function$; +-- SPLIT_STATEMENT_SENTINEL + +-- Require pg_cron setup. We fail fast if extension setup is unavailable. +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +DO $$ +BEGIN + CREATE EXTENSION IF NOT EXISTS pg_cron; +EXCEPTION + WHEN insufficient_privilege OR undefined_file OR feature_not_supported OR object_not_in_prerequisite_state OR raise_exception THEN + RAISE EXCEPTION 'Failed to set up pg_cron extension for bulldozer timefold worker.' + USING DETAIL = SQLERRM; +END +$$; +-- SPLIT_STATEMENT_SENTINEL + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +DO $$ +BEGIN + IF to_regnamespace('cron') IS NULL THEN + RETURN; + END IF; + + PERFORM cron.unschedule("jobid") + FROM cron.job + WHERE "jobname" = 'bulldozer-timefold-worker'; + + PERFORM cron.schedule( + 'bulldozer-timefold-worker', + '1 second', + 'SELECT public.bulldozer_timefold_process_queue();' + ); +EXCEPTION + WHEN insufficient_privilege OR undefined_function OR feature_not_supported THEN + RAISE EXCEPTION 'Failed to schedule pg_cron bulldozer timefold worker.' + USING DETAIL = SQLERRM; +END +$$; +-- SPLIT_STATEMENT_SENTINEL diff --git a/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/tests/process-queue.ts b/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/tests/process-queue.ts new file mode 100644 index 0000000000..8474180c68 --- /dev/null +++ b/apps/backend/prisma/migrations/20260323150000_add_bulldozer_timefold_queue/tests/process-queue.ts @@ -0,0 +1,129 @@ +import type { Sql } from "postgres"; +import { expect } from "vitest"; + +export const postMigration = async (sql: Sql) => { + const tableStoragePathSql = `ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:test-timefold'::text), + to_jsonb('storage'::text) + ]::jsonb[]`; + const groupsPathSql = `${tableStoragePathSql} || ARRAY[to_jsonb('groups'::text)]::jsonb[]`; + const groupPathSql = `${groupsPathSql} || ARRAY[to_jsonb('alpha'::text)]::jsonb[]`; + const rowsPathSql = `${groupPathSql} || ARRAY[to_jsonb('rows'::text)]::jsonb[]`; + const statesPathSql = `${groupPathSql} || ARRAY[to_jsonb('states'::text)]::jsonb[]`; + const stateRowPathSql = `${statesPathSql} || ARRAY[to_jsonb('u1'::text)]::jsonb[]`; + const oldOutputPathSql = `${rowsPathSql} || ARRAY[to_jsonb('u1:1'::text)]::jsonb[]`; + + await sql.unsafe(` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ARRAY[to_jsonb('table'::text), to_jsonb('external:test-timefold'::text)]::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${tableStoragePathSql}, 'null'::jsonb), + (gen_random_uuid(), ${groupsPathSql}, 'null'::jsonb), + (gen_random_uuid(), ${groupPathSql}, 'null'::jsonb), + (gen_random_uuid(), ${rowsPathSql}, 'null'::jsonb), + (gen_random_uuid(), ${statesPathSql}, 'null'::jsonb) + ON CONFLICT ("keyPath") DO NOTHING + `); + + await sql.unsafe(` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + ( + gen_random_uuid(), + ${stateRowPathSql}, + jsonb_build_object( + 'rowData', '{"value": 2}'::jsonb, + 'stateAfter', '{"counter": 1}'::jsonb, + 'emittedRowsData', jsonb_build_array(jsonb_build_object('value', 100)), + 'nextTimestamp', 'null'::jsonb + ) + ), + ( + gen_random_uuid(), + ${oldOutputPathSql}, + jsonb_build_object('rowData', jsonb_build_object('value', 100)) + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `); + + await sql.unsafe(` + INSERT INTO "BulldozerTimeFoldQueue" ( + "id", + "tableStoragePath", + "groupKey", + "rowIdentifier", + "scheduledAt", + "stateAfter", + "rowData", + "reducerSql" + ) + VALUES ( + gen_random_uuid(), + ${tableStoragePathSql}, + to_jsonb('alpha'::text), + 'u1', + now() - interval '1 minute', + '{"counter": 1}'::jsonb, + '{"value": 2}'::jsonb, + 'jsonb_build_object(''counter'', COALESCE(("oldState"->>''counter'')::int, 0) + (("oldRowData"->>''value'')::int)) AS "newState", jsonb_build_array(jsonb_build_object(''value'', (("oldRowData"->>''value'')::int), ''counter'', COALESCE(("oldState"->>''counter'')::int, 0) + (("oldRowData"->>''value'')::int))) AS "newRowsData", ("timestamp" + interval ''1 day'') AS "nextTimestamp"' + ) + ON CONFLICT ("tableStoragePath", "groupKey", "rowIdentifier") DO UPDATE + SET + "scheduledAt" = EXCLUDED."scheduledAt", + "stateAfter" = EXCLUDED."stateAfter", + "rowData" = EXCLUDED."rowData", + "reducerSql" = EXCLUDED."reducerSql", + "updatedAt" = CURRENT_TIMESTAMP + `); + + await sql.unsafe(`SELECT public.bulldozer_timefold_process_queue()`); + + const stateRows = await sql.unsafe(` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${stateRowPathSql} + `); + expect(stateRows).toHaveLength(1); + expect(stateRows[0].value).toEqual({ + rowData: { value: 2 }, + stateAfter: { counter: 3 }, + emittedRowsData: [{ value: 100 }, { value: 2, counter: 3 }], + nextTimestamp: expect.any(String), + }); + + const oldOutputRows = await sql.unsafe(` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${oldOutputPathSql} + `); + expect(oldOutputRows).toHaveLength(1); + expect(oldOutputRows[0].value).toEqual({ rowData: { value: 100 } }); + + const newOutputRows = await sql.unsafe(` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${rowsPathSql} || ARRAY[to_jsonb('u1:2'::text)]::jsonb[] + `); + expect(newOutputRows).toHaveLength(1); + expect(newOutputRows[0].value).toEqual({ rowData: { value: 2, counter: 3 } }); + + const queueRows = await sql.unsafe(` + SELECT "scheduledAt", "stateAfter" + FROM "BulldozerTimeFoldQueue" + WHERE "tableStoragePath" = ${tableStoragePathSql} + AND "groupKey" = to_jsonb('alpha'::text) + AND "rowIdentifier" = 'u1' + `); + expect(queueRows).toHaveLength(1); + expect(queueRows[0].stateAfter).toEqual({ counter: 3 }); + + const metadataRows = await sql.unsafe(` + SELECT "lastProcessedAt" + FROM "BulldozerTimeFoldMetadata" + WHERE "key" = 'singleton' + `); + expect(metadataRows).toHaveLength(1); + expect(new Date(metadataRows[0].lastProcessedAt).getTime()).toBeGreaterThan(Date.now() - 60_000); +}; diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 48403f9daf..af8c27ed94 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -276,15 +276,15 @@ model ProjectUser { restrictedByAdminPrivateDetails String? // Private details (server access only) // Sign-up metadata - signedUpAt DateTime @default(now()) - signUpIp String? - signUpIpTrusted Boolean? - signUpEmailNormalized String? - signUpEmailBase String? + signedUpAt DateTime @default(now()) + signUpIp String? + signUpIpTrusted Boolean? + signUpEmailNormalized String? + signUpEmailBase String? // Sign-up risk scores (0-100, set at sign-up time) - signUpRiskScoreBot Int @default(0) @db.SmallInt - signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt + signUpRiskScoreBot Int @default(0) @db.SmallInt + signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt projectUserOAuthAccounts ProjectUserOAuthAccount[] teamMembers TeamMember[] @@ -1247,6 +1247,39 @@ model OutgoingRequest { @@index([startedFulfillingAt, deduplicationKey]) } +model BulldozerStorageEngine { + id String @id @default(uuid()) @db.Uuid + keyPath Json[] + keyPathParent Unsupported("jsonb[]") + value Json + + @@unique([keyPath], map: "BulldozerStorageEngine_keyPath_key") + @@index([keyPathParent], map: "BulldozerStorageEngine_keyPathParent_idx") +} + +model BulldozerTimeFoldQueue { + id String @id @default(uuid()) @db.Uuid + tableStoragePath Json[] + groupKey Json + rowIdentifier String + scheduledAt DateTime + stateAfter Json + rowData Json + reducerSql String + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + + @@unique([tableStoragePath, groupKey, rowIdentifier], map: "BulldozerTimeFoldQueue_table_group_row_key") + @@index([scheduledAt], map: "BulldozerTimeFoldQueue_scheduledAt_idx") +} + +model BulldozerTimeFoldMetadata { + key String @id + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + lastProcessedAt DateTime +} + model DeletedRow { id String @id @default(uuid()) @db.Uuid tenancyId String @db.Uuid diff --git a/apps/backend/scripts/run-bulldozer-studio.ts b/apps/backend/scripts/run-bulldozer-studio.ts new file mode 100644 index 0000000000..33938b3e48 --- /dev/null +++ b/apps/backend/scripts/run-bulldozer-studio.ts @@ -0,0 +1,2651 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings"; +import ELK from "elkjs/lib/elk.bundled.js"; +import http from "node:http"; +import { exampleFungibleLedgerSchema } from "../src/lib/bulldozer/db/example-schema"; +import { toExecutableSqlStatements, toQueryableSqlQuery } from "../src/lib/bulldozer/db/index"; +import { quoteSqlJsonbLiteral } from "../src/lib/bulldozer/db/utilities"; +import { globalPrismaClient, retryTransaction } from "../src/prisma-client"; + +type JsonPrimitive = string | number | boolean | null; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; + +type StudioTable = { + tableId: unknown, + inputTables?: StudioTable[], + debugArgs?: Record, + listGroups(options: { start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + listRowsInGroup(options: { groupKey?: SqlExpression, start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + init(): SqlStatement[], + delete(): SqlStatement[], + isInitialized(): SqlExpression, + registerRowChangeTrigger(trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[]): { deregister: () => void }, +}; + +type StudioStoredTable = StudioTable & { + setRow(rowIdentifier: string, rowData: SqlExpression>): SqlStatement[], + deleteRow(rowIdentifier: string): SqlStatement[], +}; + +type StudioTableRecord = { + id: string, + name: string, + table: StudioTable, +}; + +const STUDIO_PORT = Number(`${getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81")}39`); +const STUDIO_HOST = "127.0.0.1"; +const BULLDOZER_LOCK_ID = 7857391; +const STUDIO_INSTANCE_ID = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +const STUDIO_AUTH_TOKEN = getEnvVariable("STACK_BULLDOZER_STUDIO_AUTH_TOKEN", STUDIO_INSTANCE_ID); +const STUDIO_AUTH_HEADER = "x-stack-bulldozer-studio-token"; +const MAX_REQUEST_BODY_BYTES = 1024 * 1024; +const GRAPH_NODE_WIDTH = 260; +const GRAPH_NODE_HEIGHT = 126; +const GRAPH_LEVEL_GAP_Y = 230; +const GRAPH_COLUMN_GAP_X = 320; +const GRAPH_SCENE_MARGIN = 40; +const elk = new ELK(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isStudioTable(value: unknown): value is StudioTable { + if (!isRecord(value)) return false; + return typeof Reflect.get(value, "listGroups") === "function" + && typeof Reflect.get(value, "listRowsInGroup") === "function" + && typeof Reflect.get(value, "init") === "function" + && typeof Reflect.get(value, "delete") === "function" + && typeof Reflect.get(value, "isInitialized") === "function" + && typeof Reflect.get(value, "registerRowChangeTrigger") === "function"; +} + +function isStudioStoredTable(value: StudioTable): value is StudioStoredTable { + return typeof Reflect.get(value, "setRow") === "function" + && typeof Reflect.get(value, "deleteRow") === "function"; +} + +function requireRecord(value: unknown, errorMessage: string): Record { + if (!isRecord(value)) throw new StackAssertionError(errorMessage); + return value; +} + +function requireString(value: unknown, errorMessage: string): string { + if (typeof value !== "string") throw new StackAssertionError(errorMessage); + return value; +} + +function requireStringArray(value: unknown, errorMessage: string): string[] { + if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) { + throw new StackAssertionError(errorMessage); + } + return value; +} + +function isJsonValue(value: unknown): value is JsonValue { + if ( + value === null + || typeof value === "string" + || typeof value === "number" + || typeof value === "boolean" + ) { + return true; + } + if (Array.isArray(value)) { + return value.every((item) => isJsonValue(item)); + } + if (isRecord(value)) { + return Object.values(value).every((item) => isJsonValue(item)); + } + return false; +} + +function requireJsonValue(value: unknown, errorMessage: string): JsonValue { + if (!isJsonValue(value)) { + throw new StackAssertionError(errorMessage); + } + return value; +} + +function keyPathSqlLiteral(pathSegments: string[]): string { + if (pathSegments.length === 0) return "ARRAY[]::jsonb[]"; + return `ARRAY[${pathSegments.map((segment) => quoteSqlJsonbLiteral(segment).sql).join(", ")}]::jsonb[]`; +} + +function splitSqlStatements(sqlScript: string): string[] { + const statements: string[] = []; + let statementStart = 0; + let index = 0; + let inSingleQuote = false; + let inDoubleQuote = false; + let inLineComment = false; + let blockCommentDepth = 0; + let dollarQuoteTag: null | string = null; + while (index < sqlScript.length) { + const current = sqlScript[index]; + const next = sqlScript[index + 1]; + + if (inLineComment) { + if (current === "\n") inLineComment = false; + index++; + continue; + } + if (blockCommentDepth > 0) { + if (current === "/" && next === "*") { + blockCommentDepth++; + index += 2; + continue; + } + if (current === "*" && next === "/") { + blockCommentDepth--; + index += 2; + continue; + } + index++; + continue; + } + if (dollarQuoteTag !== null) { + if (sqlScript.startsWith(dollarQuoteTag, index)) { + index += dollarQuoteTag.length; + dollarQuoteTag = null; + } else { + index++; + } + continue; + } + if (inSingleQuote) { + if (current === "'") { + if (next === "'") { + index += 2; + continue; + } + inSingleQuote = false; + } + index++; + continue; + } + if (inDoubleQuote) { + if (current === "\"") inDoubleQuote = false; + index++; + continue; + } + + if (current === "-" && next === "-") { + inLineComment = true; + index += 2; + continue; + } + if (current === "/" && next === "*") { + blockCommentDepth = 1; + index += 2; + continue; + } + if (current === "'") { + inSingleQuote = true; + index++; + continue; + } + if (current === "\"") { + inDoubleQuote = true; + index++; + continue; + } + if (current === "$") { + let tagEnd = index + 1; + while (tagEnd < sqlScript.length && /[a-zA-Z0-9_]/.test(sqlScript[tagEnd] ?? "")) { + tagEnd++; + } + if (sqlScript[tagEnd] === "$") { + dollarQuoteTag = sqlScript.slice(index, tagEnd + 1); + index = tagEnd + 1; + continue; + } + } + if (current === ";") { + const statement = sqlScript.slice(statementStart, index).trim(); + if (statement.length > 0) { + statements.push(statement); + } + statementStart = index + 1; + } + index++; + } + + const trailingStatement = sqlScript.slice(statementStart).trim(); + if (trailingStatement.length > 0) { + statements.push(trailingStatement); + } + return statements; +} + +function tableIdToString(tableId: unknown): string { + if (typeof tableId === "string") return tableId; + return JSON.stringify(tableId); +} + +function createTableRegistry(schema: Record): { + tables: StudioTableRecord[], + tableById: Map, + idByTable: Map, +} { + const tables: StudioTableRecord[] = []; + const idByTable = new Map(); + + for (const [name, value] of Object.entries(schema)) { + if (!isStudioTable(value)) continue; + const id = name; + const record: StudioTableRecord = { id, name, table: value }; + tables.push(record); + idByTable.set(value, id); + } + + if (tables.length === 0) { + throw new StackAssertionError("No studio-compatible tables found in schema object."); + } + + const tableById = new Map(tables.map((table) => [table.id, table])); + return { tables, tableById, idByTable }; +} + +const schemaObject: Record = exampleFungibleLedgerSchema; +const registry = createTableRegistry(schemaObject); + +async function executeStatements(statements: SqlStatement[]): Promise { + const sqlScript = toExecutableSqlStatements(statements); + const executableStatements = splitSqlStatements(sqlScript); + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL jit = off`); + await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID})`); + for (const statement of executableStatements) { + await tx.$executeRawUnsafe(statement); + } + }); +} + +async function queryRows(query: SqlQuery): Promise { + const rows = await retryTransaction(globalPrismaClient, async (tx) => { + return await tx.$queryRawUnsafe(toQueryableSqlQuery(query)); + }); + if (!Array.isArray(rows)) throw new StackAssertionError("Expected SQL query to return an array of rows."); + return rows; +} + +async function readBoolean(expression: SqlExpression): Promise { + const rows = await retryTransaction(globalPrismaClient, async (tx) => { + return await tx.$queryRawUnsafe>>(`SELECT (${expression.sql}) AS "value"`); + }); + if (!Array.isArray(rows) || rows.length === 0 || !isRecord(rows[0])) { + throw new StackAssertionError("Expected boolean expression query to return one row."); + } + return Reflect.get(rows[0], "value") === true; +} + +function valueFromRow(row: unknown, key: string): unknown { + if (!isRecord(row)) return null; + return Reflect.get(row, key); +} + +async function getTableSnapshot(record: StudioTableRecord): Promise<{ + id: string, + name: string, + tableId: string, + operator: string, + dependencies: string[], + debugArgs: Record, + supportsSetRow: boolean, + supportsDeleteRow: boolean, + initialized: boolean, +}> { + const inputTables = record.table.inputTables ?? []; + const debugArgs = record.table.debugArgs ?? {}; + const dependsOn = inputTables.map((inputTable) => { + return registry.idByTable.get(inputTable) ?? tableIdToString(inputTable.tableId); + }); + const operatorValue = Reflect.get(debugArgs, "operator"); + const operator = typeof operatorValue === "string" ? operatorValue : "unknown"; + + return { + id: record.id, + name: record.name, + tableId: tableIdToString(record.table.tableId), + operator, + dependencies: dependsOn, + debugArgs, + supportsSetRow: isStudioStoredTable(record.table), + supportsDeleteRow: isStudioStoredTable(record.table), + initialized: await readBoolean(record.table.isInitialized()), + }; +} + +function topologicallySortTableIds( + tables: Array>>, +): string[] { + const ids = new Set(tables.map((table) => table.id)); + const outgoing = new Map(); + const inDegree = new Map(); + + for (const table of tables) { + outgoing.set(table.id, []); + inDegree.set(table.id, 0); + } + + for (const table of tables) { + for (const dependencyId of table.dependencies) { + if (!ids.has(dependencyId)) continue; + const next = outgoing.get(dependencyId); + if (next == null) continue; + next.push(table.id); + const currentInDegree = inDegree.get(table.id); + if (currentInDegree == null) continue; + inDegree.set(table.id, currentInDegree + 1); + } + } + + const queue = [...inDegree.entries()] + .filter((entry) => entry[1] === 0) + .map((entry) => entry[0]) + .sort(stringCompare); + const ordered: string[] = []; + + while (queue.length > 0) { + const id = queue.shift(); + if (id == null) continue; + ordered.push(id); + const nextIds = outgoing.get(id) ?? []; + for (const nextId of nextIds) { + const currentInDegree = inDegree.get(nextId); + if (currentInDegree == null) continue; + const updatedInDegree = currentInDegree - 1; + inDegree.set(nextId, updatedInDegree); + if (updatedInDegree === 0) { + queue.push(nextId); + queue.sort(stringCompare); + } + } + } + + if (ordered.length === tables.length) return ordered; + + const remaining = [...ids].filter((id) => !ordered.includes(id)).sort(stringCompare); + return [...ordered, ...remaining]; +} + +async function rebindInitializedDerivedTables(): Promise { + const snapshots = await Promise.all(registry.tables.map((table) => getTableSnapshot(table))); + const initializedDerivedTableIds = new Set( + snapshots + .filter((table) => table.initialized && !table.supportsSetRow) + .map((table) => table.id), + ); + if (initializedDerivedTableIds.size === 0) return; + + const sortedIds = topologicallySortTableIds(snapshots); + const recordsToDelete = [...sortedIds] + .reverse() + .map((id) => registry.tableById.get(id)) + .filter((record): record is StudioTableRecord => record != null && initializedDerivedTableIds.has(record.id)); + const recordsToInit = sortedIds + .map((id) => registry.tableById.get(id)) + .filter((record): record is StudioTableRecord => record != null && initializedDerivedTableIds.has(record.id)); + + for (const record of recordsToDelete) { + await executeStatements(record.table.delete()); + } + for (const record of recordsToInit) { + await executeStatements(record.table.init()); + } + + console.log(`[studio] rebound ${recordsToInit.length} initialized derived tables`); +} + +async function initAllTablesInTopologicalOrder(): Promise { + const snapshots = await Promise.all(registry.tables.map((table) => getTableSnapshot(table))); + const snapshotById = new Map(snapshots.map((snapshot) => [snapshot.id, snapshot])); + const sortedIds = topologicallySortTableIds(snapshots); + const initializedIds: string[] = []; + + for (const id of sortedIds) { + const snapshot = snapshotById.get(id); + if (snapshot == null || snapshot.initialized) continue; + const record = registry.tableById.get(id); + if (record == null) continue; + await executeStatements(record.table.init()); + initializedIds.push(id); + } + + return initializedIds; +} + +async function computeStudioLayout(tables: Array>>): Promise, + sceneWidth: number, + sceneHeight: number, +}> { + try { + const layout = await elk.layout({ + id: "bulldozer-studio", + layoutOptions: { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.padding": `[top=${GRAPH_SCENE_MARGIN},left=${GRAPH_SCENE_MARGIN},bottom=${GRAPH_SCENE_MARGIN},right=${GRAPH_SCENE_MARGIN}]`, + "elk.spacing.nodeNode": String(Math.floor(GRAPH_COLUMN_GAP_X / 2)), + "elk.layered.spacing.nodeNodeBetweenLayers": String(Math.floor(GRAPH_LEVEL_GAP_Y / 2)), + "elk.layered.crossingMinimization.strategy": "LAYER_SWEEP", + "elk.layered.nodePlacement.strategy": "NETWORK_SIMPLEX", + "elk.layered.considerModelOrder.strategy": "NODES_AND_EDGES", + "elk.layered.thoroughness": "40", + }, + children: tables.map((table) => ({ + id: table.id, + width: GRAPH_NODE_WIDTH, + height: GRAPH_NODE_HEIGHT, + })), + edges: tables.flatMap((table) => { + return table.dependencies.map((dependencyId, index) => ({ + id: `${dependencyId}->${table.id}:${index}`, + sources: [dependencyId], + targets: [table.id], + })); + }), + }); + + const positions = new Map(); + for (const child of layout.children ?? []) { + if (typeof child.id !== "string") continue; + positions.set(child.id, { + x: Number(child.x ?? 0), + y: Number(child.y ?? 0), + }); + } + + return { + positions: Object.fromEntries(positions), + sceneWidth: Number(Reflect.get(layout, "width") ?? 600), + sceneHeight: Number(Reflect.get(layout, "height") ?? 600), + }; + } catch (error) { + return null; + } +} + +async function getTableDetails(record: StudioTableRecord): Promise<{ + table: Awaited>, + groups: Array<{ groupKey: unknown, rows: Array<{ rowIdentifier: unknown, rowSortKey: unknown, rowData: unknown }> }>, + totalRows: number, +}> { + const table = record.table; + const tableSnapshot = await getTableSnapshot(record); + const groupsRaw = await queryRows(table.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const allRowsRaw = await queryRows(table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + + const rowsByGroup = new Map }>(); + + for (const groupRow of groupsRaw) { + const groupKey = valueFromRow(groupRow, "groupkey"); + const key = JSON.stringify(groupKey); + rowsByGroup.set(key, { groupKey, rows: [] }); + } + + for (const row of allRowsRaw) { + const hasGroupKey = isRecord(row) && Reflect.has(row, "groupkey"); + const groupKey = hasGroupKey ? valueFromRow(row, "groupkey") : null; + const key = JSON.stringify(groupKey); + const existing = rowsByGroup.get(key) ?? { groupKey, rows: [] }; + existing.rows.push({ + rowIdentifier: valueFromRow(row, "rowidentifier"), + rowSortKey: valueFromRow(row, "rowsortkey"), + rowData: valueFromRow(row, "rowdata"), + }); + rowsByGroup.set(key, existing); + } + + const groups = [...rowsByGroup.values()].sort((a, b) => { + return stringCompare(JSON.stringify(a.groupKey), JSON.stringify(b.groupKey)); + }); + + return { + table: tableSnapshot, + groups, + totalRows: allRowsRaw.length, + }; +} + +async function getTimefoldDebugSnapshot(): Promise<{ + queueTableExists: boolean, + metadataTableExists: boolean, + pgCronInstalled: boolean, + lastProcessedAt: unknown, + queue: Array>, +}> { + return await retryTransaction(globalPrismaClient, async (tx) => { + const relationRows = await tx.$queryRawUnsafe>>(` + SELECT + to_regclass('"BulldozerTimeFoldQueue"') IS NOT NULL AS "queueTableExists", + to_regclass('"BulldozerTimeFoldMetadata"') IS NOT NULL AS "metadataTableExists", + to_regclass('cron.job') IS NOT NULL AS "pgCronInstalled" + `); + const relationRow = requireRecord(relationRows[0], "timefold relation probe returned invalid row"); + const queueTableExists = Reflect.get(relationRow, "queueTableExists") === true || Reflect.get(relationRow, "queuetableexists") === true; + const metadataTableExists = Reflect.get(relationRow, "metadataTableExists") === true || Reflect.get(relationRow, "metadatatableexists") === true; + const pgCronInstalled = Reflect.get(relationRow, "pgCronInstalled") === true || Reflect.get(relationRow, "pgcroninstalled") === true; + + let lastProcessedAt: unknown = null; + if (metadataTableExists) { + const metadataRows = await tx.$queryRawUnsafe>>(` + SELECT "lastProcessedAt" + FROM "BulldozerTimeFoldMetadata" + WHERE "key" = 'singleton' + LIMIT 1 + `); + if (metadataRows.length > 0) { + const metadataRow = requireRecord(metadataRows[0], "timefold metadata query returned invalid row"); + lastProcessedAt = Reflect.get(metadataRow, "lastProcessedAt") ?? Reflect.get(metadataRow, "lastprocessedat") ?? null; + } + } + + let queue: Array> = []; + if (queueTableExists) { + queue = await tx.$queryRawUnsafe>>(` + SELECT + "id", + "tableStoragePath", + "groupKey", + "rowIdentifier", + "scheduledAt", + "stateAfter", + "rowData", + "reducerSql", + "createdAt", + "updatedAt" + FROM "BulldozerTimeFoldQueue" + ORDER BY "scheduledAt" ASC, "id" ASC + LIMIT 500 + `); + } + + return { + queueTableExists, + metadataTableExists, + pgCronInstalled, + lastProcessedAt, + queue, + }; + }); +} + +async function getRawNode(pathSegments: string[]): Promise<{ + path: string[], + value: unknown, + children: Array<{ segment: string, hasChildren: boolean }>, +}> { + const keyPathLiteral = keyPathSqlLiteral(pathSegments); + const { valueRows, childrenRows } = await retryTransaction(globalPrismaClient, async (tx) => { + const valueRows = await tx.$queryRawUnsafe>>(` + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${keyPathLiteral} + `); + const childrenRows = await tx.$queryRawUnsafe>>(` + SELECT + ("child"."keyPath"[cardinality("child"."keyPath")] #>> '{}') AS "segment", + EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "grandChild" + WHERE "grandChild"."keyPathParent" = "child"."keyPath" + ) AS "hasChildren" + FROM "BulldozerStorageEngine" AS "child" + WHERE "child"."keyPathParent" = ${keyPathLiteral} + ORDER BY "segment" + `); + return { valueRows, childrenRows }; + }); + + const children = childrenRows + .filter((row) => isRecord(row) && typeof Reflect.get(row, "segment") === "string") + .map((row) => ({ + segment: requireString(Reflect.get(row, "segment"), "Expected segment to be a string."), + hasChildren: Reflect.get(row, "hasChildren") === true, + })); + + return { + path: pathSegments, + value: Array.isArray(valueRows) && valueRows.length > 0 ? valueFromRow(valueRows[0], "value") : null, + children, + }; +} + +async function readRequestBody(request: http.IncomingMessage): Promise { + const chunks: Buffer[] = []; + let totalBytes = 0; + for await (const chunk of request) { + const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += chunkBuffer.byteLength; + if (totalBytes > MAX_REQUEST_BODY_BYTES) { + throw new StackAssertionError("Request body exceeds maximum size.", { + maxRequestBodyBytes: MAX_REQUEST_BODY_BYTES, + receivedBytes: totalBytes, + }); + } + if (Buffer.isBuffer(chunk)) { + chunks.push(chunkBuffer); + } else if (typeof chunk === "string") { + chunks.push(chunkBuffer); + } + } + return Buffer.concat(chunks).toString("utf8"); +} + +async function readJsonBody(request: http.IncomingMessage): Promise { + const rawBody = await readRequestBody(request); + if (rawBody.trim() === "") return {}; + return JSON.parse(rawBody); +} + +function sendJson(response: http.ServerResponse, statusCode: number, payload: unknown): void { + response.statusCode = statusCode; + response.setHeader("Content-Type", "application/json; charset=utf-8"); + response.end(JSON.stringify(payload)); +} + +function sendHtml(response: http.ServerResponse, html: string): void { + response.statusCode = 200; + response.setHeader("Content-Type", "text/html; charset=utf-8"); + response.end(html); +} + +function isLoopbackAddress(remoteAddress: string | undefined): boolean { + if (remoteAddress == null) return false; + return remoteAddress === "127.0.0.1" + || remoteAddress === "::1" + || remoteAddress === "::ffff:127.0.0.1"; +} + +function requireAuthorizedMutationRequest(request: http.IncomingMessage, requestUrl: URL): void { + const authHeader = request.headers[STUDIO_AUTH_HEADER]; + const token = typeof authHeader === "string" ? authHeader : null; + if (token !== STUDIO_AUTH_TOKEN) { + throw new StackAssertionError("Invalid or missing studio mutation token."); + } + + const originHeader = request.headers.origin; + if (typeof originHeader === "string") { + let originUrl: URL; + try { + originUrl = new URL(originHeader); + } catch { + throw new StackAssertionError("Mutation origin is not allowed.", { + origin: originHeader, + path: requestUrl.pathname, + }); + } + + const portMatches = originUrl.port === String(STUDIO_PORT); + const hostname = originUrl.hostname.toLowerCase(); + const hostnameAllowed = hostname === "localhost" + || hostname === "127.0.0.1" + || hostname === "::1" + || hostname.endsWith(".localhost"); + if (!portMatches || !hostnameAllowed) { + throw new StackAssertionError("Mutation origin is not allowed.", { + origin: originHeader, + path: requestUrl.pathname, + }); + } + } +} + +function getStudioPageHtml(): string { + return ` + + + + + Bulldozer Studio + + + +
+
+
+
Bulldozer Studio
+ + + + + + + +
+
+
ready
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+ + +
+
Action failed
+

+      
+ +
+
+
+ + + +`; +} + +async function handleRequest(request: http.IncomingMessage, response: http.ServerResponse): Promise { + if (!isLoopbackAddress(request.socket.remoteAddress)) { + throw new StackAssertionError("Bulldozer Studio only accepts loopback requests.", { + remoteAddress: request.socket.remoteAddress, + }); + } + + const requestUrl = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`); + const pathname = requestUrl.pathname; + const method = request.method ?? "GET"; + if (method === "POST") { + requireAuthorizedMutationRequest(request, requestUrl); + } + + if (method === "GET" && pathname === "/") { + sendHtml(response, getStudioPageHtml()); + return; + } + + if (method === "GET" && pathname === "/api/version") { + sendJson(response, 200, { version: STUDIO_INSTANCE_ID }); + return; + } + + if (method === "GET" && pathname === "/api/schema") { + const tables = await Promise.all(registry.tables.map((table) => getTableSnapshot(table))); + const layout = await computeStudioLayout(tables); + sendJson(response, 200, { tables, layout }); + return; + } + + if (method === "GET" && pathname === "/api/timefold/debug") { + const snapshot = await getTimefoldDebugSnapshot(); + sendJson(response, 200, snapshot); + return; + } + + if (method === "POST" && pathname === "/api/tables/init-all") { + const initializedTableIds = await initAllTablesInTopologicalOrder(); + sendJson(response, 200, { ok: true, initializedTableIds }); + return; + } + + if (pathname.startsWith("/api/table/")) { + const pathParts = pathname.split("/").filter(Boolean); + const tableId = decodeURIComponent(pathParts[2] ?? ""); + const record = registry.tableById.get(tableId); + if (!record) { + sendJson(response, 404, { error: `Unknown table: ${tableId}` }); + return; + } + + if (method === "GET" && pathParts[3] === "details") { + const details = await getTableDetails(record); + sendJson(response, 200, details); + return; + } + + if (method === "POST" && pathParts[3] === "init") { + await executeStatements(record.table.init()); + sendJson(response, 200, { ok: true }); + return; + } + + if (method === "POST" && pathParts[3] === "delete") { + await executeStatements(record.table.delete()); + sendJson(response, 200, { ok: true }); + return; + } + + if (method === "POST" && pathParts[3] === "set-row") { + if (!isStudioStoredTable(record.table)) { + sendJson(response, 400, { error: "This table does not support setRow." }); + return; + } + const body = requireRecord(await readJsonBody(request), "set-row body must be an object."); + const rowIdentifier = requireString(Reflect.get(body, "rowIdentifier"), "rowIdentifier must be a string."); + const rowData = requireJsonValue(Reflect.get(body, "rowData"), "rowData must be valid JSON."); + if (!isRecord(rowData)) { + throw new StackAssertionError("rowData must be a JSON object."); + } + await executeStatements(record.table.setRow( + rowIdentifier, + { type: "expression", sql: quoteSqlJsonbLiteral(rowData).sql }, + )); + sendJson(response, 200, { ok: true }); + return; + } + + if (method === "POST" && pathParts[3] === "delete-row") { + if (!isStudioStoredTable(record.table)) { + sendJson(response, 400, { error: "This table does not support deleteRow." }); + return; + } + const body = requireRecord(await readJsonBody(request), "delete-row body must be an object."); + const rowIdentifier = requireString(Reflect.get(body, "rowIdentifier"), "rowIdentifier must be a string."); + await executeStatements(record.table.deleteRow(rowIdentifier)); + sendJson(response, 200, { ok: true }); + return; + } + } + + if (method === "GET" && pathname === "/api/raw/node") { + const pathParam = requestUrl.searchParams.get("path") ?? "[]"; + const parsedPath = JSON.parse(pathParam); + const pathSegments = requireStringArray(parsedPath, "path must be a string[]"); + const node = await getRawNode(pathSegments); + sendJson(response, 200, node); + return; + } + + if (method === "POST" && pathname === "/api/raw/upsert") { + const body = requireRecord(await readJsonBody(request), "raw upsert body must be an object."); + const pathSegments = requireStringArray(Reflect.get(body, "pathSegments"), "pathSegments must be a string[]"); + const value = requireJsonValue(Reflect.get(body, "value") ?? null, "value must be valid JSON."); + const keyPathSql = keyPathSqlLiteral(pathSegments); + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL jit = off`); + await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID})`); + await tx.$executeRawUnsafe(` + WITH "targetPath" AS ( + SELECT ${keyPathSql} AS "path" + ) + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "targetPath"."path"[1:"prefixes"."prefixLength"] AS "keyPath", + 'null'::jsonb AS "value" + FROM "targetPath" + CROSS JOIN LATERAL generate_series(0, GREATEST(cardinality("targetPath"."path") - 1, 0)) AS "prefixes"("prefixLength") + ON CONFLICT ("keyPath") DO NOTHING + `); + await tx.$executeRawUnsafe(` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES (gen_random_uuid(), ${keyPathSql}, ${quoteSqlJsonbLiteral(value).sql}) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `); + }); + sendJson(response, 200, { ok: true }); + return; + } + + if (method === "POST" && pathname === "/api/raw/delete") { + const body = requireRecord(await readJsonBody(request), "raw delete body must be an object."); + const pathSegments = requireStringArray(Reflect.get(body, "pathSegments"), "pathSegments must be a string[]"); + if ( + pathSegments.length === 0 + || (pathSegments.length === 1 && pathSegments[0] === "table") + ) { + throw new StackAssertionError("Deleting reserved root paths is not allowed."); + } + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL jit = off`); + await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID})`); + await tx.$executeRawUnsafe(` + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${keyPathSqlLiteral(pathSegments)} + `); + }); + sendJson(response, 200, { ok: true }); + return; + } + + sendJson(response, 404, { error: `Route not found: ${method} ${pathname}` }); +} + +async function main(): Promise { + await rebindInitializedDerivedTables(); + + const server = http.createServer((request, response) => { + handleRequest(request, response).then( + () => undefined, + (error) => { + console.error(error); + const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + sendJson(response, 500, { error: message }); + }, + ); + }); + + server.listen(STUDIO_PORT, STUDIO_HOST, () => { + console.log(`Bulldozer Studio running on http://${STUDIO_HOST}:${STUDIO_PORT}`); + }); + + const shutdown = async () => { + server.close(); + }; + process.on("SIGINT", () => { + shutdown().then(() => process.exit(0), () => process.exit(1)); + }); + process.on("SIGTERM", () => { + shutdown().then(() => process.exit(0), () => process.exit(1)); + }); +} + +main().then( + () => undefined, + (error) => { + console.error(error); + process.exit(1); + }, +); diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts index 98b9680ce5..e6d2c643d5 100644 --- a/apps/backend/scripts/run-cron-jobs.ts +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -25,6 +25,7 @@ async function main() { for (const endpoint of endpoints) { runAsynchronously(async () => { + await wait(30_000); // Wait 30 seconds to make sure the server is fully started while (true) { const runResult = await Result.fromPromise(run(endpoint)); if (runResult.status === "error") { diff --git a/apps/backend/src/app/api/latest/auth/passkey/initiate-passkey-registration/route.tsx b/apps/backend/src/app/api/latest/auth/passkey/initiate-passkey-registration/route.tsx index 76a7493fab..1cd78ba7fe 100644 --- a/apps/backend/src/app/api/latest/auth/passkey/initiate-passkey-registration/route.tsx +++ b/apps/backend/src/app/api/latest/auth/passkey/initiate-passkey-registration/route.tsx @@ -56,10 +56,12 @@ export const POST = createSmartRouteHandler({ }; const registrationOptionsRaw = await generateRegistrationOptions(opts); - const registrationOptions = registrationOptionsRaw.hints != null && registrationOptionsRaw.hints.length === 0 + const registrationHints = Reflect.get(registrationOptionsRaw, "hints"); + const registrationOptions = Array.isArray(registrationHints) && registrationHints.length === 0 ? (() => { - const { hints: _, ...rest } = registrationOptionsRaw; - return rest; + const optionsWithoutHints = { ...registrationOptionsRaw }; + Reflect.deleteProperty(optionsWithoutHints, "hints"); + return optionsWithoutHints; })() : registrationOptionsRaw; diff --git a/apps/backend/src/app/api/latest/auth/passkey/register/verification-code-handler.tsx b/apps/backend/src/app/api/latest/auth/passkey/register/verification-code-handler.tsx index 0dd7842fe0..4c81b963b1 100644 --- a/apps/backend/src/app/api/latest/auth/passkey/register/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/auth/passkey/register/verification-code-handler.tsx @@ -80,6 +80,12 @@ export const registerVerificationCodeHandler = createVerificationCodeHandler({ } const registrationInfo = verification.registrationInfo; + if (registrationInfo == null) { + throw new StackAssertionError("Passkey registration verification succeeded without registration info", { + tenancyId: tenancy.id, + projectUserId: user.id, + }); + } const prisma = await getPrismaClientForTenancy(tenancy); await retryTransaction(prisma, async (tx) => { diff --git a/apps/backend/src/lib/bulldozer/bulldozer-schema.ts b/apps/backend/src/lib/bulldozer/bulldozer-schema.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/backend/src/lib/bulldozer/db/bulldozer-sort-helpers-sql.ts b/apps/backend/src/lib/bulldozer/db/bulldozer-sort-helpers-sql.ts new file mode 100644 index 0000000000..1591308ce9 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/bulldozer-sort-helpers-sql.ts @@ -0,0 +1,812 @@ +import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; + +export const BULLDOZER_SORT_HELPERS_SQL = deindent` + CREATE TEMP TABLE IF NOT EXISTS pg_temp.bulldozer_side_effects ( + "note" text + ) ON COMMIT DROP; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_group_path(groups_path jsonb[], group_key jsonb) + RETURNS jsonb[] LANGUAGE sql IMMUTABLE AS $$ + SELECT groups_path || ARRAY[group_key]::jsonb[] + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_group_metadata_path(groups_path jsonb[], group_key jsonb) + RETURNS jsonb[] LANGUAGE sql IMMUTABLE AS $$ + SELECT pg_temp.bulldozer_sort_group_path(groups_path, group_key) || ARRAY[to_jsonb('metadata'::text)]::jsonb[] + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_group_rows_path(groups_path jsonb[], group_key jsonb) + RETURNS jsonb[] LANGUAGE sql IMMUTABLE AS $$ + SELECT pg_temp.bulldozer_sort_group_path(groups_path, group_key) || ARRAY[to_jsonb('rows'::text)]::jsonb[] + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_group_row_path(groups_path jsonb[], group_key jsonb, row_identifier text) + RETURNS jsonb[] LANGUAGE sql IMMUTABLE AS $$ + SELECT pg_temp.bulldozer_sort_group_rows_path(groups_path, group_key) || ARRAY[to_jsonb(row_identifier)]::jsonb[] + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_nullable_text_jsonb(input_text text) + RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$ + SELECT CASE + WHEN input_text IS NULL THEN 'null'::jsonb + ELSE to_jsonb(input_text) + END + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_make_group_metadata(root_row_identifier text, head_row_identifier text, tail_row_identifier text, row_count integer) + RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$ + SELECT jsonb_build_object( + 'rootRowIdentifier', root_row_identifier, + 'headRowIdentifier', head_row_identifier, + 'tailRowIdentifier', tail_row_identifier, + 'rowCount', row_count + ) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_make_row_value( + row_sort_key jsonb, + row_data jsonb, + left_row_identifier text, + right_row_identifier text, + priority bigint, + prev_row_identifier text, + next_row_identifier text + ) + RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$ + SELECT jsonb_build_object( + 'rowSortKey', row_sort_key, + 'rowData', row_data, + 'leftRowIdentifier', left_row_identifier, + 'rightRowIdentifier', right_row_identifier, + 'priority', priority, + 'prevRowIdentifier', prev_row_identifier, + 'nextRowIdentifier', next_row_identifier + ) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_get_group_metadata(groups_path jsonb[], group_key jsonb) + RETURNS jsonb LANGUAGE sql STABLE AS $$ + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = pg_temp.bulldozer_sort_group_metadata_path(groups_path, group_key) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_get_row(groups_path jsonb[], group_key jsonb, row_identifier text) + RETURNS jsonb LANGUAGE sql STABLE AS $$ + SELECT "value" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = pg_temp.bulldozer_sort_group_row_path(groups_path, group_key, row_identifier) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_compare_sort_keys(compare_sort_keys_sql text, left_sort_key jsonb, right_sort_key jsonb) + RETURNS integer LANGUAGE plpgsql AS $$ + DECLARE + cmp integer; + BEGIN + EXECUTE 'SELECT (' || compare_sort_keys_sql || ')::int' + INTO cmp + USING left_sort_key, right_sort_key; + IF cmp < 0 THEN RETURN -1; END IF; + IF cmp > 0 THEN RETURN 1; END IF; + RETURN 0; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql text, + left_sort_key jsonb, + left_row_identifier text, + right_sort_key jsonb, + right_row_identifier text + ) + RETURNS integer LANGUAGE plpgsql AS $$ + DECLARE + cmp integer; + BEGIN + cmp := pg_temp.bulldozer_sort_compare_sort_keys(compare_sort_keys_sql, left_sort_key, right_sort_key); + IF cmp <> 0 THEN + RETURN cmp; + END IF; + IF left_row_identifier < right_row_identifier THEN RETURN -1; END IF; + IF left_row_identifier > right_row_identifier THEN RETURN 1; END IF; + RETURN 0; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_put_group_metadata(groups_path jsonb[], group_key jsonb, root_row_identifier text, head_row_identifier text, tail_row_identifier text, row_count integer) + RETURNS void LANGUAGE sql VOLATILE AS $$ + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + pg_temp.bulldozer_sort_group_metadata_path(groups_path, group_key), + pg_temp.bulldozer_sort_make_group_metadata(root_row_identifier, head_row_identifier, tail_row_identifier, row_count) + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_put_row_value(groups_path jsonb[], group_key jsonb, row_identifier text, row_value jsonb) + RETURNS void LANGUAGE sql VOLATILE AS $$ + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + pg_temp.bulldozer_sort_group_row_path(groups_path, group_key, row_identifier), + row_value + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_put_row( + groups_path jsonb[], + group_key jsonb, + row_identifier text, + row_sort_key jsonb, + row_data jsonb, + left_row_identifier text, + right_row_identifier text, + priority bigint, + prev_row_identifier text, + next_row_identifier text + ) + RETURNS void LANGUAGE sql VOLATILE AS $$ + SELECT pg_temp.bulldozer_sort_put_row_value( + groups_path, + group_key, + row_identifier, + pg_temp.bulldozer_sort_make_row_value( + row_sort_key, + row_data, + left_row_identifier, + right_row_identifier, + priority, + prev_row_identifier, + next_row_identifier + ) + ) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_delete_row_storage(groups_path jsonb[], group_key jsonb, row_identifier text) + RETURNS void LANGUAGE sql VOLATILE AS $$ + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" = pg_temp.bulldozer_sort_group_row_path(groups_path, group_key, row_identifier) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_random_priority() + RETURNS bigint LANGUAGE sql VOLATILE AS $$ + SELECT abs(hashtextextended(gen_random_uuid()::text, 0)) + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_ensure_group(groups_path jsonb[], group_key jsonb) + RETURNS void LANGUAGE plpgsql AS $$ + BEGIN + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + groups_path[1:"prefixLength"]::jsonb[], + 'null'::jsonb + FROM generate_series(2, cardinality(groups_path)) AS "prefixLength" + ON CONFLICT ("keyPath") DO NOTHING; + + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), pg_temp.bulldozer_sort_group_path(groups_path, group_key), 'null'::jsonb), + (gen_random_uuid(), pg_temp.bulldozer_sort_group_rows_path(groups_path, group_key), 'null'::jsonb) + ON CONFLICT ("keyPath") DO NOTHING; + + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + pg_temp.bulldozer_sort_group_metadata_path(groups_path, group_key), + pg_temp.bulldozer_sort_make_group_metadata(NULL, NULL, NULL, 0) + ) + ON CONFLICT ("keyPath") DO NOTHING; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_find_predecessor( + groups_path jsonb[], + group_key jsonb, + compare_sort_keys_sql text, + target_row_identifier text, + target_row_sort_key jsonb + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + metadata_value jsonb; + current_row_identifier text; + current_row_value jsonb; + best_row_identifier text; + cmp integer; + BEGIN + metadata_value := pg_temp.bulldozer_sort_get_group_metadata(groups_path, group_key); + current_row_identifier := metadata_value->>'rootRowIdentifier'; + best_row_identifier := NULL; + + WHILE current_row_identifier IS NOT NULL LOOP + current_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, current_row_identifier); + cmp := pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql, + current_row_value->'rowSortKey', + current_row_identifier, + target_row_sort_key, + target_row_identifier + ); + IF cmp < 0 THEN + best_row_identifier := current_row_identifier; + current_row_identifier := current_row_value->>'rightRowIdentifier'; + ELSE + current_row_identifier := current_row_value->>'leftRowIdentifier'; + END IF; + END LOOP; + + RETURN best_row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_find_successor( + groups_path jsonb[], + group_key jsonb, + compare_sort_keys_sql text, + target_row_identifier text, + target_row_sort_key jsonb + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + metadata_value jsonb; + current_row_identifier text; + current_row_value jsonb; + best_row_identifier text; + cmp integer; + BEGIN + metadata_value := pg_temp.bulldozer_sort_get_group_metadata(groups_path, group_key); + current_row_identifier := metadata_value->>'rootRowIdentifier'; + best_row_identifier := NULL; + + WHILE current_row_identifier IS NOT NULL LOOP + current_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, current_row_identifier); + cmp := pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql, + current_row_value->'rowSortKey', + current_row_identifier, + target_row_sort_key, + target_row_identifier + ); + IF cmp > 0 THEN + best_row_identifier := current_row_identifier; + current_row_identifier := current_row_value->>'leftRowIdentifier'; + ELSE + current_row_identifier := current_row_value->>'rightRowIdentifier'; + END IF; + END LOOP; + + RETURN best_row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_merge( + groups_path jsonb[], + group_key jsonb, + left_root_row_identifier text, + right_root_row_identifier text + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + left_row_value jsonb; + right_row_value jsonb; + merged_child_row_identifier text; + BEGIN + IF left_root_row_identifier IS NULL THEN + RETURN right_root_row_identifier; + END IF; + IF right_root_row_identifier IS NULL THEN + RETURN left_root_row_identifier; + END IF; + + left_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, left_root_row_identifier); + right_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, right_root_row_identifier); + + IF COALESCE((left_row_value->>'priority')::bigint, 0) <= COALESCE((right_row_value->>'priority')::bigint, 0) THEN + merged_child_row_identifier := pg_temp.bulldozer_sort_merge( + groups_path, + group_key, + left_row_value->>'rightRowIdentifier', + right_root_row_identifier + ); + left_row_value := jsonb_set(left_row_value, '{rightRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(merged_child_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, left_root_row_identifier, left_row_value); + RETURN left_root_row_identifier; + END IF; + + merged_child_row_identifier := pg_temp.bulldozer_sort_merge( + groups_path, + group_key, + left_root_row_identifier, + right_row_value->>'leftRowIdentifier' + ); + right_row_value := jsonb_set(right_row_value, '{leftRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(merged_child_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, right_root_row_identifier, right_row_value); + RETURN right_root_row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_split( + groups_path jsonb[], + group_key jsonb, + root_row_identifier text, + split_row_sort_key jsonb, + split_row_identifier text, + compare_sort_keys_sql text, + OUT left_root_row_identifier text, + OUT right_root_row_identifier text + ) + RETURNS record LANGUAGE plpgsql AS $$ + DECLARE + root_row_value jsonb; + child_split_result record; + cmp integer; + BEGIN + IF root_row_identifier IS NULL THEN + left_root_row_identifier := NULL; + right_root_row_identifier := NULL; + RETURN; + END IF; + + root_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, root_row_identifier); + cmp := pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql, + root_row_value->'rowSortKey', + root_row_identifier, + split_row_sort_key, + split_row_identifier + ); + + IF cmp < 0 THEN + SELECT * + INTO child_split_result + FROM pg_temp.bulldozer_sort_split( + groups_path, + group_key, + root_row_value->>'rightRowIdentifier', + split_row_sort_key, + split_row_identifier, + compare_sort_keys_sql + ) AS "splitResult"; + root_row_value := jsonb_set(root_row_value, '{rightRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(child_split_result.left_root_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, root_row_identifier, root_row_value); + left_root_row_identifier := root_row_identifier; + right_root_row_identifier := child_split_result.right_root_row_identifier; + RETURN; + END IF; + + SELECT * + INTO child_split_result + FROM pg_temp.bulldozer_sort_split( + groups_path, + group_key, + root_row_value->>'leftRowIdentifier', + split_row_sort_key, + split_row_identifier, + compare_sort_keys_sql + ) AS "splitResult"; + root_row_value := jsonb_set(root_row_value, '{leftRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(child_split_result.right_root_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, root_row_identifier, root_row_value); + left_root_row_identifier := child_split_result.left_root_row_identifier; + right_root_row_identifier := root_row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_insert( + groups_path jsonb[], + group_key jsonb, + compare_sort_keys_sql text, + row_identifier text, + row_sort_key jsonb, + row_data jsonb + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + metadata_value jsonb; + predecessor_row_identifier text; + successor_row_identifier text; + predecessor_row_value jsonb; + successor_row_value jsonb; + split_left_root_row_identifier text; + split_right_root_row_identifier text; + merged_left_root_row_identifier text; + new_root_row_identifier text; + new_head_row_identifier text; + new_tail_row_identifier text; + row_count integer; + BEGIN + PERFORM pg_temp.bulldozer_sort_ensure_group(groups_path, group_key); + metadata_value := pg_temp.bulldozer_sort_get_group_metadata(groups_path, group_key); + row_count := COALESCE((metadata_value->>'rowCount')::int, 0); + + predecessor_row_identifier := pg_temp.bulldozer_sort_find_predecessor( + groups_path, + group_key, + compare_sort_keys_sql, + row_identifier, + row_sort_key + ); + successor_row_identifier := pg_temp.bulldozer_sort_find_successor( + groups_path, + group_key, + compare_sort_keys_sql, + row_identifier, + row_sort_key + ); + + PERFORM pg_temp.bulldozer_sort_put_row( + groups_path, + group_key, + row_identifier, + row_sort_key, + row_data, + NULL, + NULL, + pg_temp.bulldozer_sort_random_priority(), + predecessor_row_identifier, + successor_row_identifier + ); + + IF predecessor_row_identifier IS NOT NULL THEN + predecessor_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, predecessor_row_identifier); + IF predecessor_row_value IS NOT NULL THEN + predecessor_row_value := jsonb_set(predecessor_row_value, '{nextRowIdentifier}', to_jsonb(row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, predecessor_row_identifier, predecessor_row_value); + END IF; + END IF; + IF successor_row_identifier IS NOT NULL THEN + successor_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, successor_row_identifier); + IF successor_row_value IS NOT NULL THEN + successor_row_value := jsonb_set(successor_row_value, '{prevRowIdentifier}', to_jsonb(row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, successor_row_identifier, successor_row_value); + END IF; + END IF; + + SELECT "left_root_row_identifier", "right_root_row_identifier" + INTO split_left_root_row_identifier, split_right_root_row_identifier + FROM pg_temp.bulldozer_sort_split( + groups_path, + group_key, + metadata_value->>'rootRowIdentifier', + row_sort_key, + row_identifier, + compare_sort_keys_sql + ); + merged_left_root_row_identifier := pg_temp.bulldozer_sort_merge( + groups_path, + group_key, + split_left_root_row_identifier, + row_identifier + ); + new_root_row_identifier := pg_temp.bulldozer_sort_merge( + groups_path, + group_key, + merged_left_root_row_identifier, + split_right_root_row_identifier + ); + + new_head_row_identifier := COALESCE(metadata_value->>'headRowIdentifier', row_identifier); + IF predecessor_row_identifier IS NULL THEN + new_head_row_identifier := row_identifier; + END IF; + new_tail_row_identifier := COALESCE(metadata_value->>'tailRowIdentifier', row_identifier); + IF successor_row_identifier IS NULL THEN + new_tail_row_identifier := row_identifier; + END IF; + + PERFORM pg_temp.bulldozer_sort_put_group_metadata( + groups_path, + group_key, + new_root_row_identifier, + new_head_row_identifier, + new_tail_row_identifier, + row_count + 1 + ); + RETURN row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_build_balanced_group( + groups_path jsonb[], + group_key jsonb, + ordered_rows jsonb[], + start_index integer, + end_index integer, + level integer + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + midpoint integer; + current_row jsonb; + row_identifier text; + left_root_row_identifier text; + right_root_row_identifier text; + prev_row_identifier text; + next_row_identifier text; + BEGIN + IF start_index > end_index THEN + RETURN NULL; + END IF; + + midpoint := (start_index + end_index) / 2; + current_row := ordered_rows[midpoint]; + row_identifier := current_row->>'rowIdentifier'; + left_root_row_identifier := pg_temp.bulldozer_sort_build_balanced_group( + groups_path, + group_key, + ordered_rows, + start_index, + midpoint - 1, + level + 1 + ); + right_root_row_identifier := pg_temp.bulldozer_sort_build_balanced_group( + groups_path, + group_key, + ordered_rows, + midpoint + 1, + end_index, + level + 1 + ); + prev_row_identifier := CASE WHEN midpoint > 1 THEN ordered_rows[midpoint - 1]->>'rowIdentifier' ELSE NULL END; + next_row_identifier := CASE WHEN midpoint < array_length(ordered_rows, 1) THEN ordered_rows[midpoint + 1]->>'rowIdentifier' ELSE NULL END; + + PERFORM pg_temp.bulldozer_sort_put_row( + groups_path, + group_key, + row_identifier, + current_row->'rowSortKey', + current_row->'rowData', + left_root_row_identifier, + right_root_row_identifier, + level, + prev_row_identifier, + next_row_identifier + ); + RETURN row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_bulk_init_from_table(groups_path jsonb[], source_table_name text, compare_sort_keys_sql text) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + current_group_key jsonb; + ordered_rows jsonb[]; + root_row_identifier text; + row_count integer; + is_order_compatible boolean; + current_index integer; + cmp integer; + current_row jsonb; + BEGIN + FOR current_group_key IN EXECUTE format('SELECT DISTINCT "groupKey" FROM %I', source_table_name) + LOOP + PERFORM pg_temp.bulldozer_sort_ensure_group(groups_path, current_group_key); + EXECUTE format( + 'SELECT array_agg(jsonb_build_object(''rowIdentifier'', "rowIdentifier", ''rowSortKey'', "rowSortKey", ''rowData'', "rowData") ORDER BY "rowSortKey" ASC, "rowIdentifier" ASC) FROM %I WHERE "groupKey" IS NOT DISTINCT FROM $1', + source_table_name + ) + INTO ordered_rows + USING current_group_key; + + row_count := COALESCE(array_length(ordered_rows, 1), 0); + IF row_count = 0 THEN + CONTINUE; + END IF; + + is_order_compatible := TRUE; + FOR current_index IN 2..row_count + LOOP + cmp := pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql, + ordered_rows[current_index - 1]->'rowSortKey', + ordered_rows[current_index - 1]->>'rowIdentifier', + ordered_rows[current_index]->'rowSortKey', + ordered_rows[current_index]->>'rowIdentifier' + ); + IF cmp > 0 THEN + is_order_compatible := FALSE; + EXIT; + END IF; + END LOOP; + + IF is_order_compatible THEN + root_row_identifier := pg_temp.bulldozer_sort_build_balanced_group( + groups_path, + current_group_key, + ordered_rows, + 1, + row_count, + 1 + ); + PERFORM pg_temp.bulldozer_sort_put_group_metadata( + groups_path, + current_group_key, + root_row_identifier, + ordered_rows[1]->>'rowIdentifier', + ordered_rows[row_count]->>'rowIdentifier', + row_count + ); + ELSE + FOREACH current_row IN ARRAY ordered_rows + LOOP + PERFORM pg_temp.bulldozer_sort_insert( + groups_path, + current_group_key, + compare_sort_keys_sql, + current_row->>'rowIdentifier', + current_row->'rowSortKey', + current_row->'rowData' + ); + END LOOP; + END IF; + END LOOP; + + RETURN source_table_name; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_delete_recursive( + groups_path jsonb[], + group_key jsonb, + root_row_identifier text, + compare_sort_keys_sql text, + target_row_identifier text, + target_row_sort_key jsonb + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + root_row_value jsonb; + updated_child_row_identifier text; + merged_row_identifier text; + cmp integer; + BEGIN + IF root_row_identifier IS NULL THEN + RETURN NULL; + END IF; + + root_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, root_row_identifier); + cmp := pg_temp.bulldozer_sort_compare_row_keys( + compare_sort_keys_sql, + target_row_sort_key, + target_row_identifier, + root_row_value->'rowSortKey', + root_row_identifier + ); + + IF cmp < 0 THEN + IF root_row_value->>'leftRowIdentifier' IS NULL THEN + RETURN root_row_identifier; + END IF; + updated_child_row_identifier := pg_temp.bulldozer_sort_delete_recursive( + groups_path, + group_key, + root_row_value->>'leftRowIdentifier', + compare_sort_keys_sql, + target_row_identifier, + target_row_sort_key + ); + root_row_value := jsonb_set(root_row_value, '{leftRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(updated_child_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, root_row_identifier, root_row_value); + RETURN root_row_identifier; + END IF; + + IF cmp > 0 THEN + IF root_row_value->>'rightRowIdentifier' IS NULL THEN + RETURN root_row_identifier; + END IF; + updated_child_row_identifier := pg_temp.bulldozer_sort_delete_recursive( + groups_path, + group_key, + root_row_value->>'rightRowIdentifier', + compare_sort_keys_sql, + target_row_identifier, + target_row_sort_key + ); + root_row_value := jsonb_set(root_row_value, '{rightRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(updated_child_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, root_row_identifier, root_row_value); + RETURN root_row_identifier; + END IF; + + merged_row_identifier := pg_temp.bulldozer_sort_merge( + groups_path, + group_key, + root_row_value->>'leftRowIdentifier', + root_row_value->>'rightRowIdentifier' + ); + PERFORM pg_temp.bulldozer_sort_delete_row_storage(groups_path, group_key, root_row_identifier); + RETURN merged_row_identifier; + END; + $$; + + CREATE OR REPLACE FUNCTION pg_temp.bulldozer_sort_delete( + groups_path jsonb[], + group_key jsonb, + compare_sort_keys_sql text, + row_identifier text + ) + RETURNS text LANGUAGE plpgsql AS $$ + DECLARE + metadata_value jsonb; + row_value jsonb; + predecessor_row_identifier text; + successor_row_identifier text; + predecessor_row_value jsonb; + successor_row_value jsonb; + new_root_row_identifier text; + current_head_row_identifier text; + current_tail_row_identifier text; + row_count integer; + BEGIN + metadata_value := pg_temp.bulldozer_sort_get_group_metadata(groups_path, group_key); + IF metadata_value IS NULL THEN + RETURN row_identifier; + END IF; + + row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, row_identifier); + IF row_value IS NULL THEN + RETURN row_identifier; + END IF; + + predecessor_row_identifier := row_value->>'prevRowIdentifier'; + successor_row_identifier := row_value->>'nextRowIdentifier'; + row_count := COALESCE((metadata_value->>'rowCount')::int, 0); + + IF predecessor_row_identifier IS NOT NULL THEN + predecessor_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, predecessor_row_identifier); + IF predecessor_row_value IS NOT NULL THEN + predecessor_row_value := jsonb_set(predecessor_row_value, '{nextRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(successor_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, predecessor_row_identifier, predecessor_row_value); + END IF; + END IF; + IF successor_row_identifier IS NOT NULL THEN + successor_row_value := pg_temp.bulldozer_sort_get_row(groups_path, group_key, successor_row_identifier); + IF successor_row_value IS NOT NULL THEN + successor_row_value := jsonb_set(successor_row_value, '{prevRowIdentifier}', pg_temp.bulldozer_sort_nullable_text_jsonb(predecessor_row_identifier), true); + PERFORM pg_temp.bulldozer_sort_put_row_value(groups_path, group_key, successor_row_identifier, successor_row_value); + END IF; + END IF; + + new_root_row_identifier := pg_temp.bulldozer_sort_delete_recursive( + groups_path, + group_key, + metadata_value->>'rootRowIdentifier', + compare_sort_keys_sql, + row_identifier, + row_value->'rowSortKey' + ); + + IF row_count <= 1 THEN + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN ( + pg_temp.bulldozer_sort_group_metadata_path(groups_path, group_key), + pg_temp.bulldozer_sort_group_rows_path(groups_path, group_key), + pg_temp.bulldozer_sort_group_path(groups_path, group_key) + ); + RETURN row_identifier; + END IF; + + current_head_row_identifier := metadata_value->>'headRowIdentifier'; + current_tail_row_identifier := metadata_value->>'tailRowIdentifier'; + IF current_head_row_identifier = row_identifier THEN + current_head_row_identifier := successor_row_identifier; + END IF; + IF current_tail_row_identifier = row_identifier THEN + current_tail_row_identifier := predecessor_row_identifier; + END IF; + + PERFORM pg_temp.bulldozer_sort_put_group_metadata( + groups_path, + group_key, + new_root_row_identifier, + current_head_row_identifier, + current_tail_row_identifier, + row_count - 1 + ); + RETURN row_identifier; + END; + $$; +`; diff --git a/apps/backend/src/lib/bulldozer/db/example-schema.ts b/apps/backend/src/lib/bulldozer/db/example-schema.ts new file mode 100644 index 0000000000..92ff28b535 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/example-schema.ts @@ -0,0 +1,327 @@ +import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLeftJoinTable, declareLFoldTable, declareLimitTable, declareMapTable, declareSortTable, declareStoredTable, declareTimeFoldTable } from "./index"; + +const mapper = (sql: string) => ({ type: "mapper" as const, sql }); +const predicate = (sql: string) => ({ type: "predicate" as const, sql }); + +/** + * Example fungible-asset ledger schema composed from Bulldozer table operators. + * + * This file intentionally declares tables only; it does not call init/delete. + */ +export const exampleFungibleLedgerSchema = (() => { + // Base append/update table for raw ledger entries. + const ledgerEntries = declareStoredTable<{ + accountId: string, + asset: string, + amount: string, + side: "credit" | "debit", + txHash: string, + blockNumber: number, + timestamp: string, + counterparty: string | null, + memo: string | null, + }>({ + tableId: "bulldozer-example-ledger-entries", + }); + + // Group the ledger by account. + const entriesByAccount = declareGroupByTable({ + tableId: "bulldozer-example-ledger-entries-by-account", + fromTable: ledgerEntries, + groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`), + }); + + // Group the ledger by asset symbol. + const entriesByAsset = declareGroupByTable({ + tableId: "bulldozer-example-ledger-entries-by-asset", + fromTable: ledgerEntries, + groupBy: mapper(`"rowData"->'asset' AS "groupKey"`), + }); + + // Enrich account-grouped rows with normalized direction and numeric amount. + const accountEntriesNormalized = declareMapTable({ + tableId: "bulldozer-example-ledger-account-entries-normalized", + fromTable: entriesByAccount, + mapper: mapper(` + ("rowData"->'accountId') AS "accountId", + ("rowData"->'asset') AS "asset", + ("rowData"->'side') AS "side", + (("rowData"->>'amount')::numeric) AS "amountNumeric", + CASE + WHEN "rowData"->>'side' = 'credit' THEN 'inflow' + ELSE 'outflow' + END AS "flowDirection", + ("rowData"->'txHash') AS "txHash", + ("rowData"->'timestamp') AS "timestamp" + `), + }); + + // Fan out each ledger entry into two directional legs for downstream views. + const accountEntryLegs = declareFlatMapTable({ + tableId: "bulldozer-example-ledger-account-entry-legs", + fromTable: entriesByAccount, + mapper: mapper(` + jsonb_build_array( + jsonb_build_object( + 'accountId', "rowData"->'accountId', + 'asset', "rowData"->'asset', + 'legType', 'entry', + 'signedAmount', + CASE + WHEN "rowData"->>'side' = 'credit' THEN (("rowData"->>'amount')::numeric) + ELSE -(("rowData"->>'amount')::numeric) + END, + 'txHash', "rowData"->'txHash' + ), + jsonb_build_object( + 'accountId', "rowData"->'accountId', + 'asset', "rowData"->'asset', + 'legType', 'counterparty', + 'signedAmount', + CASE + WHEN "rowData"->>'side' = 'credit' THEN -(("rowData"->>'amount')::numeric) + ELSE (("rowData"->>'amount')::numeric) + END, + 'txHash', "rowData"->'txHash' + ) + ) AS "rows" + `), + }); + + // Build an account+asset partition from normalized entries. + const accountAssetPartitions = declareGroupByTable({ + tableId: "bulldozer-example-ledger-account-asset-partitions", + fromTable: accountEntriesNormalized, + groupBy: mapper(` + jsonb_build_object( + 'accountId', "rowData"->'accountId', + 'asset', "rowData"->'asset' + ) AS "groupKey" + `), + }); + + // Keep only entries with a non-null counterparty for suspicious-flow style inspections. + const accountEntriesWithCounterparty = declareFilterTable({ + tableId: "bulldozer-example-ledger-account-entries-with-counterparty", + fromTable: entriesByAccount, + filter: predicate(`("rowData"->>'counterparty') IS NOT NULL`), + }); + const accountEntriesSortedByAmount = declareSortTable({ + tableId: "bulldozer-example-ledger-account-entries-sorted-by-amount", + fromTable: entriesByAccount, + getSortKey: mapper(`(("rowData"->>'amount')::numeric) AS "newSortKey"`), + compareSortKeys: (a, b) => ({ type: "expression", sql: `(((${a.sql}) #>> '{}')::numeric > ((${b.sql}) #>> '{}')::numeric)::int - (((${a.sql}) #>> '{}')::numeric < ((${b.sql}) #>> '{}')::numeric)::int` }), + }); + // Keep a small account-local sample used as reference counterparties for joins. + const accountCounterpartySample = declareLimitTable({ + tableId: "bulldozer-example-ledger-account-counterparty-sample", + fromTable: accountEntriesWithCounterparty, + limit: { type: "expression", sql: "3" }, + }); + // For each counterparty row, join to sampled rows by a computed equality key + // (counterparty + asset). This demonstrates join-key-based reference matching. + const accountCounterpartyJoinedSample = declareLeftJoinTable({ + tableId: "bulldozer-example-ledger-account-counterparty-joined-sample", + leftTable: accountEntriesWithCounterparty, + rightTable: accountCounterpartySample, + leftJoinKey: mapper(` + jsonb_build_object( + 'counterparty', "rowData"->'counterparty', + 'asset', "rowData"->'asset' + ) AS "joinKey" + `), + rightJoinKey: mapper(` + jsonb_build_object( + 'counterparty', "rowData"->'counterparty', + 'asset', "rowData"->'asset' + ) AS "joinKey" + `), + }); + const accountEntriesRunningExposure = declareLFoldTable({ + tableId: "bulldozer-example-ledger-account-entries-running-exposure", + fromTable: accountEntriesSortedByAmount, + initialState: { type: "expression", sql: "'0'::jsonb" }, + reducer: mapper(` + ( + COALESCE(("oldState"#>>'{}')::numeric, 0) + + ( + CASE + WHEN "oldRowData"->>'side' = 'credit' THEN (("oldRowData"->>'amount')::numeric) + ELSE -(("oldRowData"->>'amount')::numeric) + END + ) + ) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'accountId', "oldRowData"->'accountId', + 'asset', "oldRowData"->'asset', + 'txHash', "oldRowData"->'txHash', + 'delta', + CASE + WHEN "oldRowData"->>'side' = 'credit' THEN (("oldRowData"->>'amount')::numeric) + ELSE -(("oldRowData"->>'amount')::numeric) + END, + 'runningExposure', + ( + COALESCE(("oldState"#>>'{}')::numeric, 0) + + ( + CASE + WHEN "oldRowData"->>'side' = 'credit' THEN (("oldRowData"->>'amount')::numeric) + ELSE -(("oldRowData"->>'amount')::numeric) + END + ) + ) + ) + ) AS "newRowsData" + `), + }); + // Timefold reducers should avoid non-deterministic values (for example now()/random()) for + // output-driving fields, otherwise replaying from scratch can produce different results. + // These examples derive next timestamps from stable row timestamps. + const accountEntriesTimedExposure = declareTimeFoldTable({ + tableId: "bulldozer-example-ledger-account-entries-timed-exposure", + fromTable: entriesByAccount, + initialState: { type: "expression", sql: "'0'::jsonb" }, + reducer: mapper(` + ( + COALESCE(("oldState"#>>'{}')::numeric, 0) + + ( + CASE + WHEN "oldRowData"->>'side' = 'credit' THEN (("oldRowData"->>'amount')::numeric) + ELSE -(("oldRowData"->>'amount')::numeric) + END + ) + ) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'accountId', "oldRowData"->'accountId', + 'asset', "oldRowData"->'asset', + 'txHash', "oldRowData"->'txHash', + 'timedExposure', + ( + COALESCE(("oldState"#>>'{}')::numeric, 0) + + ( + CASE + WHEN "oldRowData"->>'side' = 'credit' THEN (("oldRowData"->>'amount')::numeric) + ELSE -(("oldRowData"->>'amount')::numeric) + END + ) + ), + 'tickTimestamp', + CASE + WHEN "timestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("timestamp") + END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (("oldRowData"->>'timestamp')::timestamptz + interval '5 minutes') + ELSE NULL::timestamptz + END AS "nextTimestamp" + `), + }); + // Emit repeated timed checkpoints for each row until a bounded step counter + // reaches completion. This showcases recurring scheduling behavior. + const accountEntriesTimedReprice = declareTimeFoldTable({ + tableId: "bulldozer-example-ledger-account-entries-timed-reprice", + fromTable: entriesByAccount, + initialState: { type: "expression", sql: "'0'::jsonb" }, + reducer: mapper(` + CASE + WHEN "timestamp" IS NULL THEN 1 + WHEN COALESCE(("oldState"#>>'{}')::int, 0) < 3 THEN (COALESCE(("oldState"#>>'{}')::int, 0) + 1) + ELSE COALESCE(("oldState"#>>'{}')::int, 0) + END AS "newState", + jsonb_build_array( + jsonb_build_object( + 'accountId', "oldRowData"->'accountId', + 'asset', "oldRowData"->'asset', + 'txHash', "oldRowData"->'txHash', + 'amount', (("oldRowData"->>'amount')::numeric), + 'step', + CASE + WHEN "timestamp" IS NULL THEN 1 + ELSE COALESCE(("oldState"#>>'{}')::int, 0) + END, + 'mode', + CASE + WHEN "timestamp" IS NULL THEN 'initial' + WHEN COALESCE(("oldState"#>>'{}')::int, 0) < 3 THEN 'follow-up' + ELSE 'terminal' + END, + 'tickTimestamp', + CASE + WHEN "timestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("timestamp") + END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (("oldRowData"->>'timestamp')::timestamptz + interval '1 minute') + WHEN COALESCE(("oldState"#>>'{}')::int, 0) < 3 THEN ("timestamp" + interval '1 minute') + ELSE NULL::timestamptz + END AS "nextTimestamp" + `), + }); + + // Keep only large-value entries to model risk/alerting-style subsets. + const highValueEntriesByAsset = declareFilterTable({ + tableId: "bulldozer-example-ledger-high-value-entries-by-asset", + fromTable: entriesByAsset, + filter: predicate(`(("rowData"->>'amount')::numeric) >= 1000`), + }); + + // Partition high-value entries by account for analyst-friendly slices. + const highValueEntriesByAssetAccount = declareGroupByTable({ + tableId: "bulldozer-example-ledger-high-value-entries-by-asset-account", + fromTable: highValueEntriesByAsset, + groupBy: mapper(`"rowData"->'accountId' AS "groupKey"`), + }); + const accountPriorityEntries = declareConcatTable({ + tableId: "bulldozer-example-ledger-account-priority-entries", + tables: [accountEntriesWithCounterparty, highValueEntriesByAssetAccount], + }); + const highValueEntriesByAssetAccountTop = declareLimitTable({ + tableId: "bulldozer-example-ledger-high-value-entries-by-asset-account-top", + fromTable: highValueEntriesByAssetAccount, + limit: { type: "expression", sql: "3" }, + }); + + // Enrich asset-grouped rows for downstream analytics views. + const assetEntriesNormalized = declareMapTable({ + tableId: "bulldozer-example-ledger-asset-entries-normalized", + fromTable: entriesByAsset, + mapper: mapper(` + ("rowData"->'asset') AS "asset", + ("rowData"->'accountId') AS "accountId", + (("rowData"->>'amount')::numeric) AS "amountNumeric", + CASE + WHEN "rowData"->>'side' = 'credit' THEN 1 + ELSE -1 + END AS "signedDirection", + ("rowData"->'blockNumber') AS "blockNumber", + ("rowData"->'txHash') AS "txHash" + `), + }); + + return { + ledgerEntries, + entriesByAccount, + entriesByAsset, + accountEntriesNormalized, + accountEntryLegs, + accountAssetPartitions, + accountEntriesWithCounterparty, + accountEntriesSortedByAmount, + accountCounterpartySample, + accountCounterpartyJoinedSample, + accountEntriesRunningExposure, + accountEntriesTimedExposure, + accountEntriesTimedReprice, + highValueEntriesByAsset, + highValueEntriesByAssetAccount, + accountPriorityEntries, + highValueEntriesByAssetAccountTop, + assetEntriesNormalized, + }; +})(); diff --git a/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts new file mode 100644 index 0000000000..db54739e25 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.fuzz.test.ts @@ -0,0 +1,1998 @@ +import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings"; +import postgres from "postgres"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest"; +import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLeftJoinTable, declareLFoldTable, declareLimitTable, declareMapTable, declareSortTable, declareStoredTable, declareTimeFoldTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_fuzz_test"; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; +type SqlMapper = { type: "mapper", sql: string }; +type QueryableTable = { + listGroups(options: { start: "start", end: "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, + listRowsInGroup(options: { groupKey?: SqlExpression, start: "start", end: "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery, +}; +type SourceRow = { team: string | null, value: number }; +type JoinRuleRow = { team: string | null, threshold: number, label: string }; +type TeamMappedRow = { team: string | null, valuePlusTen: number }; +type TeamBucketRow = { team: string | null, valueScaled: number, bucket: string }; +type TeamFlatMappedRow = { team: string | null, kind: string, mappedValue: number }; +type TeamFlatMappedPlusRow = { team: string | null, kind: string, mappedValuePlusOne: number }; +type GroupedRows> = Map }>; +type TraceSectionStats = { + count: number, + totalMs: number, + maxMs: number, + slowestExample: string, +}; +type TraceBucket = { + totalTrackedMs: number, + sections: Map, + slowOps: Array<{ opKind: string, ms: number, detail: string }>, +}; + +const FUZZ_TRACE_ENABLED = (() => { + const env = Reflect.get(import.meta, "env"); + const value = Reflect.get(env, "STACK_BULLDOZER_FUZZ_TRACE") ?? Reflect.get(env, "BULLDOZER_FUZZ_TRACE"); + return value === true || value === "true" || value === "1"; +})(); +const MAX_SLOW_OPS = 20; +const tracesByTest = new Map(); + +function getCurrentTestNameForTrace(): string { + return expect.getState().currentTestName ?? "__unknown_test__"; +} +function getTraceBucket(testName: string): TraceBucket { + const existing = tracesByTest.get(testName); + if (existing != null) return existing; + const created: TraceBucket = { totalTrackedMs: 0, sections: new Map(), slowOps: [] }; + tracesByTest.set(testName, created); + return created; +} +function trimSqlForTrace(input: string): string { + const trimmed = input.replaceAll(/\s+/g, " ").trim(); + if (trimmed.length <= 180) return trimmed; + return `${trimmed.slice(0, 177)}...`; +} +function callerForTrace(fallback: string): string { + const stack = (new Error().stack ?? "").split("\n").map((line) => line.trim()); + const preferred = stack.find((line) => + line.includes("index.fuzz.test.ts") + && !line.includes("callerForTrace") + && !line.includes("traceOperation") + && !line.includes("runStatements") + && !line.includes("readRows") + && !line.includes("readBoolean"), + ); + if (preferred != null) return preferred; + return stack[3] ?? fallback; +} +function traceOperation(options: { opKind: "tx" | "query" | "expr", section: string, ms: number, detail: string }) { + if (!FUZZ_TRACE_ENABLED) return; + const testName = getCurrentTestNameForTrace(); + const bucket = getTraceBucket(testName); + bucket.totalTrackedMs += options.ms; + const key = `${options.opKind}:${options.section}`; + const existing = bucket.sections.get(key); + if (existing != null) { + existing.count += 1; + existing.totalMs += options.ms; + if (options.ms > existing.maxMs) { + existing.maxMs = options.ms; + existing.slowestExample = options.detail; + } + } else { + bucket.sections.set(key, { + count: 1, + totalMs: options.ms, + maxMs: options.ms, + slowestExample: options.detail, + }); + } + bucket.slowOps.push({ + opKind: options.opKind, + ms: options.ms, + detail: `${options.section} :: ${options.detail}`, + }); + bucket.slowOps.sort((a, b) => b.ms - a.ms); + if (bucket.slowOps.length > MAX_SLOW_OPS) { + bucket.slowOps.length = MAX_SLOW_OPS; + } +} + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} +function mapper(sql: string): SqlMapper { + return { type: "mapper", sql }; +} + +function createRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; +} +function choose(rng: () => number, values: readonly T[]): T { + return values[Math.floor(rng() * values.length)] ?? values[0]; +} +function sqlStringLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} +function jsonbLiteral(value: unknown): string { + return `${sqlStringLiteral(JSON.stringify(value))}::jsonb`; +} +function groupDiscriminator(groupKey: string | null): string { + return groupKey === null ? "__NULL__" : `S:${groupKey}`; +} +function nullableStringCompare(a: string | null, b: string | null): number { + if (a === b) return 0; + if (a === null) return -1; + if (b === null) return 1; + return stringCompare(a, b); +} +function groupKeyExpression(groupKey: string | null): SqlExpression { + return groupKey === null + ? expr(`'null'::jsonb`) + : expr(`to_jsonb(${sqlStringLiteral(groupKey)}::text)`); +} +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function computeTeamGroups(rows: Map): GroupedRows<{ team: string | null, value: number }> { + const groups: GroupedRows<{ team: string | null, value: number }> = new Map(); + for (const [rowIdentifier, row] of rows) { + const key = groupDiscriminator(row.team); + const existing = groups.get(key); + if (existing != null) { + existing.rows.set(rowIdentifier, { team: row.team, value: row.value }); + } else { + groups.set(key, { + groupKey: row.team, + rows: new Map([[rowIdentifier, { team: row.team, value: row.value }]]), + }); + } + } + return groups; +} +function computeRuleGroups(rows: Map): GroupedRows { + const groups: GroupedRows = new Map(); + for (const [rowIdentifier, row] of rows) { + const key = groupDiscriminator(row.team); + const existing = groups.get(key); + if (existing != null) { + existing.rows.set(rowIdentifier, { team: row.team, threshold: row.threshold, label: row.label }); + } else { + groups.set(key, { + groupKey: row.team, + rows: new Map([[rowIdentifier, { team: row.team, threshold: row.threshold, label: row.label }]]), + }); + } + } + return groups; +} +function mapGroups, NewRow extends Record>( + groups: GroupedRows, + mapperFn: (row: OldRow) => NewRow, +): GroupedRows { + const mapped: GroupedRows = new Map(); + for (const [groupKey, group] of groups) { + mapped.set(groupKey, { + groupKey: group.groupKey, + rows: new Map([...group.rows.entries()].map(([rowIdentifier, rowData]) => [`${rowIdentifier}:1`, mapperFn(rowData)])), + }); + } + return mapped; +} +function regroupByField>( + groups: GroupedRows, + groupKeySelector: (row: T) => string | null, +): GroupedRows { + const regrouped: GroupedRows = new Map(); + for (const group of groups.values()) { + for (const [rowIdentifier, rowData] of group.rows) { + const groupKey = groupKeySelector(rowData); + const key = groupDiscriminator(groupKey); + const existing = regrouped.get(key); + if (existing != null) { + existing.rows.set(rowIdentifier, rowData); + } else { + regrouped.set(key, { + groupKey, + rows: new Map([[rowIdentifier, rowData]]), + }); + } + } + } + return regrouped; +} +function flatMapGroups, NewRow extends Record>( + groups: GroupedRows, + mapperFn: (row: OldRow) => NewRow[], +): GroupedRows { + const mapped: GroupedRows = new Map(); + for (const [groupKey, group] of groups) { + const rows = new Map(); + for (const [rowIdentifier, rowData] of group.rows) { + const expandedRows = mapperFn(rowData); + for (let i = 0; i < expandedRows.length; i++) { + const expandedRow = expandedRows[i] ?? (() => { + throw new Error("flatMapGroups mapper returned undefined row"); + })(); + rows.set(`${rowIdentifier}:${i + 1}`, expandedRow); + } + } + mapped.set(groupKey, { groupKey: group.groupKey, rows }); + } + return mapped; +} +function filterGroups>( + groups: GroupedRows, + predicateFn: (row: Row) => boolean, +): GroupedRows { + const filtered: GroupedRows = new Map(); + for (const [groupKey, group] of groups) { + const rows = new Map(); + for (const [rowIdentifier, rowData] of group.rows) { + if (!predicateFn(rowData)) continue; + rows.set(`${rowIdentifier}:1`, rowData); + } + filtered.set(groupKey, { groupKey: group.groupKey, rows }); + } + return filtered; +} +function limitGroups>( + groups: GroupedRows, + limit: number, +): GroupedRows { + const limited: GroupedRows = new Map(); + for (const [groupKey, group] of groups) { + const rows = new Map(); + const sortedRows = [...group.rows.entries()].sort((a, b) => stringCompare(a[0], b[0])); + for (let i = 0; i < Math.min(limit, sortedRows.length); i++) { + const entry = sortedRows[i] ?? (() => { + throw new Error("limitGroups expected sorted row entry to exist"); + })(); + rows.set(entry[0], entry[1]); + } + limited.set(groupKey, { groupKey: group.groupKey, rows }); + } + return limited; +} +function concatGroups>( + groupsList: GroupedRows[], +): GroupedRows { + const concatenated: GroupedRows = new Map(); + for (let tableIndex = 0; tableIndex < groupsList.length; tableIndex++) { + const groups = groupsList[tableIndex] ?? (() => { + throw new Error("concatGroups expected grouped rows for table index"); + })(); + for (const [groupKey, group] of groups) { + const existing = concatenated.get(groupKey) ?? { groupKey: group.groupKey, rows: new Map() }; + for (const [rowIdentifier, rowData] of group.rows) { + existing.rows.set(`${tableIndex}:${rowIdentifier}`, rowData); + } + concatenated.set(groupKey, existing); + } + } + return concatenated; +} +function leftJoinRowIdentifier(leftRowIdentifier: string, rightRowIdentifier: string | null): string { + return `[${JSON.stringify(leftRowIdentifier)}, ${rightRowIdentifier === null ? "null" : JSON.stringify(rightRowIdentifier)}]`; +} +function leftJoinGroups< + FromRow extends Record, + JoinRow extends Record, +>( + fromGroups: GroupedRows, + joinGroups: GroupedRows, + leftJoinKeyFn: (fromRow: FromRow) => unknown, + rightJoinKeyFn: (joinRow: JoinRow) => unknown, +): GroupedRows> { + const joined: GroupedRows> = new Map(); + for (const [groupKey, fromGroup] of fromGroups) { + const joinGroup = joinGroups.get(groupKey); + const rows = new Map>(); + const sortedFromRows = [...fromGroup.rows.entries()].sort((a, b) => stringCompare(a[0], b[0])); + const sortedJoinRows = joinGroup == null + ? [] + : [...joinGroup.rows.entries()].sort((a, b) => stringCompare(a[0], b[0])); + for (const [leftRowIdentifier, leftRowData] of sortedFromRows) { + const leftJoinKey = JSON.stringify(leftJoinKeyFn(leftRowData)); + const matches = sortedJoinRows.filter((joinEntry) => JSON.stringify(rightJoinKeyFn(joinEntry[1])) === leftJoinKey); + if (matches.length === 0) { + rows.set(leftJoinRowIdentifier(leftRowIdentifier, null), { + leftRowData: { ...leftRowData }, + rightRowData: null, + }); + continue; + } + for (const [rightRowIdentifier, rightRowData] of matches) { + rows.set(leftJoinRowIdentifier(leftRowIdentifier, rightRowIdentifier), { + leftRowData: { ...leftRowData }, + rightRowData: { ...rightRowData }, + }); + } + } + joined.set(groupKey, { groupKey: fromGroup.groupKey, rows }); + } + return joined; +} +function sortedRowsForGroups>(groups: GroupedRows) { + return [...groups.values()].flatMap((group) => { + return [...group.rows.entries()] + .sort((a, b) => { + const leftValue = Number(Reflect.get(a[1], "value")); + const rightValue = Number(Reflect.get(b[1], "value")); + return leftValue - rightValue || stringCompare(a[0], b[0]); + }) + .map(([rowIdentifier, rowData]) => ({ + groupKey: group.groupKey, + rowIdentifier, + rowSortKey: Number(Reflect.get(rowData, "value")), + rowData, + })); + }); +} +function lFoldGroupsForSortedInput(groups: GroupedRows<{ team: string | null, value: number }>) { + const folded: GroupedRows<{ kind: string, runningTotal: number, value: number }> = new Map(); + for (const [groupKey, group] of groups) { + const rows = new Map(); + let runningTotal = 0; + const sortedEntries = [...group.rows.entries()].sort((a, b) => { + const byValue = (a[1].value - b[1].value); + return byValue !== 0 ? byValue : stringCompare(a[0], b[0]); + }); + for (const [rowIdentifier, rowData] of sortedEntries) { + runningTotal += rowData.value; + rows.set(`${rowIdentifier}:1`, { + kind: "running", + runningTotal, + value: rowData.value, + }); + if (rowData.value % 2 === 0) { + rows.set(`${rowIdentifier}:2`, { + kind: "even-marker", + runningTotal, + value: rowData.value, + }); + } + } + folded.set(groupKey, { groupKey: group.groupKey, rows }); + } + return folded; +} +function lFoldRowsWithSortKeys(groups: GroupedRows<{ team: string | null, value: number }>) { + const rows: Array<{ groupKey: string | null, rowIdentifier: string, rowSortKey: number, rowData: { kind: string, runningTotal: number, value: number } }> = []; + for (const group of groups.values()) { + let runningTotal = 0; + const sortedEntries = [...group.rows.entries()].sort((a, b) => { + const byValue = (a[1].value - b[1].value); + return byValue !== 0 ? byValue : stringCompare(a[0], b[0]); + }); + for (const [rowIdentifier, rowData] of sortedEntries) { + runningTotal += rowData.value; + rows.push({ + groupKey: group.groupKey, + rowIdentifier: `${rowIdentifier}:1`, + rowSortKey: rowData.value, + rowData: { kind: "running", runningTotal, value: rowData.value }, + }); + if (rowData.value % 2 === 0) { + rows.push({ + groupKey: group.groupKey, + rowIdentifier: `${rowIdentifier}:2`, + rowSortKey: rowData.value, + rowData: { kind: "even-marker", runningTotal, value: rowData.value }, + }); + } + } + } + return rows.sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + if (byGroup !== 0) return byGroup; + const bySort = a.rowSortKey - b.rowSortKey; + if (bySort !== 0) return bySort; + return stringCompare(a.rowIdentifier, b.rowIdentifier); + }); +} +function timeFoldGroupsForSourceInput(groups: GroupedRows<{ team: string | null, value: number }>) { + const folded: GroupedRows<{ runningTotal: number, value: number, timestamp: null }> = new Map(); + for (const [groupKey, group] of groups) { + const rows = new Map(); + for (const [rowIdentifier, rowData] of group.rows) { + rows.set(`${rowIdentifier}:1`, { + runningTotal: rowData.value, + value: rowData.value, + timestamp: null, + }); + } + folded.set(groupKey, { groupKey: group.groupKey, rows }); + } + return folded; +} + +describe.sequential("bulldozer db fuzz composition (real postgres)", () => { + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + + async function runStatements(statements: SqlStatement[], traceSection?: string) { + const txSql = toExecutableSqlTransaction(statements); + const startedAt = performance.now(); + await sql.unsafe(txSql); + const elapsedMs = performance.now() - startedAt; + let rowCountDetail = ""; + if (FUZZ_TRACE_ENABLED) { + const countRows = await sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM "BulldozerStorageEngine"`); + if (countRows.length === 0) { + throw new Error("expected count row for BulldozerStorageEngine"); + } + const firstCountRow = countRows[0]; + rowCountDetail = ` storageRows=${Number(firstCountRow.count)}`; + } + const detail = `statements=${statements.length} txSqlChars=${txSql.length}${rowCountDetail} first=${trimSqlForTrace(statements[0]?.sql ?? "none")}`; + traceOperation({ + opKind: "tx", + section: traceSection ?? callerForTrace("runStatements"), + ms: elapsedMs, + detail, + }); + } + async function readBoolean(expression: SqlExpression, traceSection?: string) { + const startedAt = performance.now(); + const rows = await sql.unsafe(`SELECT (${expression.sql}) AS "value"`); + const elapsedMs = performance.now() - startedAt; + traceOperation({ + opKind: "expr", + section: traceSection ?? callerForTrace("readBoolean"), + ms: elapsedMs, + detail: trimSqlForTrace(expression.sql), + }); + return rows[0].value === true; + } + async function readRows(query: SqlQuery, traceSection?: string) { + const startedAt = performance.now(); + const rows = await sql.unsafe(toQueryableSqlQuery(query)); + const elapsedMs = performance.now() - startedAt; + traceOperation({ + opKind: "query", + section: traceSection ?? callerForTrace("readRows"), + ms: elapsedMs, + detail: trimSqlForTrace(toQueryableSqlQuery(query)), + }); + return rows; + } + + async function assertTableMatches>(table: QueryableTable, expected: GroupedRows) { + const tableLabel = (() => { + const maybeRecord = table as unknown; + if (isRecord(maybeRecord)) { + const debugArgs = Reflect.get(maybeRecord, "debugArgs"); + if (isRecord(debugArgs)) { + const tableId = Reflect.get(debugArgs, "tableId"); + const operator = Reflect.get(debugArgs, "operator"); + if (typeof tableId === "string" && typeof operator === "string") { + return `${operator}:${tableId}`; + } + } + } + return "table"; + })(); + const expectedGroups = [...expected.values()] + .filter((group) => group.rows.size > 0) + .map((group) => group.groupKey) + .sort(nullableStringCompare); + + const actualGroups = (await readRows(table.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }), `${tableLabel}.listGroups`)) + .map((row) => row.groupkey as string | null) + .sort(nullableStringCompare); + + expect(actualGroups).toEqual(expectedGroups); + + const expectedAllRows = [...expected.values()] + .flatMap((group) => [...group.rows.entries()].map(([rowIdentifier, rowData]) => ({ groupKey: group.groupKey, rowIdentifier, rowData }))) + .sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + return byGroup !== 0 ? byGroup : stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + + const actualAllRows = (await readRows(table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }), `${tableLabel}.listRowsInGroup(all)`)) + .map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: row.rowidentifier as string, + rowData: row.rowdata as Record, + })) + .sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + return byGroup !== 0 ? byGroup : stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + + expect(actualAllRows).toEqual(expectedAllRows); + + for (const expectedGroup of expected.values()) { + if (expectedGroup.rows.size === 0) continue; + const expectedRows = [...expectedGroup.rows.entries()] + .map(([rowIdentifier, rowData]) => ({ rowIdentifier, rowData })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + const actualRows = (await readRows(table.listRowsInGroup({ + groupKey: groupKeyExpression(expectedGroup.groupKey), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }), `${tableLabel}.listRowsInGroup(group)`)) + .map((row) => ({ rowIdentifier: row.rowidentifier as string, rowData: row.rowdata as Record })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + expect(actualRows).toEqual(expectedRows); + } + + const missingRows = await readRows(table.listRowsInGroup({ + groupKey: groupKeyExpression("__missing_group__"), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }), `${tableLabel}.listRowsInGroup(missing)`); + expect(missingRows).toEqual([]); + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + const createExtensionStartedAt = performance.now(); + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + traceOperation({ + opKind: "query", + section: "beforeEach.createExtension", + ms: performance.now() - createExtensionStartedAt, + detail: "CREATE EXTENSION IF NOT EXISTS pgcrypto", + }); + + const dropTableStartedAt = performance.now(); + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + traceOperation({ + opKind: "query", + section: "beforeEach.dropTable", + ms: performance.now() - dropTableStartedAt, + detail: `DROP TABLE IF EXISTS "BulldozerStorageEngine"`, + }); + + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldQueue"`; + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldMetadata"`; + + const createTableStartedAt = performance.now(); + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + traceOperation({ + opKind: "query", + section: "beforeEach.createTable", + ms: performance.now() - createTableStartedAt, + detail: `CREATE TABLE "BulldozerStorageEngine"`, + }); + + const createIndexStartedAt = performance.now(); + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + traceOperation({ + opKind: "query", + section: "beforeEach.createIndex", + ms: performance.now() - createIndexStartedAt, + detail: `CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx"`, + }); + + const seedRootsStartedAt = performance.now(); + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; + traceOperation({ + opKind: "query", + section: "beforeEach.seedRoots", + ms: performance.now() - seedRootsStartedAt, + detail: `INSERT root key paths`, + }); + + await sql` + CREATE TABLE "BulldozerTimeFoldQueue" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "tableStoragePath" JSONB[] NOT NULL, + "groupKey" JSONB NOT NULL, + "rowIdentifier" TEXT NOT NULL, + "scheduledAt" TIMESTAMPTZ NOT NULL, + "stateAfter" JSONB NOT NULL, + "rowData" JSONB NOT NULL, + "reducerSql" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "BulldozerTimeFoldQueue_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerTimeFoldQueue_table_group_row_key" UNIQUE ("tableStoragePath", "groupKey", "rowIdentifier") + ) + `; + await sql`CREATE INDEX "BulldozerTimeFoldQueue_scheduledAt_idx" ON "BulldozerTimeFoldQueue"("scheduledAt")`; + await sql` + CREATE TABLE "BulldozerTimeFoldMetadata" ( + "key" TEXT PRIMARY KEY, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastProcessedAt" TIMESTAMPTZ NOT NULL + ) + `; + await sql` + INSERT INTO "BulldozerTimeFoldMetadata" ("key", "lastProcessedAt") + VALUES ('singleton', now()) + `; + }); + + afterEach(() => { + if (!FUZZ_TRACE_ENABLED) return; + const testName = getCurrentTestNameForTrace(); + const bucket = tracesByTest.get(testName); + if (bucket == null) return; + + const topSections = [...bucket.sections.entries()] + .sort((a, b) => b[1].totalMs - a[1].totalMs) + .slice(0, 12); + const topOps = bucket.slowOps.slice(0, 12); + + console.log(`\n[bulldozer-fuzz-trace] ${testName}`); + console.log(`[bulldozer-fuzz-trace] tracked_total_ms=${bucket.totalTrackedMs.toFixed(1)} sections=${bucket.sections.size}`); + for (const [sectionName, stats] of topSections) { + console.log( + `[bulldozer-fuzz-trace] section=${sectionName} count=${stats.count} total_ms=${stats.totalMs.toFixed(1)} avg_ms=${(stats.totalMs / stats.count).toFixed(2)} max_ms=${stats.maxMs.toFixed(1)} slowest="${stats.slowestExample}"`, + ); + } + for (const op of topOps) { + console.log( + `[bulldozer-fuzz-trace] slow_op kind=${op.opKind} ms=${op.ms.toFixed(1)} detail="${op.detail}"`, + ); + } + + tracesByTest.delete(testName); + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + test("fuzz: stacked group/map/group pipelines preserve invariants under random mutations", async () => { + const identifiers = ["u1", "u2", "u3", "u4", "u:5", "u 6", "u/7", "u'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [101]) { + const rng = createRng(seed); + const sourceRows = new Map(); + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const mapTable1 = declareMapTable({ + tableId: `fuzz-users-map-level-1-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen" + `), + }); + const mapTable2 = declareMapTable({ + tableId: `fuzz-users-map-level-2-${seed}`, + fromTable: mapTable1, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", + ( + CASE + WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + `), + }); + const groupedByBucket = declareGroupByTable({ + tableId: `fuzz-users-by-bucket-${seed}`, + fromTable: mapTable2, + groupBy: mapper(`"rowData"->'bucket' AS "groupKey"`), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTable1.init()); + await runStatements(mapTable2.init()); + await runStatements(groupedByBucket.init()); + + for (let step = 0; step < 24; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 50), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.94) { + await runStatements(groupedByBucket.delete()); + await runStatements(mapTable2.delete()); + await runStatements(mapTable1.delete()); + await runStatements(mapTable1.init()); + await runStatements(mapTable2.init()); + await runStatements(groupedByBucket.init()); + } else { + const rowIdentifier = choose(rng, identifiers); + const rowData = sourceRows.get(rowIdentifier); + if (rowData != null) { + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else { + await runStatements(fromTable.deleteRow(rowIdentifier)); + } + } + + if (step % 3 === 0 || step === 23) { + const expectedGrouped = computeTeamGroups(sourceRows); + const expectedMap1 = mapGroups(expectedGrouped, (row): TeamMappedRow => ({ + team: (row.team as string | null), + valuePlusTen: (row.value as number) + 10, + })); + const expectedMap2 = mapGroups(expectedMap1, (row): TeamBucketRow => { + const valueScaled = (row.valuePlusTen as number) * 2; + return { + team: (row.team as string | null), + valueScaled, + bucket: valueScaled >= 30 ? "high" : "low", + }; + }); + const expectedBucket = regroupByField(expectedMap2, (row) => row.bucket as string); + + await assertTableMatches(groupedTable, expectedGrouped); + await assertTableMatches(mapTable1, expectedMap1); + await assertTableMatches(mapTable2, expectedMap2); + await assertTableMatches(groupedByBucket, expectedBucket); + } + } + } + }, 120_000); + + test("fuzz: flatMap/map/group pipelines preserve invariants under random mutations and re-inits", async () => { + const identifiers = ["f1", "f2", "f3", "f4", "f:5", "f 6", "f/7", "f'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [501]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let pipelineInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `flat-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `flat-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const flatMapTable = declareFlatMapTable({ + tableId: `flat-fuzz-users-expanded-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + CASE + WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb + ELSE jsonb_build_array( + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'base', + 'mappedValue', (("rowData"->>'value')::int + 100) + ), + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'double', + 'mappedValue', (("rowData"->>'value')::int * 2) + ) + ) + END AS "rows" + `), + }); + const mapAfterFlat = declareMapTable({ + tableId: `flat-fuzz-users-expanded-plus-${seed}`, + fromTable: flatMapTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ("rowData"->'kind') AS "kind", + (("rowData"->>'mappedValue')::int + 1) AS "mappedValuePlusOne" + `), + }); + const groupedByKind = declareGroupByTable({ + tableId: `flat-fuzz-users-by-kind-${seed}`, + fromTable: mapAfterFlat, + groupBy: mapper(`"rowData"->'kind' AS "groupKey"`), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMapTable.init()); + await runStatements(mapAfterFlat.init()); + await runStatements(groupedByKind.init()); + + for (let step = 0; step < 24; step++) { + const roll = rng(); + if (roll < 0.6) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 80) - 20, + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.84) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.92) { + if (pipelineInitialized) { + await runStatements(groupedByKind.delete()); + await runStatements(mapAfterFlat.delete()); + await runStatements(flatMapTable.delete()); + pipelineInitialized = false; + } + } else { + if (!pipelineInitialized) { + await runStatements(flatMapTable.init()); + await runStatements(mapAfterFlat.init()); + await runStatements(groupedByKind.init()); + pipelineInitialized = true; + } + } + + if (step % 3 === 0 || step === 23) { + const expectedGrouped = computeTeamGroups(sourceRows); + const expectedFlat = flatMapGroups(expectedGrouped, (row): TeamFlatMappedRow[] => { + if ((row.value as number) < 0) return []; + return [ + { + team: row.team as string | null, + kind: "base", + mappedValue: (row.value as number) + 100, + }, + { + team: row.team as string | null, + kind: "double", + mappedValue: (row.value as number) * 2, + }, + ]; + }); + const expectedMapped = mapGroups(expectedFlat, (row): TeamFlatMappedPlusRow => ({ + team: row.team as string | null, + kind: row.kind as string, + mappedValuePlusOne: (row.mappedValue as number) + 1, + })); + const expectedKind = regroupByField(expectedMapped, (row) => row.kind as string); + + await assertTableMatches(groupedTable, expectedGrouped); + if (pipelineInitialized) { + expect(await readBoolean(flatMapTable.isInitialized())).toBe(true); + expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(true); + expect(await readBoolean(groupedByKind.isInitialized())).toBe(true); + await assertTableMatches(flatMapTable, expectedFlat); + await assertTableMatches(mapAfterFlat, expectedMapped); + await assertTableMatches(groupedByKind, expectedKind); + } else { + expect(await readBoolean(flatMapTable.isInitialized())).toBe(false); + expect(await readBoolean(mapAfterFlat.isInitialized())).toBe(false); + expect(await readBoolean(groupedByKind.isInitialized())).toBe(false); + + const flatGroups = await readRows(flatMapTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const mappedGroups = await readRows(mapAfterFlat.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const kindGroups = await readRows(groupedByKind.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(flatGroups).toEqual([]); + expect(mappedGroups).toEqual([]); + expect(kindGroups).toEqual([]); + } + } + } + } + }, 120_000); + + test("fuzz: filter/map pipelines preserve invariants under random mutations and re-inits", async () => { + const identifiers = ["ff1", "ff2", "ff3", "ff:4", "ff 5"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [701]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let filterPipelineInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `filter-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `filter-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const filterTable = declareFilterTable({ + tableId: `filter-fuzz-users-threshold-${seed}`, + fromTable: groupedTable, + filter: { type: "predicate", sql: `("rowData"->'team') IS NOT NULL AND (("rowData"->>'value')::int) >= 10` }, + }); + const mappedAfterFilter = declareMapTable({ + tableId: `filter-fuzz-users-mapped-${seed}`, + fromTable: filterTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int * 10) AS "scaledValue" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(filterTable.init()); + await runStatements(mappedAfterFilter.init()); + + for (let step = 0; step < 28; step++) { + const roll = rng(); + if (roll < 0.6) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 35) - 5, + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.82) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.9) { + if (filterPipelineInitialized) { + await runStatements(mappedAfterFilter.delete()); + await runStatements(filterTable.delete()); + filterPipelineInitialized = false; + } + } else { + if (!filterPipelineInitialized) { + await runStatements(filterTable.init()); + await runStatements(mappedAfterFilter.init()); + filterPipelineInitialized = true; + } + } + + if (step % 3 === 0 || step === 27) { + const expectedGrouped = computeTeamGroups(sourceRows); + const expectedFiltered = filterGroups(expectedGrouped, (row) => row.team != null && row.value >= 10); + const expectedMapped = mapGroups(expectedFiltered, (row) => { + if (row.team == null) { + throw new Error("expected non-null team after filter predicate"); + } + return { + team: row.team, + scaledValue: row.value * 10, + }; + }); + + await assertTableMatches(groupedTable, expectedGrouped); + if (filterPipelineInitialized) { + expect(await readBoolean(filterTable.isInitialized())).toBe(true); + expect(await readBoolean(mappedAfterFilter.isInitialized())).toBe(true); + await assertTableMatches(filterTable, expectedFiltered); + await assertTableMatches(mappedAfterFilter, expectedMapped); + } else { + expect(await readBoolean(filterTable.isInitialized())).toBe(false); + expect(await readBoolean(mappedAfterFilter.isInitialized())).toBe(false); + expect(await readRows(filterTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + expect(await readRows(mappedAfterFilter.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + } + } + } + } + }, 120_000); + + test("fuzz: grouped limit table remains consistent under random mutations and re-inits", async () => { + const identifiers = ["l1", "l2", "l3", "l4", "l 5", "l:6"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [801]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let limitInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `limit-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `limit-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const limitedByTeam = declareLimitTable({ + tableId: `limit-fuzz-users-top2-${seed}`, + fromTable: groupedTable, + limit: expr(`2`), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(limitedByTeam.init()); + + for (let step = 0; step < 36; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 100), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.93) { + if (limitInitialized) { + await runStatements(limitedByTeam.delete()); + limitInitialized = false; + } + } else { + if (!limitInitialized) { + await runStatements(limitedByTeam.init()); + limitInitialized = true; + } + } + + if (step % 3 === 0 || step === 35) { + const expectedGrouped = computeTeamGroups(sourceRows); + const expectedLimited = limitGroups(expectedGrouped, 2); + await assertTableMatches(groupedTable, expectedGrouped); + if (limitInitialized) { + expect(await readBoolean(limitedByTeam.isInitialized())).toBe(true); + await assertTableMatches(limitedByTeam, expectedLimited); + } else { + expect(await readBoolean(limitedByTeam.isInitialized())).toBe(false); + expect(await readRows(limitedByTeam.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + } + } + } + } + }, 120_000); + + test("fuzz: virtual concat table preserves prefixed rows across parallel source mutations", async () => { + const identifiers = ["c1", "c2", "c3", "c:4", "c 5", "c/6", "c'7"] as const; + const teams = ["alpha", "beta", "gamma"] as const; + + for (const seed of [1801]) { + const rng = createRng(seed); + const sourceRowsA = new Map(); + const sourceRowsB = new Map(); + let secondInputInitialized = true; + let concatInitialized = true; + + const fromTableA = declareStoredTable<{ value: number, team: string | null }>({ tableId: `concat-fuzz-users-a-${seed}` }); + const fromTableB = declareStoredTable<{ value: number, team: string | null }>({ tableId: `concat-fuzz-users-b-${seed}` }); + const groupedTableA = declareGroupByTable({ + tableId: `concat-fuzz-users-a-by-team-${seed}`, + fromTable: fromTableA, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableB = declareGroupByTable({ + tableId: `concat-fuzz-users-b-by-team-${seed}`, + fromTable: fromTableB, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const concatenatedTable = declareConcatTable({ + tableId: `concat-fuzz-users-by-team-${seed}`, + tables: [groupedTableA, groupedTableB], + }); + + await runStatements(fromTableA.init()); + await runStatements(fromTableB.init()); + await runStatements(groupedTableA.init()); + await runStatements(groupedTableB.init()); + await runStatements(concatenatedTable.init()); + + for (let step = 0; step < 24; step++) { + const roll = rng(); + const mutateTableA = roll < 0.42; + const mutateTableB = roll >= 0.42 && roll < 0.84; + const targetRows = mutateTableA ? sourceRowsA : sourceRowsB; + const targetTable = mutateTableA ? fromTableA : fromTableB; + + if (mutateTableA || mutateTableB) { + if (rng() < 0.68) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 60), + }; + targetRows.set(rowIdentifier, rowData); + await runStatements(targetTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else { + const rowIdentifier = choose(rng, identifiers); + targetRows.delete(rowIdentifier); + await runStatements(targetTable.deleteRow(rowIdentifier)); + } + } else if (roll < 0.90) { + if (secondInputInitialized) { + await runStatements(groupedTableB.delete()); + secondInputInitialized = false; + } + } else if (roll < 0.95) { + if (concatInitialized) { + await runStatements(concatenatedTable.delete()); + concatInitialized = false; + } + } else { + if (!secondInputInitialized) { + await runStatements(groupedTableB.init()); + secondInputInitialized = true; + } else if (!concatInitialized) { + await runStatements(concatenatedTable.init()); + concatInitialized = true; + } + } + + if (step % 3 === 0 || step === 23) { + const expectedA = computeTeamGroups(sourceRowsA); + const expectedB = computeTeamGroups(sourceRowsB); + if (!concatInitialized) { + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false); + const groups = await readRows(concatenatedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } else if (secondInputInitialized) { + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + await assertTableMatches(concatenatedTable, concatGroups([expectedA, expectedB])); + } else { + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + await assertTableMatches(concatenatedTable, concatGroups([expectedA])); + } + } + } + } + }, 120_000); + + test("fuzz: sort table preserves sorted order under random mutations and re-inits", async () => { + const identifiers = ["s1", "s2", "s3", "s4", "s:5", "s 6", "s/7", "s'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [2201]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let sortInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `sort-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `sort-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const sortedTable = declareSortTable({ + tableId: `sort-fuzz-users-sorted-${seed}`, + fromTable: groupedTable, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + + for (let step = 0; step < 24; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 80), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.93) { + if (sortInitialized) { + await runStatements(sortedTable.delete()); + sortInitialized = false; + } + } else { + if (!sortInitialized) { + await runStatements(sortedTable.init()); + sortInitialized = true; + } + } + + if (step % 3 === 0 || step === 23) { + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + if (!sortInitialized) { + expect(await readBoolean(sortedTable.isInitialized())).toBe(false); + expect(await readRows(sortedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + continue; + } + + expect(await readBoolean(sortedTable.isInitialized())).toBe(true); + const actualRows = (await readRows(sortedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: row.rowidentifier as string, + rowSortKey: Number(row.rowsortkey), + rowData: row.rowdata as Record, + })); + expect(actualRows).toEqual(sortedRowsForGroups(expectedGrouped)); + } + } + } + }, 120_000); + + test("fuzz: lfold table preserves folded suffix invariants under random mutations and re-inits", async () => { + const identifiers = ["lf1", "lf2", "lf3", "lf4", "lf:5", "lf 6", "lf/7"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [2601]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let lFoldInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `lfold-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `lfold-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const sortedTable = declareSortTable({ + tableId: `lfold-fuzz-users-sorted-${seed}`, + fromTable: groupedTable, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + const lFoldTable = declareLFoldTable({ + tableId: `lfold-fuzz-users-folded-${seed}`, + fromTable: sortedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + ( + COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int) + ) AS "newState", + ( + CASE + WHEN ((("oldRowData"->>'value')::int) % 2) = 0 THEN jsonb_build_array( + jsonb_build_object( + 'kind', 'running', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ), + jsonb_build_object( + 'kind', 'even-marker', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ) + ) + ELSE jsonb_build_array( + jsonb_build_object( + 'kind', 'running', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ) + ) + END + ) AS "newRowsData" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + + for (let step = 0; step < 30; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 90), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.93) { + if (lFoldInitialized) { + await runStatements(lFoldTable.delete()); + lFoldInitialized = false; + } + } else if (!lFoldInitialized) { + await runStatements(lFoldTable.init()); + lFoldInitialized = true; + } + + if (step % 3 === 0 || step === 29) { + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + const expectedSortedRows = sortedRowsForGroups(expectedGrouped); + const actualSortedRows = (await readRows(sortedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: row.rowidentifier as string, + rowSortKey: Number(row.rowsortkey), + rowData: row.rowdata as Record, + })).sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + if (byGroup !== 0) return byGroup; + const bySortKey = a.rowSortKey - b.rowSortKey; + if (bySortKey !== 0) return bySortKey; + return stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + const sortedExpectedRows = [...expectedSortedRows].sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + if (byGroup !== 0) return byGroup; + const bySortKey = a.rowSortKey - b.rowSortKey; + if (bySortKey !== 0) return bySortKey; + return stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + expect(actualSortedRows).toEqual(sortedExpectedRows); + + if (!lFoldInitialized) { + expect(await readBoolean(lFoldTable.isInitialized())).toBe(false); + expect(await readRows(lFoldTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + continue; + } + + expect(await readBoolean(lFoldTable.isInitialized())).toBe(true); + await assertTableMatches(lFoldTable, lFoldGroupsForSortedInput(expectedGrouped)); + const actualFoldRows = (await readRows(lFoldTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: row.rowidentifier as string, + rowSortKey: Number(row.rowsortkey), + rowData: row.rowdata as { kind: string, runningTotal: number, value: number }, + })).sort((a, b) => { + const byGroup = nullableStringCompare(a.groupKey, b.groupKey); + if (byGroup !== 0) return byGroup; + const bySort = a.rowSortKey - b.rowSortKey; + if (bySort !== 0) return bySort; + return stringCompare(a.rowIdentifier, b.rowIdentifier); + }); + expect(actualFoldRows).toEqual(lFoldRowsWithSortKeys(expectedGrouped)); + } + } + } + }, 120_000); + + test("fuzz: timefold table preserves output and queue invariants under random mutations and re-inits", async () => { + const identifiers = ["tf1", "tf2", "tf3", "tf4", "tf:5", "tf 6", "tf/7"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + + for (const seed of [3601]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let timeFoldInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `timefold-fuzz-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `timefold-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const timeFoldTable = declareTimeFoldTable({ + tableId: `timefold-fuzz-result-${seed}`, + fromTable: groupedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + (("oldRowData"->>'value')::int) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'runningTotal', (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int), + 'timestamp', CASE WHEN "timestamp" IS NULL THEN 'null'::jsonb ELSE to_jsonb("timestamp") END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (now() + interval '15 minutes') + ELSE NULL::timestamptz + END AS "nextTimestamp" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + + for (let step = 0; step < 32; step++) { + const roll = rng(); + if (roll < 0.62) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 90), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.86) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.93) { + if (timeFoldInitialized) { + await runStatements(timeFoldTable.delete()); + timeFoldInitialized = false; + } + } else if (!timeFoldInitialized) { + await runStatements(timeFoldTable.init()); + timeFoldInitialized = true; + } + + if (step % 3 === 0 || step === 31) { + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + if (!timeFoldInitialized) { + expect(await readBoolean(timeFoldTable.isInitialized())).toBe(false); + expect(await readRows(timeFoldTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + const queueRows = await sql>` + SELECT COUNT(*)::int AS "count" + FROM "BulldozerTimeFoldQueue" + `; + const firstRow = queueRows[0]; + expect(firstRow.count).toBe(0); + continue; + } + + expect(await readBoolean(timeFoldTable.isInitialized())).toBe(true); + await assertTableMatches(timeFoldTable, timeFoldGroupsForSourceInput(expectedGrouped)); + + const queueRowsRaw = await sql>>` + SELECT + "rowIdentifier", + "groupKey"#>>'{}' AS "groupKey", + ("stateAfter"#>>'{}')::int AS "stateAfter", + "rowData" + FROM "BulldozerTimeFoldQueue" + ORDER BY "rowIdentifier" + `; + const queueRows = queueRowsRaw.map((row) => ({ + rowIdentifier: (() => { + const raw = Reflect.get(row, "rowIdentifier") ?? Reflect.get(row, "rowidentifier"); + if (typeof raw !== "string") throw new Error("expected queue rowIdentifier string"); + return raw; + })(), + groupKey: (() => { + const raw = Reflect.get(row, "groupKey") ?? Reflect.get(row, "groupkey"); + if (raw === null || typeof raw === "string") return raw; + throw new Error("expected queue groupKey nullable string"); + })(), + stateAfter: (() => { + const raw = Reflect.get(row, "stateAfter") ?? Reflect.get(row, "stateafter"); + if (typeof raw !== "number") throw new Error("expected queue stateAfter number"); + return raw; + })(), + rowData: (() => { + const raw = Reflect.get(row, "rowData") ?? Reflect.get(row, "rowdata"); + if (!isRecord(raw)) throw new Error("expected queue rowData object"); + const teamRaw = Reflect.get(raw, "team"); + const valueRaw = Reflect.get(raw, "value"); + if (!(teamRaw === null || typeof teamRaw === "string")) { + throw new Error("expected queue rowData.team nullable string"); + } + if (typeof valueRaw !== "number") { + throw new Error("expected queue rowData.value number"); + } + return { team: teamRaw, value: valueRaw }; + })(), + })); + const expectedQueueRows = [...sourceRows.entries()] + .map(([rowIdentifier, rowData]) => ({ + rowIdentifier, + groupKey: rowData.team, + stateAfter: rowData.value, + rowData, + })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + const sortedQueueRows = [...queueRows].sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + expect(sortedQueueRows).toEqual(expectedQueueRows); + } + } + } + }, 120_000); + + test("fuzz: left join table preserves join invariants under random mutations and re-inits", async () => { + const userIdentifiers = ["lj-u1", "lj-u2", "lj-u3", "lj-u4", "lj-u:5", "lj-u 6"] as const; + const ruleIdentifiers = ["lj-r1", "lj-r2", "lj-r3", "lj-r4", "lj-r:5", "lj-r 6"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + const labels = ["bronze", "silver", "gold", "vip"] as const; + + for (const seed of [3001]) { + const rng = createRng(seed); + const sourceRows = new Map(); + const ruleRows = new Map(); + let leftJoinInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `left-join-fuzz-users-${seed}` }); + const joinTable = declareStoredTable<{ team: string | null, threshold: number, label: string }>({ tableId: `left-join-fuzz-rules-${seed}` }); + const groupedFromTable = declareGroupByTable({ + tableId: `left-join-fuzz-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedJoinTable = declareGroupByTable({ + tableId: `left-join-fuzz-rules-by-team-${seed}`, + fromTable: joinTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const leftJoinedTable = declareLeftJoinTable({ + tableId: `left-join-fuzz-result-${seed}`, + leftTable: groupedFromTable, + rightTable: groupedJoinTable, + leftJoinKey: { type: "mapper", sql: `(("rowData"->>'value')::int) AS "joinKey"` }, + rightJoinKey: { type: "mapper", sql: `(("rowData"->>'threshold')::int) AS "joinKey"` }, + }); + + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + + for (let step = 0; step < 36; step++) { + const roll = rng(); + if (roll < 0.42) { + const rowIdentifier = choose(rng, userIdentifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 90), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.56) { + const rowIdentifier = choose(rng, userIdentifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.82) { + const rowIdentifier = choose(rng, ruleIdentifiers); + const rowData: JoinRuleRow = { + team: choose(rng, teams), + threshold: Math.floor(rng() * 90), + label: choose(rng, labels), + }; + ruleRows.set(rowIdentifier, rowData); + await runStatements(joinTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.90) { + const rowIdentifier = choose(rng, ruleIdentifiers); + ruleRows.delete(rowIdentifier); + await runStatements(joinTable.deleteRow(rowIdentifier)); + } else if (roll < 0.95) { + if (leftJoinInitialized) { + await runStatements(leftJoinedTable.delete()); + leftJoinInitialized = false; + } + } else if (!leftJoinInitialized) { + await runStatements(leftJoinedTable.init()); + leftJoinInitialized = true; + } + + if (step % 3 === 0 || step === 35) { + const expectedGroupedFrom = computeTeamGroups(sourceRows); + const expectedGroupedJoin = computeRuleGroups(ruleRows); + await assertTableMatches(groupedFromTable, expectedGroupedFrom); + await assertTableMatches(groupedJoinTable, expectedGroupedJoin); + + if (!leftJoinInitialized) { + expect(await readBoolean(leftJoinedTable.isInitialized())).toBe(false); + expect(await readRows(leftJoinedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + continue; + } + + expect(await readBoolean(leftJoinedTable.isInitialized())).toBe(true); + const expectedLeftJoined = leftJoinGroups( + expectedGroupedFrom, + expectedGroupedJoin, + (fromRow) => Number(Reflect.get(fromRow, "value")), + (joinRow) => Number(Reflect.get(joinRow, "threshold")), + ); + await assertTableMatches(leftJoinedTable, expectedLeftJoined); + } + } + } + }, 120_000); + + test("fuzz: parallel map tables remain isolated with independent re-inits", async () => { + const identifiers = ["m1", "m2", "m3", "m 4", "m:5"] as const; + const teams = ["alpha", "beta", null] as const; + + for (const seed of [401, 402]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let mapAInitialized = true; + let mapBInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `parallel-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `parallel-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const mapTableA = declareMapTable({ + tableId: `parallel-users-map-a-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValueA" + `), + }); + const mapTableB = declareMapTable({ + tableId: `parallel-users-map-b-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ((("rowData"->>'value')::int) * -1) AS "mappedValueB" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTableA.init()); + await runStatements(mapTableB.init()); + + for (let step = 0; step < 50; step++) { + const roll = rng(); + if (roll < 0.6) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 40), + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.82) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.9) { + if (mapAInitialized) { + await runStatements(mapTableA.delete()); + mapAInitialized = false; + } + } else if (roll < 0.94) { + if (!mapAInitialized) { + await runStatements(mapTableA.init()); + mapAInitialized = true; + } + } else if (roll < 0.98) { + if (mapBInitialized) { + await runStatements(mapTableB.delete()); + mapBInitialized = false; + } + } else { + if (!mapBInitialized) { + await runStatements(mapTableB.init()); + mapBInitialized = true; + } + } + + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + const expectedMapA = mapGroups(expectedGrouped, (row) => ({ + team: row.team as string | null, + mappedValueA: (row.value as number) + 100, + })); + const expectedMapB = mapGroups(expectedGrouped, (row) => ({ + team: row.team as string | null, + mappedValueB: -1 * (row.value as number), + })); + + if (mapAInitialized) { + expect(await readBoolean(mapTableA.isInitialized())).toBe(true); + await assertTableMatches(mapTableA, expectedMapA); + } else { + expect(await readBoolean(mapTableA.isInitialized())).toBe(false); + const groups = await readRows(mapTableA.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + + if (mapBInitialized) { + expect(await readBoolean(mapTableB.isInitialized())).toBe(true); + await assertTableMatches(mapTableB, expectedMapB); + } else { + expect(await readBoolean(mapTableB.isInitialized())).toBe(false); + const groups = await readRows(mapTableB.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + } + } + }, 120_000); + + test("fuzz: parallel flatMap tables remain isolated with independent re-inits", async () => { + const identifiers = ["pf1", "pf2", "pf3", "pf 4", "pf:5"] as const; + const teams = ["alpha", "beta", null] as const; + + for (const seed of [601, 602]) { + const rng = createRng(seed); + const sourceRows = new Map(); + let flatAInitialized = true; + let flatBInitialized = true; + + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: `parallel-flat-users-${seed}` }); + const groupedTable = declareGroupByTable({ + tableId: `parallel-flat-users-by-team-${seed}`, + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const flatMapA = declareFlatMapTable({ + tableId: `parallel-flat-users-a-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + CASE + WHEN (("rowData"->>'value')::int) % 2 = 0 THEN jsonb_build_array( + jsonb_build_object( + 'team', "rowData"->'team', + 'lane', 'even', + 'metricA', (("rowData"->>'value')::int + 1000) + ) + ) + ELSE '[]'::jsonb + END AS "rows" + `), + }); + const flatMapB = declareFlatMapTable({ + tableId: `parallel-flat-users-b-${seed}`, + fromTable: groupedTable, + mapper: mapper(` + CASE + WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb + ELSE jsonb_build_array( + jsonb_build_object( + 'team', "rowData"->'team', + 'lane', 'base', + 'metricB', (("rowData"->>'value')::int) + ), + jsonb_build_object( + 'team', "rowData"->'team', + 'lane', 'triple', + 'metricB', (("rowData"->>'value')::int * 3) + ) + ) + END AS "rows" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMapA.init()); + await runStatements(flatMapB.init()); + + for (let step = 0; step < 55; step++) { + const roll = rng(); + if (roll < 0.6) { + const rowIdentifier = choose(rng, identifiers); + const rowData: SourceRow = { + team: choose(rng, teams), + value: Math.floor(rng() * 50) - 10, + }; + sourceRows.set(rowIdentifier, rowData); + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } else if (roll < 0.82) { + const rowIdentifier = choose(rng, identifiers); + sourceRows.delete(rowIdentifier); + await runStatements(fromTable.deleteRow(rowIdentifier)); + } else if (roll < 0.9) { + if (flatAInitialized) { + await runStatements(flatMapA.delete()); + flatAInitialized = false; + } + } else if (roll < 0.94) { + if (!flatAInitialized) { + await runStatements(flatMapA.init()); + flatAInitialized = true; + } + } else if (roll < 0.98) { + if (flatBInitialized) { + await runStatements(flatMapB.delete()); + flatBInitialized = false; + } + } else { + if (!flatBInitialized) { + await runStatements(flatMapB.init()); + flatBInitialized = true; + } + } + + const expectedGrouped = computeTeamGroups(sourceRows); + await assertTableMatches(groupedTable, expectedGrouped); + + const expectedFlatA = flatMapGroups(expectedGrouped, (row) => { + const value = row.value as number; + if (value % 2 !== 0) return []; + return [{ + team: row.team as string | null, + lane: "even", + metricA: value + 1000, + }]; + }); + const expectedFlatB = flatMapGroups(expectedGrouped, (row) => { + const value = row.value as number; + if (value < 0) return []; + return [ + { + team: row.team as string | null, + lane: "base", + metricB: value, + }, + { + team: row.team as string | null, + lane: "triple", + metricB: value * 3, + }, + ]; + }); + + if (flatAInitialized) { + expect(await readBoolean(flatMapA.isInitialized())).toBe(true); + await assertTableMatches(flatMapA, expectedFlatA); + } else { + expect(await readBoolean(flatMapA.isInitialized())).toBe(false); + const groups = await readRows(flatMapA.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + + if (flatBInitialized) { + expect(await readBoolean(flatMapB.isInitialized())).toBe(true); + await assertTableMatches(flatMapB, expectedFlatB); + } else { + expect(await readBoolean(flatMapB.isInitialized())).toBe(false); + const groups = await readRows(flatMapB.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + } + } + } + }, 120_000); +}); diff --git a/apps/backend/src/lib/bulldozer/db/index.perf.test.ts b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts new file mode 100644 index 0000000000..c07faeaa56 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.perf.test.ts @@ -0,0 +1,1215 @@ +import { stringCompare } from "@stackframe/stack-shared/dist/utils/strings"; +import postgres from "postgres"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLeftJoinTable, declareLFoldTable, declareLimitTable, declareMapTable, declareSortTable, declareStoredTable, declareTimeFoldTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; + +type WorkloadOperation = + | { type: "upsert", rowIdentifier: string, team: string | null, value: number } + | { type: "delete", rowIdentifier: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_perf_test"; +const DEFAULT_WARMUP_OPS = 40; +const DEFAULT_MEASURED_OPS = 200; +const IS_CI = (() => { + const env = Reflect.get(import.meta, "env"); + const ci = Reflect.get(env, "CI"); + const cursorAgent = Reflect.get(env, "CURSOR_AGENT"); + return (ci === true || ci === "true" || ci === "1") && (cursorAgent !== true && cursorAgent !== 'true' && cursorAgent !== "1"); +})(); +const CI_PERF_MAX_MS_MULTIPLIER = IS_CI ? 2 : 1; +const withCiPerfHeadroom = (maxMs: number) => maxMs * CI_PERF_MAX_MS_MULTIPLIER; +const LOAD_ROW_COUNTS = [20_000, 50_000, 200_000] as const; +const LOAD_PREFILL_MAX_MS = withCiPerfHeadroom(30_000); +const LOAD_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(5_000); +const LOAD_POINT_MUTATION_MAX_MS = withCiPerfHeadroom(400); +const LOAD_SET_ROW_AVG_ITERATIONS = 10; +const LOAD_SET_ROW_AVG_MAX_MS = withCiPerfHeadroom(50); +const LOAD_ONLINE_MUTATION_ITERATIONS = 5; +const LOAD_ONLINE_MUTATION_MAX_MS = withCiPerfHeadroom(50); +const LOAD_SUBSET_ITERATION_MAX_MS = withCiPerfHeadroom(50); +const LOAD_SUBSET_ITERATION_ROW_COUNT = 1_000; +const LOAD_SUBSET_ITERATION_MEASURED_RUNS = 5; +const LOAD_TABLE_DELETE_MAX_MS = withCiPerfHeadroom(20_000); +const LOAD_DERIVED_INIT_MAX_MS = withCiPerfHeadroom(90_000); +const LOAD_DERIVED_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(10_000); +const LOAD_EXPANDING_INIT_MAX_MS = withCiPerfHeadroom(120_000); +const LOAD_EXPANDING_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(15_000); +const LOAD_FILTERED_QUERY_MAX_MS = withCiPerfHeadroom(4_000); +const LOAD_FILTER_TABLE_INIT_MAX_MS = withCiPerfHeadroom(90_000); +const LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(8_000); +const LOAD_LIMIT_TABLE_INIT_MAX_MS = withCiPerfHeadroom(90_000); +const LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(8_000); +const LOAD_CONCAT_TABLE_INIT_MAX_MS = withCiPerfHeadroom(10_000); +const LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(8_000); +const LOAD_SORT_TABLE_INIT_MAX_MS = withCiPerfHeadroom(90_000); +const LOAD_SORT_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(8_000); +const LOAD_LFOLD_TABLE_INIT_MAX_MS = withCiPerfHeadroom(130_000); +const LOAD_LFOLD_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(12_000); +const LOAD_TIMEFOLD_TABLE_INIT_MAX_MS = withCiPerfHeadroom(130_000); +const LOAD_TIMEFOLD_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(12_000); +const LOAD_LEFT_JOIN_TABLE_INIT_MAX_MS = withCiPerfHeadroom(90_000); +const LOAD_LEFT_JOIN_TABLE_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(8_000); +const STACKED_MAP_PIPELINE_MUTATION_MAX_MS = withCiPerfHeadroom(400); +const VIRTUAL_CONCAT_COUNT_QUERY_MAX_MS = withCiPerfHeadroom(500); +const VIRTUAL_CONCAT_LOAD_ROW_COUNT = 5_000; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} + +function jsonbLiteral(value: unknown): string { + return `'${JSON.stringify(value).replaceAll("'", "''")}'::jsonb`; +} + +function createRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; +} + +function choose(rng: () => number, values: readonly T[]): T { + return values[Math.floor(rng() * values.length)] ?? values[0]; +} + +function createWorkload(seed: number, operationCount: number): WorkloadOperation[] { + const rng = createRng(seed); + const identifiers = ["u1", "u2", "u3", "u4", "u:5", "u 6", "u/7", "u'8"] as const; + const teams = ["alpha", "beta", "gamma", null] as const; + const existing = new Set(); + const operations: WorkloadOperation[] = []; + + for (let i = 0; i < operationCount; i++) { + const roll = rng(); + if (roll < 0.74) { + const rowIdentifier = choose(rng, identifiers); + const team = choose(rng, teams); + const value = Math.floor(rng() * 100); + operations.push({ type: "upsert", rowIdentifier, team, value }); + existing.add(rowIdentifier); + } else { + const rowIdentifier = existing.size > 0 + ? choose(rng, [...existing]) + : choose(rng, identifiers); + operations.push({ type: "delete", rowIdentifier }); + existing.delete(rowIdentifier); + } + } + + return operations; +} + +function logLine(message: string): void { + console.log(`${message}\n`); +} + +describe.sequential("bulldozer db performance (real postgres)", () => { + vi.setConfig({ testTimeout: 180_000 }); + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + const PERF_STATEMENT_TIMEOUT = "180s"; + + async function runStatements(statements: SqlStatement[]) { + await sql.unsafe(toExecutableSqlTransaction(statements, { statementTimeout: PERF_STATEMENT_TIMEOUT })); + } + + async function readRows(query: SqlQuery) { + return await sql.unsafe(toQueryableSqlQuery(query)); + } + + async function measureMs(label: string, fn: () => Promise): Promise<{ result: T, elapsedMs: number }> { + const startedAt = performance.now(); + const result = await fn(); + const elapsedMs = performance.now() - startedAt; + logLine(`[bulldozer-perf] ${label}: ${elapsedMs.toFixed(1)} ms`); + return { result, elapsedMs }; + } + + function summarizeMs(samplesMs: number[]): { + averageMs: number, + trimmedAverageMs: number, + medianMs: number, + varianceMs2: number, + stdDevMs: number, + minMs: number, + maxMs: number, + } { + const sortedMs = [...samplesMs].sort((a, b) => a - b); + const averageMs = samplesMs.reduce((acc, value) => acc + value, 0) / samplesMs.length; + const varianceMs2 = samplesMs.reduce((acc, value) => acc + ((value - averageMs) ** 2), 0) / samplesMs.length; + const stdDevMs = Math.sqrt(varianceMs2); + const minMs = sortedMs[0] ?? 0; + const maxMs = sortedMs[sortedMs.length - 1] ?? 0; + const midpoint = Math.floor(sortedMs.length / 2); + const medianMs = sortedMs.length % 2 === 0 + ? (((sortedMs[midpoint - 1] ?? 0) + (sortedMs[midpoint] ?? 0)) / 2) + : (sortedMs[midpoint] ?? 0); + const trimmedSamples = sortedMs.length >= 5 ? sortedMs.slice(1, -1) : sortedMs; + const trimmedAverageMs = trimmedSamples.reduce((acc, value) => acc + value, 0) / trimmedSamples.length; + return { averageMs, trimmedAverageMs, medianMs, varianceMs2, stdDevMs, minMs, maxMs }; + } + + async function prefillStoredTableInSingleStatement(tableId: string, rowCount: number): Promise { + const externalId = `external:${tableId}`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + SELECT "seedRows"."keyPath", "seedRows"."value" + FROM ( + VALUES + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text), to_jsonb('rows'::text)]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text), to_jsonb(${externalId}::text), to_jsonb('storage'::text), to_jsonb('metadata'::text)]::jsonb[], '{ "version": 1 }'::jsonb) + ) AS "seedRows"("keyPath", "value") + UNION ALL + SELECT + ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalId}::text), + to_jsonb('storage'::text), + to_jsonb('rows'::text), + to_jsonb(('seed-' || "n"::text)::text) + ]::jsonb[], + jsonb_build_object( + 'rowData', + jsonb_build_object( + 'team', + CASE + WHEN "n" % 4 = 0 THEN 'null'::jsonb + WHEN "n" % 4 = 1 THEN to_jsonb('alpha'::text) + WHEN "n" % 4 = 2 THEN to_jsonb('beta'::text) + ELSE to_jsonb('gamma'::text) + END, + 'value', + to_jsonb(("n" % 1000)::int) + ) + ) + FROM generate_series(1, ${rowCount}) AS "n" + `; + } + + async function executeWorkload( + fromTable: ReturnType>, + operations: WorkloadOperation[], + ): Promise { + for (const operation of operations) { + if (operation.type === "upsert") { + await runStatements(fromTable.setRow( + operation.rowIdentifier, + expr(jsonbLiteral({ team: operation.team, value: operation.value })), + )); + } else { + await runStatements(fromTable.deleteRow(operation.rowIdentifier)); + } + } + } + + async function benchmarkScenario(options: { + name: string, + warmupOperations: WorkloadOperation[], + measuredOperations: WorkloadOperation[], + beforeRun: () => Promise<{ fromTable: ReturnType>, validate: () => Promise }>, + }) { + const setup = await options.beforeRun(); + await executeWorkload(setup.fromTable, options.warmupOperations); + const startedAt = performance.now(); + await executeWorkload(setup.fromTable, options.measuredOperations); + const elapsedMs = performance.now() - startedAt; + await setup.validate(); + + const operationsPerSecond = options.measuredOperations.length / (elapsedMs / 1000); + logLine(`[bulldozer-perf] ${options.name}: ${operationsPerSecond.toFixed(1)} ops/s (${options.measuredOperations.length} ops in ${elapsedMs.toFixed(1)} ms)`); + return { operationsPerSecond, elapsedMs }; + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldQueue"`; + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldMetadata"`; + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; + await sql` + CREATE TABLE "BulldozerTimeFoldQueue" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "tableStoragePath" JSONB[] NOT NULL, + "groupKey" JSONB NOT NULL, + "rowIdentifier" TEXT NOT NULL, + "scheduledAt" TIMESTAMPTZ NOT NULL, + "stateAfter" JSONB NOT NULL, + "rowData" JSONB NOT NULL, + "reducerSql" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "BulldozerTimeFoldQueue_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerTimeFoldQueue_table_group_row_key" UNIQUE ("tableStoragePath", "groupKey", "rowIdentifier") + ) + `; + await sql`CREATE INDEX "BulldozerTimeFoldQueue_scheduledAt_idx" ON "BulldozerTimeFoldQueue"("scheduledAt")`; + await sql` + CREATE TABLE "BulldozerTimeFoldMetadata" ( + "key" TEXT PRIMARY KEY, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastProcessedAt" TIMESTAMPTZ NOT NULL + ) + `; + await sql` + INSERT INTO "BulldozerTimeFoldMetadata" ("key", "lastProcessedAt") + VALUES ('singleton', now()) + `; + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + it("reports ops/sec for baseline and composed example setup", async () => { + const warmupOperations = createWorkload(111, DEFAULT_WARMUP_OPS); + const measuredOperations = createWorkload(222, DEFAULT_MEASURED_OPS); + + const baseline = await benchmarkScenario({ + name: "stored-table baseline", + warmupOperations, + measuredOperations, + beforeRun: async () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-baseline-users" }); + await runStatements(fromTable.init()); + return { + fromTable, + validate: async () => { + const rows = await readRows(fromTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(Array.isArray(rows)).toBe(true); + }, + }; + }, + }); + + const composed = await benchmarkScenario({ + name: "group+map+group composed pipeline", + warmupOperations, + measuredOperations, + beforeRun: async () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-composed-users" }); + const groupedByTeam = declareGroupByTable({ + tableId: "perf-composed-users-by-team", + fromTable, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const mapped = declareMapTable({ + tableId: "perf-composed-users-mapped", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen", + ( + CASE + WHEN (("rowData"->>'value')::int + 10) >= 40 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + ` }, + }); + const groupedByBucket = declareGroupByTable({ + tableId: "perf-composed-users-by-bucket", + fromTable: mapped, + groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` }, + }); + + await runStatements(fromTable.init()); + await runStatements(groupedByTeam.init()); + await runStatements(mapped.init()); + await runStatements(groupedByBucket.init()); + + return { + fromTable, + validate: async () => { + const rows = await readRows(groupedByBucket.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(Array.isArray(rows)).toBe(true); + }, + }; + }, + }); + + const slowdownFactor = baseline.operationsPerSecond / composed.operationsPerSecond; + logLine(`[bulldozer-perf] slowdown factor (baseline/composed): ${slowdownFactor.toFixed(2)}x`); + logLine(`[bulldozer-perf] config: warmup=${DEFAULT_WARMUP_OPS}, measured=${DEFAULT_MEASURED_OPS}`); + + expect(baseline.operationsPerSecond).toBeGreaterThan(0); + expect(composed.operationsPerSecond).toBeGreaterThan(0); + }); + + it("regression: stacked group-map-group mutations avoid the postgres JIT cliff", async () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "perf-regression-users" }); + const groupedByTeam = declareGroupByTable({ + tableId: "perf-regression-users-by-team", + fromTable, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const mappedLevel1 = declareMapTable({ + tableId: "perf-regression-users-map-level-1", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 1) AS "value", + ( + CASE + WHEN ((("rowData"->>'value')::int + 1) % 2) = 0 THEN 'even' + ELSE 'odd' + END + ) AS "bucket" + ` }, + }); + const mappedLevel2 = declareMapTable({ + tableId: "perf-regression-users-map-level-2", + fromTable: mappedLevel1, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + ("rowData"->'bucket') AS "bucket", + (("rowData"->>'value')::int * 3) AS "score" + ` }, + }); + const groupedByBucket = declareGroupByTable({ + tableId: "perf-regression-users-by-bucket", + fromTable: mappedLevel2, + groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` }, + }); + + await runStatements(fromTable.init()); + await runStatements(groupedByTeam.init()); + await runStatements(mappedLevel1.init()); + await runStatements(mappedLevel2.init()); + await runStatements(groupedByBucket.init()); + + const seedRows = [ + ["u1", { team: "alpha", value: 5 }], + ["u2", { team: "beta", value: 7 }], + ["u3", { team: "gamma", value: 9 }], + ["u:4", { team: "alpha", value: 11 }], + ["u 5", { team: null, value: 13 }], + ] as const; + for (const [rowIdentifier, rowData] of seedRows) { + await runStatements(fromTable.setRow(rowIdentifier, expr(jsonbLiteral(rowData)))); + } + + await runStatements(fromTable.setRow("u1", expr(jsonbLiteral({ team: "alpha", value: 15 })))); + + const setRowMutation = await measureMs("regression stacked pipeline setRow", async () => { + await runStatements(fromTable.setRow("u2", expr(jsonbLiteral({ team: "beta", value: 19 })))); + }); + expect(setRowMutation.elapsedMs).toBeLessThan(STACKED_MAP_PIPELINE_MUTATION_MAX_MS); + + const deleteMutation = await measureMs("regression stacked pipeline deleteRow", async () => { + await runStatements(fromTable.deleteRow("u3")); + }); + expect(deleteMutation.elapsedMs).toBeLessThan(STACKED_MAP_PIPELINE_MUTATION_MAX_MS); + }); + + it("regression: virtual concat queries stay fast after metadata-only initialization", async () => { + const tableAId = "perf-concat-users-a"; + const tableBId = "perf-concat-users-b"; + const fromTableA = declareStoredTable<{ value: number, team: string | null }>({ tableId: tableAId }); + const fromTableB = declareStoredTable<{ value: number, team: string | null }>({ tableId: tableBId }); + const groupedByTeamA = declareGroupByTable({ + tableId: "perf-concat-users-a-by-team", + fromTable: fromTableA, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const groupedByTeamB = declareGroupByTable({ + tableId: "perf-concat-users-b-by-team", + fromTable: fromTableB, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const concatenatedByTeam = declareConcatTable({ + tableId: "perf-concat-users-by-team", + tables: [groupedByTeamA, groupedByTeamB], + }); + + expect((await readRows(concatenatedByTeam.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })))).toEqual([]); + + await prefillStoredTableInSingleStatement(tableAId, VIRTUAL_CONCAT_LOAD_ROW_COUNT); + await prefillStoredTableInSingleStatement(tableBId, VIRTUAL_CONCAT_LOAD_ROW_COUNT); + await runStatements(groupedByTeamA.init()); + await runStatements(groupedByTeamB.init()); + await runStatements(concatenatedByTeam.init()); + expect(await readRows(concatenatedByTeam.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).not.toEqual([]); + + const concatenatedCountQuery = concatenatedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const countRows = await measureMs("virtual concat count query", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(concatenatedCountQuery)}) AS "rows" + `); + }); + expect(countRows.elapsedMs).toBeLessThan(VIRTUAL_CONCAT_COUNT_QUERY_MAX_MS); + expect(Number(countRows.result[0].count)).toBe(VIRTUAL_CONCAT_LOAD_ROW_COUNT * 2); + }); + + it.each(LOAD_ROW_COUNTS)("load test: prefilled stored table with hundreds of thousands of rows stays functional and fast (%i rows)", async (loadRowCount) => { + const tableId = "load-prefilled-users"; + const externalTableId = `external:${tableId}`; + const table = declareStoredTable<{ value: number, team: string | null }>({ tableId }); + + const prefill = await measureMs(`load prefill (${loadRowCount} rows)`, async () => { + await prefillStoredTableInSingleStatement(tableId, loadRowCount); + }); + expect(prefill.elapsedMs).toBeLessThan(LOAD_PREFILL_MAX_MS); + + const metadataInitializedRows = await sql` + SELECT EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalTableId}::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + ) AS "initialized" + `; + expect(metadataInitializedRows[0].initialized).toBe(true); + + const listRowsQuery = table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const countRows = await measureMs("load count via listRowsInGroup", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(listRowsQuery)}) AS "rows" + `); + }); + expect(countRows.elapsedMs).toBeLessThan(LOAD_COUNT_QUERY_MAX_MS); + expect(Number(countRows.result[0].count)).toBe(loadRowCount); + + const setRowIterationTimes: number[] = []; + for (let i = 0; i < LOAD_SET_ROW_AVG_ITERATIONS; i++) { + const startedAt = performance.now(); + await runStatements(table.setRow( + `seed-${Math.floor(loadRowCount / 2) + i}`, + expr(jsonbLiteral({ team: "beta", value: 777 + i })), + )); + setRowIterationTimes.push(performance.now() - startedAt); + } + const setRowAverageMs = setRowIterationTimes.reduce((acc, value) => acc + value, 0) / setRowIterationTimes.length; + logLine(`[bulldozer-perf] load setRow average (${LOAD_SET_ROW_AVG_ITERATIONS} iterations): ${setRowAverageMs.toFixed(1)} ms`); + expect(setRowAverageMs).toBeLessThanOrEqual(LOAD_SET_ROW_AVG_MAX_MS); + const onlineInsertTimes: number[] = []; + const onlineUpdateTimes: number[] = []; + const onlineDeleteTimes: number[] = []; + for (let i = 0; i < LOAD_ONLINE_MUTATION_ITERATIONS; i++) { + const rowIdentifier = `perf-online-row-${i}`; + const insertStartedAt = performance.now(); + await runStatements(table.setRow(rowIdentifier, expr(jsonbLiteral({ team: "beta", value: 111 + i })))); + onlineInsertTimes.push(performance.now() - insertStartedAt); + const updateStartedAt = performance.now(); + await runStatements(table.setRow(rowIdentifier, expr(jsonbLiteral({ team: "beta", value: 211 + i })))); + onlineUpdateTimes.push(performance.now() - updateStartedAt); + const deleteStartedAt = performance.now(); + await runStatements(table.deleteRow(rowIdentifier)); + onlineDeleteTimes.push(performance.now() - deleteStartedAt); + } + const onlineInsertAvgMs = onlineInsertTimes.reduce((acc, value) => acc + value, 0) / onlineInsertTimes.length; + const onlineUpdateAvgMs = onlineUpdateTimes.reduce((acc, value) => acc + value, 0) / onlineUpdateTimes.length; + const onlineDeleteAvgMs = onlineDeleteTimes.reduce((acc, value) => acc + value, 0) / onlineDeleteTimes.length; + logLine(`[bulldozer-perf] load online setRow insert average (${LOAD_ONLINE_MUTATION_ITERATIONS} iterations): ${onlineInsertAvgMs.toFixed(1)} ms`); + logLine(`[bulldozer-perf] load online setRow update average (${LOAD_ONLINE_MUTATION_ITERATIONS} iterations): ${onlineUpdateAvgMs.toFixed(1)} ms`); + logLine(`[bulldozer-perf] load online deleteRow average (${LOAD_ONLINE_MUTATION_ITERATIONS} iterations): ${onlineDeleteAvgMs.toFixed(1)} ms`); + expect(onlineInsertAvgMs).toBeLessThanOrEqual(LOAD_ONLINE_MUTATION_MAX_MS); + expect(onlineUpdateAvgMs).toBeLessThanOrEqual(LOAD_ONLINE_MUTATION_MAX_MS); + expect(onlineDeleteAvgMs).toBeLessThanOrEqual(LOAD_ONLINE_MUTATION_MAX_MS); + + const pointDelete = await measureMs("load point delete (deleteRow existing)", async () => { + await runStatements(table.deleteRow(`seed-${Math.floor(loadRowCount / 2) - 1}`)); + }); + expect(pointDelete.elapsedMs).toBeLessThan(LOAD_POINT_MUTATION_MAX_MS); + + const countAfterDelete = await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(listRowsQuery)}) AS "rows" + `); + expect(Number(countAfterDelete[0].count)).toBe(loadRowCount - 1); + + const groupedByTeam = declareGroupByTable({ + tableId: "load-prefilled-users-by-team", + fromTable: table, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const leftJoinRulesTable = declareStoredTable<{ team: string | null, threshold: number, label: string }>({ + tableId: "load-prefilled-users-left-join-rules", + }); + const leftJoinRulesByTeam = declareGroupByTable({ + tableId: "load-prefilled-users-left-join-rules-by-team", + fromTable: leftJoinRulesTable, + groupBy: { type: "mapper", sql: `"rowData"->'team' AS "groupKey"` }, + }); + const mappedByTeam = declareMapTable({ + tableId: "load-prefilled-users-mapped", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen", + ( + CASE + WHEN (("rowData"->>'value')::int + 10) >= 700 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + ` }, + }); + const mappedTwice = declareMapTable({ + tableId: "load-prefilled-users-mapped-twice", + fromTable: mappedByTeam, + mapper: { type: "mapper", sql: ` + ("rowData"->'team') AS "team", + ("rowData"->'bucket') AS "bucket", + ((("rowData"->>'valuePlusTen')::int * 2)) AS "valueScaled" + ` }, + }); + const groupedByBucket = declareGroupByTable({ + tableId: "load-prefilled-users-by-bucket", + fromTable: mappedTwice, + groupBy: { type: "mapper", sql: `"rowData"->'bucket' AS "groupKey"` }, + }); + const filteredHighValue = declareFilterTable({ + tableId: "load-prefilled-users-high-value", + fromTable: groupedByTeam, + filter: { type: "predicate", sql: `( ("rowData"->>'value')::int ) >= 700` }, + }); + const concatenatedByTeam = declareConcatTable({ + tableId: "load-prefilled-users-concat", + tables: [groupedByTeam, filteredHighValue], + }); + const limitedByTeam = declareLimitTable({ + tableId: "load-prefilled-users-top-team-rows", + fromTable: groupedByTeam, + limit: expr(`25`), + }); + const leftJoinedTopByTeam = declareLeftJoinTable({ + tableId: "load-prefilled-users-left-join-top-team-rows", + leftTable: limitedByTeam, + rightTable: leftJoinRulesByTeam, + leftJoinKey: { type: "mapper", sql: `(("rowData"->>'value')::int) AS "joinKey"` }, + rightJoinKey: { type: "mapper", sql: `(("rowData"->>'threshold')::int) AS "joinKey"` }, + }); + const expandedByTeam = declareFlatMapTable({ + tableId: "load-prefilled-users-expanded", + fromTable: groupedByTeam, + mapper: { type: "mapper", sql: ` + jsonb_build_array( + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'base', + 'mappedValue', (("rowData"->>'value')::int + 10) + ), + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'double', + 'mappedValue', (("rowData"->>'value')::int * 2) + ) + ) AS "rows" + ` }, + }); + + await runStatements(leftJoinRulesTable.init()); + await runStatements(leftJoinRulesTable.setRow("rule-alpha", expr(jsonbLiteral({ team: "alpha", threshold: 0, label: "alpha-rule" })))); + await runStatements(leftJoinRulesTable.setRow("rule-beta", expr(jsonbLiteral({ team: "beta", threshold: 0, label: "beta-rule" })))); + await runStatements(leftJoinRulesTable.setRow("rule-gamma", expr(jsonbLiteral({ team: "gamma", threshold: 0, label: "gamma-rule" })))); + await runStatements(leftJoinRulesTable.setRow("rule-null", expr(jsonbLiteral({ team: null, threshold: 0, label: "null-rule" })))); + const leftJoinRulesInit = await measureMs("load init leftJoinRulesByTeam", async () => { + await runStatements(leftJoinRulesByTeam.init()); + }); + expect(leftJoinRulesInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + + const groupInit = await measureMs("load init groupedByTeam", async () => { + await runStatements(groupedByTeam.init()); + }); + expect(groupInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const mapInit = await measureMs("load init mappedByTeam", async () => { + await runStatements(mappedByTeam.init()); + }); + expect(mapInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const mapTwiceInit = await measureMs("load init mappedTwice", async () => { + await runStatements(mappedTwice.init()); + }); + expect(mapTwiceInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const bucketInit = await measureMs("load init groupedByBucket", async () => { + await runStatements(groupedByBucket.init()); + }); + expect(bucketInit.elapsedMs).toBeLessThan(LOAD_DERIVED_INIT_MAX_MS); + const filterInit = await measureMs("load init filteredHighValue", async () => { + await runStatements(filteredHighValue.init()); + }); + expect(filterInit.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_INIT_MAX_MS); + const concatInit = await measureMs("load init concatenatedByTeam", async () => { + await runStatements(concatenatedByTeam.init()); + }); + expect(concatInit.elapsedMs).toBeLessThan(LOAD_CONCAT_TABLE_INIT_MAX_MS); + const limitInit = await measureMs("load init limitedByTeam", async () => { + await runStatements(limitedByTeam.init()); + }); + expect(limitInit.elapsedMs).toBeLessThan(LOAD_LIMIT_TABLE_INIT_MAX_MS); + const expandInit = await measureMs("load init expandedByTeam", async () => { + await runStatements(expandedByTeam.init()); + }); + expect(expandInit.elapsedMs).toBeLessThan(LOAD_EXPANDING_INIT_MAX_MS); + + const groupedCountQuery = groupedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const mappedCountQuery = mappedTwice.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const bucketCountQuery = groupedByBucket.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const expandedCountQuery = expandedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const filteredHighValueCountQuery = filteredHighValue.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const concatenatedByTeamCountQuery = concatenatedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const limitedByTeamCountQuery = limitedByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const derivedCounts = await measureMs("load count derived tables", async () => { + return await Promise.all([ + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(groupedCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(mappedCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(bucketCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(concatenatedByTeamCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(limitedByTeamCountQuery)}) AS "rows"`), + sql.unsafe(`SELECT COUNT(*)::int AS "count" FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows"`), + ]); + }); + expect(derivedCounts.elapsedMs).toBeLessThan(LOAD_DERIVED_COUNT_QUERY_MAX_MS); + expect(Number(derivedCounts.result[0][0].count)).toBe(loadRowCount - 1); + expect(Number(derivedCounts.result[1][0].count)).toBe(loadRowCount - 1); + expect(Number(derivedCounts.result[2][0].count)).toBe(loadRowCount - 1); + expect(Number(derivedCounts.result[3][0].count)).toBeGreaterThan(0); + expect(Number(derivedCounts.result[3][0].count)).toBeLessThan(loadRowCount); + expect(Number(derivedCounts.result[4][0].count)).toBeGreaterThan(loadRowCount - 1); + expect(Number(derivedCounts.result[4][0].count)).toBeLessThan((loadRowCount - 1) * 2); + expect(Number(derivedCounts.result[5][0].count)).toBeGreaterThan(0); + expect(Number(derivedCounts.result[5][0].count)).toBeLessThanOrEqual(100); + expect(Number(derivedCounts.result[6][0].count)).toBe((loadRowCount - 1) * 2); + + const filteredHighValueCountOnly = await measureMs("load count filteredHighValue table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(filteredHighValueCountQuery)}) AS "rows" + `); + }); + expect(filteredHighValueCountOnly.elapsedMs).toBeLessThan(LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS); + expect(Number(filteredHighValueCountOnly.result[0].count)).toBeGreaterThan(0); + + const concatenatedByTeamCountOnly = await measureMs("load count concatenatedByTeam table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(concatenatedByTeamCountQuery)}) AS "rows" + `); + }); + expect(concatenatedByTeamCountOnly.elapsedMs).toBeLessThan(LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS); + expect(Number(concatenatedByTeamCountOnly.result[0].count)).toBeGreaterThan(loadRowCount - 1); + expect(Number(concatenatedByTeamCountOnly.result[0].count)).toBeLessThan((loadRowCount - 1) * 2); + + const limitedByTeamCountOnly = await measureMs("load count limitedByTeam table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(limitedByTeamCountQuery)}) AS "rows" + `); + }); + expect(limitedByTeamCountOnly.elapsedMs).toBeLessThan(LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS); + expect(Number(limitedByTeamCountOnly.result[0].count)).toBeGreaterThan(0); + expect(Number(limitedByTeamCountOnly.result[0].count)).toBeLessThanOrEqual(100); + + const expandedCountOnly = await measureMs("load count expanded table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(expandedCountQuery)}) AS "rows" + `); + }); + expect(expandedCountOnly.elapsedMs).toBeLessThan(LOAD_EXPANDING_COUNT_QUERY_MAX_MS); + expect(Number(expandedCountOnly.result[0].count)).toBe((loadRowCount - 1) * 2); + + const filteredExpandedBetaBase = await measureMs("load filtered expanded query (team=beta, kind=base)", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM ( + ${toQueryableSqlQuery(expandedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))} + ) AS "rows" + WHERE "rows"."rowdata"->>'kind' = 'base' + `); + }); + expect(filteredExpandedBetaBase.elapsedMs).toBeLessThan(LOAD_FILTERED_QUERY_MAX_MS); + expect(Number(filteredExpandedBetaBase.result[0].count)).toBeGreaterThan(0); + + await runStatements(table.setRow( + "seed-100000", + expr(jsonbLiteral({ team: "delta", value: 999 })), + )); + const deltaGroupedRows = await readRows(groupedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(deltaGroupedRows.some((row) => row.rowidentifier === "seed-100000")).toBe(true); + const highBucketRows = await readRows(groupedByBucket.listRowsInGroup({ + groupKey: expr(`to_jsonb('high'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const highBucketRow = highBucketRows.find((row) => row.rowidentifier === "seed-100000:1:1"); + expect(highBucketRow).toBeDefined(); + expect(highBucketRow?.rowdata).toEqual({ + team: "delta", + bucket: "high", + valueScaled: 2018, + }); + const expandedDeltaRows = await readRows(expandedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(expandedDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(String(a.rowIdentifier), String(b.rowIdentifier)))).toEqual([ + { rowIdentifier: "seed-100000:1", rowData: { team: "delta", kind: "base", mappedValue: 1009 } }, + { rowIdentifier: "seed-100000:2", rowData: { team: "delta", kind: "double", mappedValue: 1998 } }, + ]); + const filteredDeltaRows = await readRows(filteredHighValue.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(filteredDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "seed-100000:1", rowData: { team: "delta", value: 999 } }, + ]); + const groupedSubsetSql = ` + SELECT * + FROM (${toQueryableSqlQuery(groupedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))}) AS "rows" + LIMIT ${LOAD_SUBSET_ITERATION_ROW_COUNT} + `; + // Warm once so we measure steady-state subset iteration instead of first-touch planner/cache cost. + await sql.unsafe(groupedSubsetSql); + const groupedSubsetSamplesMs: number[] = []; + for (let runIndex = 0; runIndex < LOAD_SUBSET_ITERATION_MEASURED_RUNS; runIndex++) { + const groupedSubsetRun = await measureMs(`load iterate groupedByTeam subset from start (${LOAD_SUBSET_ITERATION_ROW_COUNT} rows) run ${runIndex + 1}/${LOAD_SUBSET_ITERATION_MEASURED_RUNS}`, async () => { + return await sql.unsafe(groupedSubsetSql); + }); + groupedSubsetSamplesMs.push(groupedSubsetRun.elapsedMs); + expect(groupedSubsetRun.result).toHaveLength(LOAD_SUBSET_ITERATION_ROW_COUNT); + } + const groupedSubsetStats = summarizeMs(groupedSubsetSamplesMs); + logLine( + `[bulldozer-perf] load iterate groupedByTeam subset stats (${LOAD_SUBSET_ITERATION_MEASURED_RUNS} runs): ` + + `avg=${groupedSubsetStats.averageMs.toFixed(1)} ms, ` + + `trimmedAvg=${groupedSubsetStats.trimmedAverageMs.toFixed(1)} ms, ` + + `median=${groupedSubsetStats.medianMs.toFixed(1)} ms, ` + + `stddev=${groupedSubsetStats.stdDevMs.toFixed(1)} ms, ` + + `variance=${groupedSubsetStats.varianceMs2.toFixed(1)} ms^2, ` + + `min=${groupedSubsetStats.minMs.toFixed(1)} ms, ` + + `max=${groupedSubsetStats.maxMs.toFixed(1)} ms` + ); + expect(groupedSubsetStats.trimmedAverageMs).toBeLessThanOrEqual(LOAD_SUBSET_ITERATION_MAX_MS); + const sortedHighValueByTeam = declareSortTable({ + tableId: "load-prefilled-users-high-value-sorted", + fromTable: filteredHighValue, + getSortKey: { type: "mapper", sql: `( ("rowData"->>'value')::int ) AS "newSortKey"` }, + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + const foldedHighValueByTeam = declareLFoldTable({ + tableId: "load-prefilled-users-high-value-folded", + fromTable: sortedHighValueByTeam, + initialState: expr(`'0'::jsonb`), + reducer: { type: "mapper", sql: ` + ( + COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int) + ) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'team', "oldRowData"->'team', + 'value', (("oldRowData"->>'value')::int), + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int) + ) + ) AS "newRowsData" + ` }, + }); + const timedExposureByTeam = declareTimeFoldTable({ + tableId: "load-prefilled-users-timefold", + fromTable: groupedByTeam, + initialState: expr(`'0'::jsonb`), + reducer: { type: "mapper", sql: ` + (("oldRowData"->>'value')::int) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'team', "oldRowData"->'team', + 'value', (("oldRowData"->>'value')::int), + 'timestamp', + CASE + WHEN "timestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("timestamp") + END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (now() + interval '15 minutes') + ELSE NULL::timestamptz + END AS "nextTimestamp" + ` }, + }); + const sortInit = await measureMs("load init sortedHighValueByTeam", async () => { + await runStatements(sortedHighValueByTeam.init()); + }); + expect(sortInit.elapsedMs).toBeLessThan(LOAD_SORT_TABLE_INIT_MAX_MS); + const approxRowsPerValuePerTeam = Math.max(1, Math.floor(loadRowCount / 4 / 1000)); + const sortedSubsetRequiredSortKeySpan = Math.ceil(LOAD_SUBSET_ITERATION_ROW_COUNT / approxRowsPerValuePerTeam); + const sortedSubsetFromStartMaxSortKey = Math.min(999, 699 + sortedSubsetRequiredSortKeySpan); + const sortedSubsetFromCursorMinSortKey = Math.max(700, 1000 - sortedSubsetRequiredSortKeySpan); + const sortedSubsetFromStartSql = ` + SELECT * + FROM (${toQueryableSqlQuery(sortedHighValueByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: expr(`to_jsonb(${sortedSubsetFromStartMaxSortKey}::int)`), + startInclusive: true, + endInclusive: true, + }))}) AS "rows" + LIMIT ${LOAD_SUBSET_ITERATION_ROW_COUNT} + `; + await sql.unsafe(sortedSubsetFromStartSql); + const sortedSubsetFromStartSamplesMs: number[] = []; + for (let runIndex = 0; runIndex < LOAD_SUBSET_ITERATION_MEASURED_RUNS; runIndex++) { + const sortedSubsetFromStartRun = await measureMs(`load iterate sortedHighValueByTeam subset from start (${LOAD_SUBSET_ITERATION_ROW_COUNT} rows) run ${runIndex + 1}/${LOAD_SUBSET_ITERATION_MEASURED_RUNS}`, async () => { + return await sql.unsafe(sortedSubsetFromStartSql); + }); + sortedSubsetFromStartSamplesMs.push(sortedSubsetFromStartRun.elapsedMs); + expect(sortedSubsetFromStartRun.result).toHaveLength(LOAD_SUBSET_ITERATION_ROW_COUNT); + } + const sortedSubsetFromStartStats = summarizeMs(sortedSubsetFromStartSamplesMs); + logLine( + `[bulldozer-perf] load iterate sortedHighValueByTeam subset from start stats (${LOAD_SUBSET_ITERATION_MEASURED_RUNS} runs): ` + + `avg=${sortedSubsetFromStartStats.averageMs.toFixed(1)} ms, ` + + `trimmedAvg=${sortedSubsetFromStartStats.trimmedAverageMs.toFixed(1)} ms, ` + + `median=${sortedSubsetFromStartStats.medianMs.toFixed(1)} ms, ` + + `stddev=${sortedSubsetFromStartStats.stdDevMs.toFixed(1)} ms, ` + + `variance=${sortedSubsetFromStartStats.varianceMs2.toFixed(1)} ms^2, ` + + `min=${sortedSubsetFromStartStats.minMs.toFixed(1)} ms, ` + + `max=${sortedSubsetFromStartStats.maxMs.toFixed(1)} ms` + ); + expect(sortedSubsetFromStartStats.trimmedAverageMs).toBeLessThanOrEqual(LOAD_SUBSET_ITERATION_MAX_MS); + const sortedSubsetFromSortKeySql = ` + SELECT * + FROM (${toQueryableSqlQuery(sortedHighValueByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: expr(`to_jsonb(${sortedSubsetFromCursorMinSortKey}::int)`), + end: expr(`to_jsonb(999::int)`), + startInclusive: true, + endInclusive: true, + }))}) AS "rows" + LIMIT ${LOAD_SUBSET_ITERATION_ROW_COUNT} + `; + await sql.unsafe(sortedSubsetFromSortKeySql); + const sortedSubsetFromSortKeySamplesMs: number[] = []; + for (let runIndex = 0; runIndex < LOAD_SUBSET_ITERATION_MEASURED_RUNS; runIndex++) { + const sortedSubsetFromSortKeyRun = await measureMs(`load iterate sortedHighValueByTeam subset from sort-key cursor (${LOAD_SUBSET_ITERATION_ROW_COUNT} rows) run ${runIndex + 1}/${LOAD_SUBSET_ITERATION_MEASURED_RUNS}`, async () => { + return await sql.unsafe(sortedSubsetFromSortKeySql); + }); + sortedSubsetFromSortKeySamplesMs.push(sortedSubsetFromSortKeyRun.elapsedMs); + expect(sortedSubsetFromSortKeyRun.result).toHaveLength(LOAD_SUBSET_ITERATION_ROW_COUNT); + } + const sortedSubsetFromSortKeyStats = summarizeMs(sortedSubsetFromSortKeySamplesMs); + logLine( + `[bulldozer-perf] load iterate sortedHighValueByTeam subset from sort-key cursor stats (${LOAD_SUBSET_ITERATION_MEASURED_RUNS} runs): ` + + `avg=${sortedSubsetFromSortKeyStats.averageMs.toFixed(1)} ms, ` + + `trimmedAvg=${sortedSubsetFromSortKeyStats.trimmedAverageMs.toFixed(1)} ms, ` + + `median=${sortedSubsetFromSortKeyStats.medianMs.toFixed(1)} ms, ` + + `stddev=${sortedSubsetFromSortKeyStats.stdDevMs.toFixed(1)} ms, ` + + `variance=${sortedSubsetFromSortKeyStats.varianceMs2.toFixed(1)} ms^2, ` + + `min=${sortedSubsetFromSortKeyStats.minMs.toFixed(1)} ms, ` + + `max=${sortedSubsetFromSortKeyStats.maxMs.toFixed(1)} ms` + ); + expect(sortedSubsetFromSortKeyStats.trimmedAverageMs).toBeLessThanOrEqual(LOAD_SUBSET_ITERATION_MAX_MS); + const lFoldInit = await measureMs("load init foldedHighValueByTeam", async () => { + await runStatements(foldedHighValueByTeam.init()); + }); + expect(lFoldInit.elapsedMs).toBeLessThan(LOAD_LFOLD_TABLE_INIT_MAX_MS); + const timeFoldInit = await measureMs("load init timedExposureByTeam", async () => { + await runStatements(timedExposureByTeam.init()); + }); + expect(timeFoldInit.elapsedMs).toBeLessThan(LOAD_TIMEFOLD_TABLE_INIT_MAX_MS); + const sortedDeltaRows = await readRows(sortedHighValueByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(sortedDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowSortKey: row.rowsortkey, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "seed-100000:1", rowSortKey: 999, rowData: { team: "delta", value: 999 } }, + ]); + const foldedDeltaRows = await readRows(foldedHighValueByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(foldedDeltaRows).toHaveLength(1); + expect(foldedDeltaRows[0].rowidentifier).toBe("seed-100000:1:1"); + expect(foldedDeltaRows[0].rowdata).toEqual({ + team: "delta", + value: 999, + runningTotal: 999, + }); + const timedExposureDeltaRows = await readRows(timedExposureByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(timedExposureDeltaRows).toHaveLength(1); + expect(timedExposureDeltaRows[0].rowidentifier).toBe("seed-100000:1"); + expect(timedExposureDeltaRows[0].rowdata).toEqual({ + team: "delta", + value: 999, + timestamp: null, + }); + const foldedHighValueCountOnly = await measureMs("load count foldedHighValueByTeam table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(foldedHighValueByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))}) AS "rows" + `); + }); + expect(foldedHighValueCountOnly.elapsedMs).toBeLessThan(LOAD_LFOLD_TABLE_COUNT_QUERY_MAX_MS); + expect(Number(foldedHighValueCountOnly.result[0].count)).toBeGreaterThan(0); + expect(Number(foldedHighValueCountOnly.result[0].count)).toBeLessThanOrEqual(Number(filteredHighValueCountOnly.result[0].count) + 1); + const timedExposureCountOnly = await measureMs("load count timedExposureByTeam table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(timedExposureByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))}) AS "rows" + `); + }); + expect(timedExposureCountOnly.elapsedMs).toBeLessThan(LOAD_TIMEFOLD_TABLE_COUNT_QUERY_MAX_MS); + const expectedTimedExposureCount = loadRowCount >= 100_000 + ? (loadRowCount - 1) + : loadRowCount; + expect(Number(timedExposureCountOnly.result[0].count)).toBe(expectedTimedExposureCount); + const concatenatedDeltaRows = await readRows(concatenatedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(concatenatedDeltaRows + .map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))) + .toEqual([ + { rowIdentifier: "0:seed-100000", rowData: { team: "delta", value: 999 } }, + { rowIdentifier: "1:seed-100000:1", rowData: { team: "delta", value: 999 } }, + ]); + const limitedDeltaRows = await readRows(limitedByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(limitedDeltaRows).toHaveLength(1); + expect(limitedDeltaRows[0].rowidentifier).toBe("seed-100000"); + const leftJoinInit = await measureMs("load init leftJoinedTopByTeam", async () => { + await runStatements(leftJoinedTopByTeam.init()); + }); + expect(leftJoinInit.elapsedMs).toBeLessThan(LOAD_LEFT_JOIN_TABLE_INIT_MAX_MS); + const leftJoinedTopByTeamCountQuery = leftJoinedTopByTeam.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }); + const leftJoinedTopByTeamCountOnly = await measureMs("load count leftJoinedTopByTeam table only", async () => { + return await sql.unsafe(` + SELECT COUNT(*)::int AS "count" + FROM (${toQueryableSqlQuery(leftJoinedTopByTeamCountQuery)}) AS "rows" + `); + }); + expect(leftJoinedTopByTeamCountOnly.elapsedMs).toBeLessThan(LOAD_LEFT_JOIN_TABLE_COUNT_QUERY_MAX_MS); + expect(Number(leftJoinedTopByTeamCountOnly.result[0].count)).toBe(Number(limitedByTeamCountOnly.result[0].count) + 1); + const leftJoinedDeltaRows = await readRows(leftJoinedTopByTeam.listRowsInGroup({ + groupKey: expr(`to_jsonb('delta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(leftJoinedDeltaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { + rowIdentifier: `["seed-100000", null]`, + rowData: { + leftRowData: { team: "delta", value: 999 }, + rightRowData: null, + }, + }, + ]); + const bulkDelete = await measureMs("load full table delete", async () => { + await runStatements(table.delete()); + }); + expect(bulkDelete.elapsedMs).toBeLessThan(LOAD_TABLE_DELETE_MAX_MS); + + const isInitializedRows = await sql` + SELECT EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb(${externalTableId}::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + ) AS "initialized" + `; + expect(isInitializedRows[0].initialized).toBe(false); + + logLine(`[bulldozer-perf] load thresholds(ms): prefill<=${LOAD_PREFILL_MAX_MS}, baseCount<=${LOAD_COUNT_QUERY_MAX_MS}, setRowAvg<=${LOAD_SET_ROW_AVG_MAX_MS} over ${LOAD_SET_ROW_AVG_ITERATIONS}, pointDelete<=${LOAD_POINT_MUTATION_MAX_MS}, onlineMutationAvg<=${LOAD_ONLINE_MUTATION_MAX_MS} over ${LOAD_ONLINE_MUTATION_ITERATIONS}, groupedSubsetTrimmedAvg<=${LOAD_SUBSET_ITERATION_MAX_MS} for ${LOAD_SUBSET_ITERATION_ROW_COUNT} rows over ${LOAD_SUBSET_ITERATION_MEASURED_RUNS} runs, derivedInit<=${LOAD_DERIVED_INIT_MAX_MS}, filterInit<=${LOAD_FILTER_TABLE_INIT_MAX_MS}, sortInit<=${LOAD_SORT_TABLE_INIT_MAX_MS}, lfoldInit<=${LOAD_LFOLD_TABLE_INIT_MAX_MS}, timefoldInit<=${LOAD_TIMEFOLD_TABLE_INIT_MAX_MS}, leftJoinInit<=${LOAD_LEFT_JOIN_TABLE_INIT_MAX_MS}, concatInit<=${LOAD_CONCAT_TABLE_INIT_MAX_MS}, limitInit<=${LOAD_LIMIT_TABLE_INIT_MAX_MS}, expandingInit<=${LOAD_EXPANDING_INIT_MAX_MS}, derivedCount<=${LOAD_DERIVED_COUNT_QUERY_MAX_MS}, filterCount<=${LOAD_FILTER_TABLE_COUNT_QUERY_MAX_MS}, lfoldCount<=${LOAD_LFOLD_TABLE_COUNT_QUERY_MAX_MS}, timefoldCount<=${LOAD_TIMEFOLD_TABLE_COUNT_QUERY_MAX_MS}, leftJoinCount<=${LOAD_LEFT_JOIN_TABLE_COUNT_QUERY_MAX_MS}, concatCount<=${LOAD_CONCAT_TABLE_COUNT_QUERY_MAX_MS}, limitCount<=${LOAD_LIMIT_TABLE_COUNT_QUERY_MAX_MS}, expandingCount<=${LOAD_EXPANDING_COUNT_QUERY_MAX_MS}, filteredQuery<=${LOAD_FILTERED_QUERY_MAX_MS}, tableDelete<=${LOAD_TABLE_DELETE_MAX_MS}`); + }, 300_000); +}); + diff --git a/apps/backend/src/lib/bulldozer/db/index.test.ts b/apps/backend/src/lib/bulldozer/db/index.test.ts new file mode 100644 index 0000000000..1a0dec2d42 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.test.ts @@ -0,0 +1,4236 @@ +import { stringCompare, templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; +import postgres from "postgres"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, test } from "vitest"; +import { declareConcatTable, declareFilterTable, declareFlatMapTable, declareGroupByTable, declareLeftJoinTable, declareLFoldTable, declareLimitTable, declareMapTable, declareSortTable, declareStoredTable, declareTimeFoldTable, toExecutableSqlTransaction, toQueryableSqlQuery } from "./index"; + +type TestDb = { full: string, base: string }; + +const TEST_DB_PREFIX = "stack_bulldozer_db_test"; + +function getTestDbUrls(): TestDb { + const env = Reflect.get(import.meta, "env"); + const connectionString = Reflect.get(env, "STACK_DATABASE_CONNECTION_STRING"); + if (typeof connectionString !== "string" || connectionString.length === 0) { + throw new Error("Missing STACK_DATABASE_CONNECTION_STRING"); + } + const base = connectionString.replace(/\/[^/]*(\?.*)?$/, ""); + const query = connectionString.split("?")[1] ?? ""; + const dbName = `${TEST_DB_PREFIX}_${Math.random().toString(16).slice(2, 12)}`; + return { + full: query.length === 0 ? `${base}/${dbName}` : `${base}/${dbName}?${query}`, + base, + }; +} + +type SqlExpression = { type: "expression", sql: string }; +type SqlStatement = { type: "statement", sql: string, outputName?: string }; +type SqlQuery = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; +type SqlMapper = { type: "mapper", sql: string }; +type SqlPredicate = { type: "predicate", sql: string }; + +function expr(sql: string): SqlExpression { + return { type: "expression", sql }; +} +function mapper(sql: string): SqlMapper { + return { type: "mapper", sql }; +} +function predicate(sql: string): SqlPredicate { + return { type: "predicate", sql }; +} + +const sqlStringLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`; +const sqlStatement = (strings: TemplateStringsArray, ...values: { sql: string }[]): SqlStatement => ({ + type: "statement", + sql: templateIdentity(strings, ...values.map((value) => value.sql)), +}); + +describe.sequential("declareStoredTable (real postgres)", () => { + const dbUrls = getTestDbUrls(); + const dbName = dbUrls.full.replace(/^.*\//, "").replace(/\?.*$/, ""); + const adminSql = postgres(dbUrls.base, { onnotice: () => undefined }); + const sql = postgres(dbUrls.full, { onnotice: () => undefined, max: 1 }); + + async function runStatements(statements: SqlStatement[]) { + await sql.unsafe(toExecutableSqlTransaction(statements)); + } + + async function readBoolean(expression: SqlExpression) { + const rows = await sql.unsafe(`SELECT (${expression.sql}) AS "value"`); + return rows[0].value === true; + } + + async function readRows(query: SqlQuery) { + return await sql.unsafe(toQueryableSqlQuery(query)); + } + + async function readTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerTriggerAudit" + ORDER BY "id" + `); + } + async function readGroupTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "groupKey"#>>'{}' AS "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerGroupTriggerAudit" + ORDER BY "id" + `); + } + async function readMapTriggerAuditRows() { + return await sql.unsafe(` + SELECT + "event", + "groupKey"#>>'{}' AS "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM "BulldozerMapTriggerAudit" + ORDER BY "id" + `); + } + async function readTimeFoldQueueRows() { + const queueRowsRaw = await sql>>` + SELECT + "rowIdentifier", + "groupKey"#>>'{}' AS "groupKey", + ("stateAfter"#>>'{}')::int AS "stateAfter", + "rowData" + FROM "BulldozerTimeFoldQueue" + ORDER BY "rowIdentifier" ASC, "groupKey"#>>'{}' ASC NULLS FIRST + `; + return queueRowsRaw.map((row) => ({ + rowIdentifier: (() => { + const raw = Reflect.get(row, "rowIdentifier") ?? Reflect.get(row, "rowidentifier"); + if (typeof raw !== "string") throw new Error("expected string rowIdentifier"); + return raw; + })(), + groupKey: (() => { + const raw = Reflect.get(row, "groupKey") ?? Reflect.get(row, "groupkey"); + if (raw === undefined) return null; + if (raw === null || typeof raw === "string") return raw; + throw new Error("expected nullable string groupKey"); + })(), + stateAfter: (() => { + const raw = Reflect.get(row, "stateAfter") ?? Reflect.get(row, "stateafter"); + if (typeof raw !== "number") throw new Error("expected numeric stateAfter"); + return raw; + })(), + rowData: (() => { + const raw = Reflect.get(row, "rowData") ?? Reflect.get(row, "rowdata"); + if (raw == null || typeof raw !== "object") throw new Error("expected object rowData"); + return raw; + })(), + })); + } + + beforeAll(async () => { + await adminSql.unsafe(`CREATE DATABASE ${dbName}`); + }); + + beforeEach(async () => { + await sql`CREATE EXTENSION IF NOT EXISTS pgcrypto`; + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldQueue"`; + await sql`DROP TABLE IF EXISTS "BulldozerTimeFoldMetadata"`; + await sql`DROP TABLE IF EXISTS "BulldozerMapTriggerAudit"`; + await sql`DROP TABLE IF EXISTS "BulldozerGroupTriggerAudit"`; + await sql`DROP TABLE IF EXISTS "BulldozerTriggerAudit"`; + await sql`DROP TABLE IF EXISTS "BulldozerStorageEngine"`; + await sql` + CREATE TABLE "BulldozerStorageEngine" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "keyPath" JSONB[] NOT NULL, + "keyPathParent" JSONB[] GENERATED ALWAYS AS ( + CASE + WHEN cardinality("keyPath") = 0 THEN NULL + ELSE "keyPath"[1:cardinality("keyPath") - 1] + END + ) STORED, + "value" JSONB NOT NULL, + CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"), + CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey" + FOREIGN KEY ("keyPathParent") + REFERENCES "BulldozerStorageEngine"("keyPath") + ON DELETE CASCADE + ) + `; + await sql`CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent")`; + await sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES + (ARRAY[]::jsonb[], 'null'::jsonb), + (ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb) + `; + await sql` + CREATE TABLE "BulldozerTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; + await sql` + CREATE TABLE "BulldozerGroupTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "groupKey" JSONB, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; + await sql` + CREATE TABLE "BulldozerMapTriggerAudit" ( + "id" SERIAL PRIMARY KEY, + "event" TEXT NOT NULL, + "groupKey" JSONB, + "rowIdentifier" TEXT, + "oldRowData" JSONB, + "newRowData" JSONB + ) + `; + await sql` + CREATE TABLE "BulldozerTimeFoldQueue" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "tableStoragePath" JSONB[] NOT NULL, + "groupKey" JSONB NOT NULL, + "rowIdentifier" TEXT NOT NULL, + "scheduledAt" TIMESTAMPTZ NOT NULL, + "stateAfter" JSONB NOT NULL, + "rowData" JSONB NOT NULL, + "reducerSql" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "BulldozerTimeFoldQueue_pkey" PRIMARY KEY ("id"), + CONSTRAINT "BulldozerTimeFoldQueue_table_group_row_key" UNIQUE ("tableStoragePath", "groupKey", "rowIdentifier") + ) + `; + await sql` + CREATE INDEX "BulldozerTimeFoldQueue_scheduledAt_idx" + ON "BulldozerTimeFoldQueue"("scheduledAt") + `; + await sql` + CREATE TABLE "BulldozerTimeFoldMetadata" ( + "key" TEXT PRIMARY KEY, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastProcessedAt" TIMESTAMPTZ NOT NULL + ) + `; + await sql` + INSERT INTO "BulldozerTimeFoldMetadata" ("key", "lastProcessedAt") + VALUES ('singleton', now()) + `; + }); + + afterAll(async () => { + await sql.end(); + await adminSql.unsafe(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '${dbName}' + AND pid <> pg_backend_pid() + `); + await adminSql.unsafe(`DROP DATABASE IF EXISTS ${dbName}`); + await adminSql.end(); + }); + + function registerAuditTrigger( + table: ReturnType>, + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerTriggerAudit" ( + "event", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function createGroupedTable() { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users" }); + const groupedTable = declareGroupByTable({ + tableId: "users-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + return { fromTable, groupedTable }; + } + function createMappedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const mappedTable = declareMapTable({ + tableId: "users-by-team-mapped", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValue" + `), + }); + return { fromTable, groupedTable, mappedTable }; + } + function createFlatMappedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const flatMappedTable = declareFlatMapTable({ + tableId: "users-by-team-flat-mapped", + fromTable: groupedTable, + mapper: mapper(` + CASE + WHEN (("rowData"->>'value')::int) < 0 THEN '[]'::jsonb + ELSE jsonb_build_array( + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'base', + 'mappedValue', (("rowData"->>'value')::int + 100) + ), + jsonb_build_object( + 'team', "rowData"->'team', + 'kind', 'double', + 'mappedValue', (("rowData"->>'value')::int * 2) + ) + ) + END AS "rows" + `), + }); + return { fromTable, groupedTable, flatMappedTable }; + } + function createFilteredTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const filteredTable = declareFilterTable({ + tableId: "users-by-team-filtered", + fromTable: groupedTable, + filter: predicate(`(("rowData"->>'value')::int) >= 2`), + }); + return { fromTable, groupedTable, filteredTable }; + } + function createLimitedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const limitedTable = declareLimitTable({ + tableId: "users-by-team-limited", + fromTable: groupedTable, + limit: expr(`2`), + }); + return { fromTable, groupedTable, limitedTable }; + } + function createConcatenatedTable() { + const fromTableA = declareStoredTable<{ value: number, team: string }>({ tableId: "users-a" }); + const fromTableB = declareStoredTable<{ value: number, team: string }>({ tableId: "users-b" }); + const groupedTableA = declareGroupByTable({ + tableId: "users-a-by-team", + fromTable: fromTableA, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableB = declareGroupByTable({ + tableId: "users-b-by-team", + fromTable: fromTableB, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const concatenatedTable = declareConcatTable({ + tableId: "users-by-team-concat", + tables: [groupedTableA, groupedTableB], + }); + return { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable }; + } + function createSortedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const sortedTable = declareSortTable({ + tableId: "users-by-team-sorted", + fromTable: groupedTable, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + return { fromTable, groupedTable, sortedTable }; + } + function createDescendingSortedTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const sortedTable = declareSortTable({ + tableId: "users-by-team-sorted-desc", + fromTable: groupedTable, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${b.sql}) #>> '{}')::int) - (((${a.sql}) #>> '{}')::int)`), + }); + return { fromTable, groupedTable, sortedTable }; + } + function createDescendingLimitedTable() { + const { fromTable, groupedTable, sortedTable } = createDescendingSortedTable(); + const limitedTable = declareLimitTable({ + tableId: "users-by-team-limit-desc", + fromTable: sortedTable, + limit: expr(`2`), + }); + return { fromTable, groupedTable, sortedTable, limitedTable }; + } + function createDescendingLFoldTable() { + const { fromTable, groupedTable, sortedTable } = createDescendingSortedTable(); + const lFoldTable = declareLFoldTable({ + tableId: "users-by-team-lfold-desc", + fromTable: sortedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + "oldState" AS "newState", + jsonb_build_array( + jsonb_build_object( + 'value', (("oldRowData"->>'value')::int) + ) + ) AS "newRowsData" + `), + }); + return { fromTable, groupedTable, sortedTable, lFoldTable }; + } + function createLFoldTable() { + const { fromTable, groupedTable, sortedTable } = createSortedTable(); + const lFoldTable = declareLFoldTable({ + tableId: "users-by-team-lfold", + fromTable: sortedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + ( + COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int) + ) AS "newState", + ( + CASE + WHEN ((("oldRowData"->>'value')::int) % 2) = 0 THEN jsonb_build_array( + jsonb_build_object( + 'kind', 'running', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ), + jsonb_build_object( + 'kind', 'even-marker', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ) + ) + ELSE jsonb_build_array( + jsonb_build_object( + 'kind', 'running', + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int) + ) + ) + END + ) AS "newRowsData" + `), + }); + return { fromTable, groupedTable, sortedTable, lFoldTable }; + } + function createTimeFoldTable() { + const { fromTable, groupedTable } = createGroupedTable(); + const timeFoldTable = declareTimeFoldTable({ + tableId: "users-by-team-timefold", + fromTable: groupedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + ( + COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int) + ) AS "newState", + jsonb_build_array( + jsonb_build_object( + 'runningTotal', COALESCE(("oldState"#>>'{}')::int, 0) + (("oldRowData"->>'value')::int), + 'value', (("oldRowData"->>'value')::int), + 'timestamp', + CASE + WHEN "timestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("timestamp") + END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (now() + interval '10 minutes') + ELSE NULL::timestamptz + END AS "nextTimestamp" + `), + }); + return { fromTable, groupedTable, timeFoldTable }; + } + function createLeftJoinedTable() { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "left-join-users" }); + const joinTable = declareStoredTable<{ team: string | null, threshold: number, label: string }>({ tableId: "left-join-rules" }); + const groupedFromTable = declareGroupByTable({ + tableId: "left-join-users-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedJoinTable = declareGroupByTable({ + tableId: "left-join-rules-by-team", + fromTable: joinTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const leftJoinedTable = declareLeftJoinTable({ + tableId: "left-join-users-rules", + leftTable: groupedFromTable, + rightTable: groupedJoinTable, + leftJoinKey: mapper(`(("rowData"->>'value')::int) AS "joinKey"`), + rightJoinKey: mapper(`(("rowData"->>'threshold')::int) AS "joinKey"`), + }); + return { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable }; + } + function createFlatMapMapGroupPipeline() { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + const mappedAfterFlatMap = declareMapTable({ + tableId: "users-by-team-flat-map-then-map", + fromTable: flatMappedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ("rowData"->'kind') AS "kind", + (("rowData"->>'mappedValue')::int + 1) AS "mappedValuePlusOne" + `), + }); + const groupedByKind = declareGroupByTable({ + tableId: "users-by-kind", + fromTable: mappedAfterFlatMap, + groupBy: mapper(`"rowData"->'kind' AS "groupKey"`), + }); + return { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind }; + } + function createStackedMappedTables() { + const { fromTable, groupedTable } = createGroupedTable(); + const mappedTableLevel1 = declareMapTable({ + tableId: "users-by-team-map-level-1", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 10) AS "valuePlusTen" + `), + }); + const mappedTableLevel2 = declareMapTable({ + tableId: "users-by-team-map-level-2", + fromTable: mappedTableLevel1, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'valuePlusTen')::int * 2) AS "valueScaled", + ( + CASE + WHEN (("rowData"->>'valuePlusTen')::int * 2) >= 30 THEN 'high' + ELSE 'low' + END + ) AS "bucket" + `), + }); + return { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 }; + } + function createGroupMapGroupPipeline() { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + const groupedByBucketTable = declareGroupByTable({ + tableId: "users-by-bucket", + fromTable: mappedTableLevel2, + groupBy: mapper(`"rowData"->'bucket' AS "groupKey"`), + }); + return { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable }; + } + function registerGroupAuditTrigger( + table: ReturnType["groupedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerGroupTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerMapAuditTrigger( + table: ReturnType["mappedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerFlatMapAuditTrigger( + table: ReturnType["flatMappedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerFilterAuditTrigger( + table: ReturnType["filteredTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerLimitAuditTrigger( + table: ReturnType["limitedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerConcatAuditTrigger( + table: ReturnType["concatenatedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerSortAuditTrigger( + table: ReturnType["sortedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + jsonb_build_object( + 'rowSortKey', "oldRowSortKey", + 'rowData', "oldRowData" + ), + jsonb_build_object( + 'rowSortKey', "newRowSortKey", + 'rowData', "newRowData" + ) + FROM ${changesTable} + `, + ]); + } + function registerLFoldAuditTrigger( + table: ReturnType["lFoldTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + jsonb_build_object( + 'rowSortKey', "oldRowSortKey", + 'rowData', "oldRowData" + ), + jsonb_build_object( + 'rowSortKey', "newRowSortKey", + 'rowData', "newRowData" + ) + FROM ${changesTable} + `, + ]); + } + function registerTimeFoldAuditTrigger( + table: ReturnType["timeFoldTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + function registerLeftJoinAuditTrigger( + table: ReturnType["leftJoinedTable"], + event: string, + ) { + return table.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral(event))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + } + type TriggerLifecycleStats = { + registerCalls: number, + deregisterCalls: number, + activeRegistrations: number, + }; + function instrumentTriggerLifecycle< + T extends { + registerRowChangeTrigger( + trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[] + ): { deregister: () => void }, + }, + >(table: T): { table: T, getStats: () => TriggerLifecycleStats } { + const stats: TriggerLifecycleStats = { + registerCalls: 0, + deregisterCalls: 0, + activeRegistrations: 0, + }; + const instrumentedTable: T = { + ...table, + registerRowChangeTrigger: (trigger) => { + stats.registerCalls += 1; + stats.activeRegistrations += 1; + const registration = table.registerRowChangeTrigger(trigger); + return { + deregister: () => { + stats.deregisterCalls += 1; + stats.activeRegistrations -= 1; + registration.deregister(); + }, + }; + }, + }; + return { + table: instrumentedTable, + getStats: () => ({ ...stats }), + }; + } + + test("init/isInitialized/delete lifecycle", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + expect(await readBoolean(table.isInitialized())).toBe(false); + await runStatements(table.init()); + expect(await readBoolean(table.isInitialized())).toBe(true); + await runStatements(table.delete()); + expect(await readBoolean(table.isInitialized())).toBe(false); + }); + + test("groupBy registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-groupby-lifecycle" }); + const fromTableInstrumentation = instrumentTriggerLifecycle(fromTable); + const groupedTable = declareGroupByTable({ + tableId: "users-groupby-lifecycle-by-team", + fromTable: fromTableInstrumentation.table, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + groupedTable.init(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + groupedTable.init(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + groupedTable.delete(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + groupedTable.delete(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + groupedTable.init(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + groupedTable.delete(); + expect(fromTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("flatMap registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-flatmap-lifecycle" }); + const groupedTable = declareGroupByTable({ + tableId: "users-flatmap-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableInstrumentation = instrumentTriggerLifecycle(groupedTable); + const flatMappedTable = declareFlatMapTable({ + tableId: "users-flatmap-lifecycle-expanded", + fromTable: groupedTableInstrumentation.table, + mapper: mapper(`jsonb_build_array("rowData") AS "rows"`), + }); + + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + flatMappedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + flatMappedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + flatMappedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + flatMappedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("sort registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-sort-lifecycle" }); + const groupedTable = declareGroupByTable({ + tableId: "users-sort-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableInstrumentation = instrumentTriggerLifecycle(groupedTable); + const sortedTable = declareSortTable({ + tableId: "users-sort-lifecycle-sorted", + fromTable: groupedTableInstrumentation.table, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + sortedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + sortedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + sortedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + sortedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("limit registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-limit-lifecycle" }); + const groupedTable = declareGroupByTable({ + tableId: "users-limit-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableInstrumentation = instrumentTriggerLifecycle(groupedTable); + const limitedTable = declareLimitTable({ + tableId: "users-limit-lifecycle-limited", + fromTable: groupedTableInstrumentation.table, + limit: expr(`2`), + }); + + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + limitedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + limitedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + limitedTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + limitedTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("concat registers all upstream triggers in init and deregisters in delete", () => { + const fromTableA = declareStoredTable<{ value: number, team: string }>({ tableId: "users-concat-lifecycle-a" }); + const fromTableB = declareStoredTable<{ value: number, team: string }>({ tableId: "users-concat-lifecycle-b" }); + const groupedTableA = declareGroupByTable({ + tableId: "users-concat-lifecycle-a-by-team", + fromTable: fromTableA, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableB = declareGroupByTable({ + tableId: "users-concat-lifecycle-b-by-team", + fromTable: fromTableB, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableAInstrumentation = instrumentTriggerLifecycle(groupedTableA); + const groupedTableBInstrumentation = instrumentTriggerLifecycle(groupedTableB); + const concatenatedTable = declareConcatTable({ + tableId: "users-concat-lifecycle", + tables: [groupedTableAInstrumentation.table, groupedTableBInstrumentation.table], + }); + + expect(groupedTableAInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + expect(groupedTableBInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + concatenatedTable.init(); + expect(groupedTableAInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + expect(groupedTableBInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + concatenatedTable.delete(); + expect(groupedTableAInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + expect(groupedTableBInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + concatenatedTable.init(); + expect(groupedTableAInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + expect(groupedTableBInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + concatenatedTable.delete(); + expect(groupedTableAInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + expect(groupedTableBInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("lfold registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-lfold-lifecycle" }); + const groupedTable = declareGroupByTable({ + tableId: "users-lfold-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const sortedTable = declareSortTable({ + tableId: "users-lfold-lifecycle-sorted", + fromTable: groupedTable, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + const sortedTableInstrumentation = instrumentTriggerLifecycle(sortedTable); + const lFoldTable = declareLFoldTable({ + tableId: "users-lfold-lifecycle-folded", + fromTable: sortedTableInstrumentation.table, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + "oldState" AS "newState", + jsonb_build_array("oldRowData") AS "newRowsData" + `), + }); + + expect(sortedTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + lFoldTable.init(); + expect(sortedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + lFoldTable.delete(); + expect(sortedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + lFoldTable.init(); + expect(sortedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + lFoldTable.delete(); + expect(sortedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("timefold registers upstream trigger in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-timefold-lifecycle" }); + const groupedTable = declareGroupByTable({ + tableId: "users-timefold-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedTableInstrumentation = instrumentTriggerLifecycle(groupedTable); + const timeFoldTable = declareTimeFoldTable({ + tableId: "users-timefold-lifecycle-folded", + fromTable: groupedTableInstrumentation.table, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + "oldState" AS "newState", + jsonb_build_array("oldRowData") AS "newRowsData", + NULL::timestamptz AS "nextTimestamp" + `), + }); + + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + timeFoldTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + timeFoldTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + timeFoldTable.init(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + timeFoldTable.delete(); + expect(groupedTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("leftJoin registers all upstream triggers in init and deregisters in delete", () => { + const fromTable = declareStoredTable<{ value: number, team: string | null }>({ tableId: "users-left-join-lifecycle" }); + const joinTable = declareStoredTable<{ team: string | null, threshold: number, label: string }>({ tableId: "rules-left-join-lifecycle" }); + const groupedFromTable = declareGroupByTable({ + tableId: "users-left-join-lifecycle-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedJoinTable = declareGroupByTable({ + tableId: "rules-left-join-lifecycle-by-team", + fromTable: joinTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedFromTableInstrumentation = instrumentTriggerLifecycle(groupedFromTable); + const groupedJoinTableInstrumentation = instrumentTriggerLifecycle(groupedJoinTable); + const leftJoinedTable = declareLeftJoinTable({ + tableId: "users-rules-left-join-lifecycle", + leftTable: groupedFromTableInstrumentation.table, + rightTable: groupedJoinTableInstrumentation.table, + leftJoinKey: mapper(`(("rowData"->>'value')::int) AS "joinKey"`), + rightJoinKey: mapper(`(("rowData"->>'threshold')::int) AS "joinKey"`), + }); + + expect(groupedFromTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + expect(groupedJoinTableInstrumentation.getStats()).toEqual({ registerCalls: 0, deregisterCalls: 0, activeRegistrations: 0 }); + leftJoinedTable.init(); + expect(groupedFromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + expect(groupedJoinTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 0, activeRegistrations: 1 }); + leftJoinedTable.delete(); + expect(groupedFromTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + expect(groupedJoinTableInstrumentation.getStats()).toEqual({ registerCalls: 1, deregisterCalls: 1, activeRegistrations: 0 }); + leftJoinedTable.init(); + expect(groupedFromTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + expect(groupedJoinTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 1, activeRegistrations: 1 }); + leftJoinedTable.delete(); + expect(groupedFromTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + expect(groupedJoinTableInstrumentation.getStats()).toEqual({ registerCalls: 2, deregisterCalls: 2, activeRegistrations: 0 }); + }); + + test("trigger emits insert change row", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "insert"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "insert", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("trigger emits update change row with old and new values", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "update"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.setRow("alpha", expr(`'{"value":2}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "update", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "update", + rowIdentifier: "alpha", + oldRowData: { value: 1 }, + newRowData: { value: 2 }, + }, + ]); + }); + + test("trigger emits delete change row only when row existed", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "delete"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.deleteRow("missing")); + await runStatements(table.deleteRow("alpha")); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "delete", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "delete", + rowIdentifier: "alpha", + oldRowData: { value: 1 }, + newRowData: null, + }, + ]); + }); + + test("deregistered trigger no longer runs", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + const handle = registerAuditTrigger(table, "deregister"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + handle.deregister(); + await runStatements(table.setRow("beta", expr(`'{"value":2}'::jsonb`))); + + expect(await readTriggerAuditRows()).toEqual([ + { + event: "deregister", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("multiple triggers run in one transaction", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + registerAuditTrigger(table, "trigger_a"); + registerAuditTrigger(table, "trigger_b"); + + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + expect((await readTriggerAuditRows()).sort((a, b) => stringCompare(a.event, b.event))).toEqual([ + { + event: "trigger_a", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + { + event: "trigger_b", + rowIdentifier: "alpha", + oldRowData: null, + newRowData: { value: 1 }, + }, + ]); + }); + + test("setRow upserts and listRowsInGroup returns raw identifiers", async () => { + const table = declareStoredTable<{ value: number, label: string }>({ tableId: "users" }); + const weirdIdentifier = "row.with/slash and spaces"; + + await runStatements(table.init()); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":1,"label":"first"}'::jsonb`))); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":2,"label":"second"}'::jsonb`))); + await runStatements(table.setRow("plain-row", expr(`'{"value":3,"label":"third"}'::jsonb`))); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + + const mapped = rows + .map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier)); + + expect(mapped).toEqual([ + { + rowIdentifier: "plain-row", + rowData: { label: "third", value: 3 }, + }, + { + rowIdentifier: weirdIdentifier, + rowData: { label: "second", value: 2 }, + }, + ]); + }); + + test("storedTable all-groups rows include groupKey and respect non-null group filters", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("a", expr(`'{"value":1}'::jsonb`))); + + const allGroupsRows = await readRows(table.listRowsInGroup({ + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(allGroupsRows).toHaveLength(1); + expect(allGroupsRows[0].groupkey).toBe(null); + expect(allGroupsRows[0].rowidentifier).toBe("a"); + + const nonNullGroupRows = await readRows(table.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(nonNullGroupRows).toEqual([]); + }); + + test("table contents snapshot after init + upserts", async () => { + const table = declareStoredTable<{ value: number, label: string }>({ tableId: "users" }); + const weirdIdentifier = "row.with/slash and spaces"; + + await runStatements(table.init()); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":1,"label":"first"}'::jsonb`))); + await runStatements(table.setRow(weirdIdentifier, expr(`'{"value":2,"label":"second"}'::jsonb`))); + await runStatements(table.setRow("plain-row", expr(`'{"value":3,"label":"third"}'::jsonb`))); + + const rows = await sql.unsafe(` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), ' -> ') AS "keyPath", "value" + FROM "BulldozerStorageEngine" + ORDER BY "keyPath" + `); + const snapshotRows = [...rows].map((row) => ({ keyPath: row.keyPath, value: row.value })); + + expect(snapshotRows).toMatchInlineSnapshot(` + [ + { + "keyPath": "", + "value": null, + }, + { + "keyPath": "table", + "value": null, + }, + { + "keyPath": "table -> external:users", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage -> metadata", + "value": { + "version": 1, + }, + }, + { + "keyPath": "table -> external:users -> storage -> rows", + "value": null, + }, + { + "keyPath": "table -> external:users -> storage -> rows -> plain-row", + "value": { + "rowData": { + "label": "third", + "value": 3, + }, + }, + }, + { + "keyPath": "table -> external:users -> storage -> rows -> row.with/slash and spaces", + "value": { + "rowData": { + "label": "second", + "value": 2, + }, + }, + }, + ] + `); + }); + + test("generated keyPathParent rejects explicit writes", async () => { + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "keyPathParent", "value") + VALUES ( + ARRAY[to_jsonb('table'::text), to_jsonb('external:users'::text), to_jsonb('storage'::text), to_jsonb('rows'::text), to_jsonb('x'::text)]::jsonb[], + ARRAY[to_jsonb('table'::text), to_jsonb('external:users'::text), to_jsonb('storage'::text)]::jsonb[], + '{"rowData":{"value":1}}'::jsonb + ) + `).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"'); + }); + + test("keyPathParent foreign key rejects missing parent rows", async () => { + await expect(sql` + INSERT INTO "BulldozerStorageEngine" ("keyPath", "value") + VALUES ( + ARRAY[to_jsonb('missing-parent'::text), to_jsonb('child'::text)]::jsonb[], + '{"rowData":{"value":1}}'::jsonb + ) + `).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey'); + }); + + test("deleteRow removes only the target row and missing rows are no-op", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + + await runStatements(table.init()); + await runStatements(table.setRow("a", expr(`'{"value":1}'::jsonb`))); + await runStatements(table.setRow("b", expr(`'{"value":2}'::jsonb`))); + await runStatements(table.deleteRow("missing")); + await runStatements(table.deleteRow("a")); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(rows).toHaveLength(1); + expect(rows[0].rowdata).toEqual({ value: 2 }); + expect(await readBoolean(table.isInitialized())).toBe(true); + }); + + test("exclusive start/end excludes the single null group and rowSortKey", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("row", expr(`'{"value":1}'::jsonb`))); + + const groups = await readRows(table.listGroups({ + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: false, + endInclusive: false, + })); + expect(groups).toHaveLength(0); + + const rows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: false, + endInclusive: false, + })); + expect(rows).toHaveLength(0); + }); + + test("table paths are isolated by tableId", async () => { + const left = declareStoredTable<{ value: number }>({ tableId: "left" }); + const right = declareStoredTable<{ value: number }>({ tableId: "right" }); + + await runStatements(left.init()); + await runStatements(right.init()); + await runStatements(left.setRow("shared", expr(`'{"value":1}'::jsonb`))); + await runStatements(right.setRow("shared", expr(`'{"value":2}'::jsonb`))); + await runStatements(left.delete()); + + const rightRows = await readRows(right.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + + expect(await readBoolean(left.isInitialized())).toBe(false); + expect(await readBoolean(right.isInitialized())).toBe(true); + expect(rightRows).toHaveLength(1); + expect(rightRows[0].rowdata).toEqual({ value: 2 }); + }); + + test("rowIdentifier from listRowsInGroup can be passed to deleteRow", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("plain-row", expr(`'{"value":1}'::jsonb`))); + + const listedRows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(listedRows).toHaveLength(1); + + await runStatements(table.deleteRow(listedRows[0].rowidentifier)); + + const remainingRows = await readRows(table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + })); + expect(remainingRows).toHaveLength(0); + }); + + test("groupBy init backfills groups and rows from source table", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + + await runStatements(groupedTable.init()); + + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1", rowData: { team: "alpha", value: 1 } }, + { rowIdentifier: "u3", rowData: { team: "alpha", value: 3 } }, + ]); + + const allRows = await readRows(groupedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1" }, + { groupKey: "alpha", rowIdentifier: "u3" }, + { groupKey: "beta", rowIdentifier: "u2" }, + ]); + }); + + test("groupBy registerRowChangeTrigger emits insert/update/move/delete changes", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerGroupAuditTrigger(groupedTable, "group_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", value: 1 }, + }, + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", value: 1 }, + newRowData: { team: "alpha", value: 2 }, + }, + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: { team: "alpha", value: 2 }, + newRowData: null, + }, + { + event: "group_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "beta", value: 3 }, + }, + { + event: "group_change", + groupKey: "beta", + rowIdentifier: "u1", + oldRowData: { team: "beta", value: 3 }, + newRowData: null, + }, + ]); + }); + + test("groupBy deregistered trigger no longer runs", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + const handle = registerGroupAuditTrigger(groupedTable, "group_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + handle.deregister(); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "group_change", + groupKey: "alpha", + rowIdentifier: "u1", + oldRowData: null, + newRowData: { team: "alpha", value: 1 }, + }, + ]); + }); + + test("groupBy stays no-op while uninitialized", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + registerGroupAuditTrigger(groupedTable, "group_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readBoolean(groupedTable.isInitialized())).toBe(false); + expect(await readGroupTriggerAuditRows()).toEqual([]); + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + }); + + test("groupBy delete cleans up and re-init backfills from source", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(groupedTable.delete()); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readBoolean(groupedTable.isInitialized())).toBe(false); + const groupsBeforeReinit = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsBeforeReinit).toEqual([]); + + await runStatements(groupedTable.init()); + const groupsAfterReinit = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + }); + + test("groupBy listGroups applies group-key ranges", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + + const inclusive = await readRows(groupedTable.listGroups({ + start: expr(`to_jsonb('beta'::text)`), + end: expr(`to_jsonb('gamma'::text)`), + startInclusive: true, + endInclusive: true, + })); + expect(inclusive.map((row) => row.groupkey).sort(stringCompare)).toEqual(["beta", "gamma"]); + + const exclusive = await readRows(groupedTable.listGroups({ + start: expr(`to_jsonb('beta'::text)`), + end: expr(`to_jsonb('gamma'::text)`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusive).toEqual([]); + }); + + test("groupBy removes empty groups after moves and deletes", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + const groupsAfterInsert = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterInsert.map((row) => row.groupkey)).toEqual(["alpha"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`))); + const groupsAfterMove = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["beta"]); + + await runStatements(fromTable.deleteRow("u1")); + const groupsAfterDelete = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterDelete).toEqual([]); + }); + + test("groupBy deletes stale group paths from storage", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + const staleGroupPaths = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath"[1:4] = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team'::text), + to_jsonb('storage'::text), + to_jsonb('groups'::text) + ]::jsonb[] + AND cardinality("keyPath") > 4 + ORDER BY "keyPath" + `; + expect(staleGroupPaths).toEqual([]); + }); + + test("groupBy listRowsInGroup handles missing groups and exclusive bounds", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const missingGroupRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('missing'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(missingGroupRows).toEqual([]); + + const exclusiveRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusiveRows).toEqual([]); + + const inclusiveRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: true, + endInclusive: true, + })); + expect(inclusiveRows).toHaveLength(2); + }); + + test("groupBy listRowsInGroup (all groups) handles 'rows' collisions in group key and row identifier", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const allRows = await readRows(groupedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const normalizedRows = allRows + .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`)); + + expect(normalizedRows).toEqual([ + { groupKey: "alpha", rowIdentifier: "rows", rowData: { team: "alpha", value: 2 } }, + { groupKey: "rows", rowIdentifier: "u1", rowData: { team: "rows", value: 1 } }, + ]); + }); + + test("groupBy multiple triggers run in one transaction", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerGroupAuditTrigger(groupedTable, "group_trigger_a"); + registerGroupAuditTrigger(groupedTable, "group_trigger_b"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + const rows = await readGroupTriggerAuditRows(); + expect(rows.map((row) => row.event).sort(stringCompare)).toEqual(["group_trigger_a", "group_trigger_b"]); + }); + + test("groupBy supports null group keys and transitions away cleanly", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":null,"value":1}'::jsonb`))); + const nullGroupRows = await readRows(groupedTable.listRowsInGroup({ + groupKey: expr(`'null'::jsonb`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(nullGroupRows.map((row) => row.rowidentifier)).toEqual(["u1"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + const groups = await readRows(groupedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + + test("mapTable init backfills groups and mapped rows", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + + const groups = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1:1", rowData: { team: "alpha", mappedValue: 101 } }, + { rowIdentifier: "u3:1", rowData: { team: "alpha", mappedValue: 103 } }, + ]); + + const allRows = await readRows(mappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1:1" }, + { groupKey: "alpha", rowIdentifier: "u3:1" }, + { groupKey: "beta", rowIdentifier: "u2:1" }, + ]); + }); + + test("mapTable registerRowChangeTrigger emits mapped insert/update/move/delete changes", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + registerMapAuditTrigger(mappedTable, "map_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "alpha", mappedValue: 101 }, + }, + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", mappedValue: 101 }, + newRowData: { team: "alpha", mappedValue: 102 }, + }, + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", mappedValue: 102 }, + newRowData: null, + }, + { + event: "map_change", + groupKey: "beta", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "beta", mappedValue: 103 }, + }, + { + event: "map_change", + groupKey: "beta", + rowIdentifier: "u1:1", + oldRowData: { team: "beta", mappedValue: 103 }, + newRowData: null, + }, + ]); + }); + + test("mapTable uses flatMap-style rowIdentifier and skips unchanged updates", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + registerMapAuditTrigger(mappedTable, "map_change"); + + await runStatements(fromTable.setRow("user:1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("user:1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "user:1:1", + oldRowData: null, + newRowData: { team: "alpha", mappedValue: 101 }, + }, + ]); + + const alphaRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["user:1:1"]); + }); + + test("mapTable deregistered trigger no longer runs", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + const handle = registerMapAuditTrigger(mappedTable, "map_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + handle.deregister(); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_change", + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "alpha", mappedValue: 101 }, + }, + ]); + }); + + test("mapTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerMapAuditTrigger(mappedTable, "map_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readBoolean(mappedTable.isInitialized())).toBe(false); + expect(await readMapTriggerAuditRows()).toEqual([]); + const groups = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + }); + + test("mapTable delete cleans up and re-init backfills from source", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(mappedTable.delete()); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readBoolean(mappedTable.isInitialized())).toBe(false); + const groupsBeforeReinit = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsBeforeReinit).toEqual([]); + + await runStatements(mappedTable.init()); + const groupsAfterReinit = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + }); + + test("mapTable listRowsInGroup handles missing groups and exclusive bounds", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + const missingGroupRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('missing'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(missingGroupRows).toEqual([]); + + const exclusiveRows = await readRows(mappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`'null'::jsonb`), + end: expr(`'null'::jsonb`), + startInclusive: false, + endInclusive: false, + })); + expect(exclusiveRows).toEqual([]); + }); + + test("mapTable listRowsInGroup (all groups) handles 'rows' collisions in group key and row identifier", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const allRows = await readRows(mappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const normalizedRows = allRows + .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`)); + + expect(normalizedRows).toEqual([ + { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", mappedValue: 102 } }, + { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", mappedValue: 101 } }, + ]); + }); + + test("mapTable deletes stale group paths from storage", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + const staleGroupPaths = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath"[1:4] = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team-mapped'::text), + to_jsonb('storage'::text), + to_jsonb('groups'::text) + ]::jsonb[] + AND cardinality("keyPath") > 4 + ORDER BY "keyPath" + `; + expect(staleGroupPaths).toEqual([]); + }); + + test("mapTable matches equivalent single-row flatMap for rows, groups, and trigger payloads", async () => { + const { fromTable, groupedTable, mappedTable } = createMappedTable(); + const equivalentFlatMapTable = declareFlatMapTable({ + tableId: "users-by-team-mapped-equivalent-flatmap", + fromTable: groupedTable, + mapper: mapper(` + jsonb_build_array( + COALESCE( + ( + SELECT to_jsonb("mapped") + FROM ( + SELECT + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValue" + ) AS "mapped" + ), + 'null'::jsonb + ) + ) AS "rows" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTable.init()); + await runStatements(equivalentFlatMapTable.init()); + + mappedTable.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("map"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + equivalentFlatMapTable.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("flat"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + const mapGroups = await readRows(mappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const flatGroups = await readRows(equivalentFlatMapTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(mapGroups).toEqual(flatGroups); + + const normalizeRows = (rows: Iterable>) => [...rows] + .map((row) => ({ + groupKey: (Reflect.get(row, "groupkey") as string | null), + rowIdentifier: String(Reflect.get(row, "rowidentifier")), + rowData: Reflect.get(row, "rowdata"), + })) + .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`)); + const mapRows = normalizeRows(await readRows(mappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))); + const flatRows = normalizeRows(await readRows(equivalentFlatMapTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))); + expect(mapRows).toEqual(flatRows); + + const normalizeAuditRows = (rows: Iterable>) => [...rows] + .map((row) => ({ + groupKey: (Reflect.get(row, "groupKey") as string | null), + rowIdentifier: String(Reflect.get(row, "rowIdentifier")), + oldRowData: Reflect.get(row, "oldRowData"), + newRowData: Reflect.get(row, "newRowData"), + })) + .sort((a, b) => stringCompare( + `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`, + `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`, + )); + const allAuditRows = await readMapTriggerAuditRows(); + const mapAudit = normalizeAuditRows(allAuditRows.filter((row) => row.event === "map")); + const flatAudit = normalizeAuditRows(allAuditRows.filter((row) => row.event === "flat")); + expect(mapAudit).toEqual(flatAudit); + }); + + test("flatMapTable init backfills fan-out rows and skips empty expansions", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":-1}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + + const groups = await readRows(flatMappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(flatMappedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1:1", rowData: { team: "alpha", kind: "base", mappedValue: 101 } }, + { rowIdentifier: "u1:2", rowData: { team: "alpha", kind: "double", mappedValue: 2 } }, + ]); + + const allRows = await readRows(flatMappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1:1" }, + { groupKey: "alpha", rowIdentifier: "u1:2" }, + { groupKey: "beta", rowIdentifier: "u2:1" }, + { groupKey: "beta", rowIdentifier: "u2:2" }, + ]); + }); + + test("flatMapTable registerRowChangeTrigger emits per-expanded-row inserts, updates, moves, and removals", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + registerFlatMapAuditTrigger(flatMappedTable, "flat_map_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":-1}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + const normalizedAuditRows = (await readMapTriggerAuditRows()) + .map((row) => ({ + groupKey: row.groupKey, + rowIdentifier: row.rowIdentifier, + oldRowData: row.oldRowData, + newRowData: row.newRowData, + })) + .sort((a, b) => stringCompare( + `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`, + `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`, + )); + expect(normalizedAuditRows).toEqual([ + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "alpha", kind: "base", mappedValue: 101 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", kind: "base", mappedValue: 101 }, + newRowData: { team: "alpha", kind: "base", mappedValue: 102 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", kind: "base", mappedValue: 102 }, + newRowData: null, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:2", + oldRowData: null, + newRowData: { team: "alpha", kind: "double", mappedValue: 2 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:2", + oldRowData: { team: "alpha", kind: "double", mappedValue: 2 }, + newRowData: { team: "alpha", kind: "double", mappedValue: 4 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:2", + oldRowData: { team: "alpha", kind: "double", mappedValue: 4 }, + newRowData: null, + }, + { + groupKey: "beta", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "beta", kind: "base", mappedValue: 103 }, + }, + { + groupKey: "beta", + rowIdentifier: "u1:1", + oldRowData: { team: "beta", kind: "base", mappedValue: 103 }, + newRowData: null, + }, + { + groupKey: "beta", + rowIdentifier: "u1:2", + oldRowData: null, + newRowData: { team: "beta", kind: "double", mappedValue: 6 }, + }, + { + groupKey: "beta", + rowIdentifier: "u1:2", + oldRowData: { team: "beta", kind: "double", mappedValue: 6 }, + newRowData: null, + }, + ]); + }); + + test("flatMapTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerFlatMapAuditTrigger(flatMappedTable, "flat_map_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + + expect(await readBoolean(flatMappedTable.isInitialized())).toBe(false); + expect(await readMapTriggerAuditRows()).toEqual([]); + const groups = await readRows(flatMappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + }); + + test("flatMapTable delete cleans up and re-init backfills from source", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(flatMappedTable.delete()); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + expect(await readBoolean(flatMappedTable.isInitialized())).toBe(false); + const groupsBeforeReinit = await readRows(flatMappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsBeforeReinit).toEqual([]); + + await runStatements(flatMappedTable.init()); + const groupsAfterReinit = await readRows(flatMappedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterReinit.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + }); + + test("flatMapTable listRowsInGroup (all groups) handles 'rows' collisions in group key and source row identifier", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const allRows = await readRows(flatMappedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const normalizedRows = allRows + .map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`)); + + expect(normalizedRows).toEqual([ + { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", kind: "base", mappedValue: 102 } }, + { groupKey: "alpha", rowIdentifier: "rows:2", rowData: { team: "alpha", kind: "double", mappedValue: 4 } }, + { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", kind: "base", mappedValue: 101 } }, + { groupKey: "rows", rowIdentifier: "u1:2", rowData: { team: "rows", kind: "double", mappedValue: 2 } }, + ]); + }); + + test("flatMapTable deletes stale group paths from storage", async () => { + const { fromTable, groupedTable, flatMappedTable } = createFlatMappedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":-1}'::jsonb`))); + + const staleGroupPaths = await sql` + SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath" + FROM "BulldozerStorageEngine" + WHERE "keyPath"[1:4] = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team-flat-mapped'::text), + to_jsonb('storage'::text), + to_jsonb('groups'::text) + ]::jsonb[] + AND cardinality("keyPath") > 4 + ORDER BY "keyPath" + `; + expect(staleGroupPaths).toEqual([]); + }); + + test("filterTable init backfills matching rows, keeps own metadata, and deletes cleanly", async () => { + const { fromTable, groupedTable, filteredTable } = createFilteredTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"beta","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u4", expr(`'{"team":"beta","value":0}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(filteredTable.init()); + + expect(await readBoolean(filteredTable.isInitialized())).toBe(true); + + const groups = await readRows(filteredTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const allRows = await readRows(filteredTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u2:1", rowData: { team: "alpha", value: 2 } }, + { groupKey: "beta", rowIdentifier: "u3:1", rowData: { team: "beta", value: 3 } }, + ]); + + const metadataRows = await sql` + SELECT 1 + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team-filtered'::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + `; + expect(metadataRows).toHaveLength(1); + + await runStatements(filteredTable.delete()); + expect(await readBoolean(filteredTable.isInitialized())).toBe(false); + const groupsAfterDelete = await readRows(filteredTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterDelete).toEqual([]); + }); + + test("filterTable registerRowChangeTrigger emits inserts, updates, deletes, and moves", async () => { + const { fromTable, groupedTable, filteredTable } = createFilteredTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(filteredTable.init()); + registerFilterAuditTrigger(filteredTable, "filter_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":5}'::jsonb`))); + + const normalizedAuditRows = (await readMapTriggerAuditRows()) + .map((row) => ({ + groupKey: row.groupKey, + rowIdentifier: row.rowIdentifier, + oldRowData: row.oldRowData, + newRowData: row.newRowData, + })) + .sort((a, b) => stringCompare( + `${a.groupKey}:${a.rowIdentifier}:${JSON.stringify(a.oldRowData)}:${JSON.stringify(a.newRowData)}`, + `${b.groupKey}:${b.rowIdentifier}:${JSON.stringify(b.oldRowData)}:${JSON.stringify(b.newRowData)}`, + )); + expect(normalizedAuditRows).toEqual([ + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "alpha", value: 2 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", value: 2 }, + newRowData: { team: "alpha", value: 3 }, + }, + { + groupKey: "alpha", + rowIdentifier: "u1:1", + oldRowData: { team: "alpha", value: 3 }, + newRowData: null, + }, + { + groupKey: "beta", + rowIdentifier: "u1:1", + oldRowData: null, + newRowData: { team: "beta", value: 5 }, + }, + ]); + }); + + test("filterTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, filteredTable } = createFilteredTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerFilterAuditTrigger(filteredTable, "filter_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + + expect(await readBoolean(filteredTable.isInitialized())).toBe(false); + expect(await readMapTriggerAuditRows()).toEqual([]); + expect(await readRows(filteredTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + }); + + test("filterTable listRowsInGroup (all groups) handles 'rows' collisions in group key and source row identifier", async () => { + const { fromTable, groupedTable, filteredTable } = createFilteredTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(filteredTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"rows","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("rows", expr(`'{"team":"alpha","value":4}'::jsonb`))); + + const allRows = await readRows(filteredTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "rows:1", rowData: { team: "alpha", value: 4 } }, + { groupKey: "rows", rowIdentifier: "u1:1", rowData: { team: "rows", value: 5 } }, + ]); + }); + + test("limitTable init keeps only first N rows per group and stores metadata", async () => { + const { fromTable, groupedTable, limitedTable } = createLimitedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":1}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(limitedTable.init()); + + const groups = await readRows(limitedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const allRows = await readRows(limitedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "a1", rowData: { team: "alpha", value: 1 } }, + { groupKey: "alpha", rowIdentifier: "a2", rowData: { team: "alpha", value: 2 } }, + { groupKey: "beta", rowIdentifier: "b1", rowData: { team: "beta", value: 1 } }, + { groupKey: "beta", rowIdentifier: "b2", rowData: { team: "beta", value: 2 } }, + ]); + + const metadataRows = await sql` + SELECT 1 + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team-limited'::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + `; + expect(metadataRows).toHaveLength(1); + }); + + test("limitTable membership shifts when boundary rows are inserted, updated, or deleted", async () => { + const { fromTable, groupedTable, limitedTable } = createLimitedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(limitedTable.init()); + + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + let alphaRows = await readRows(limitedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u2", "u3"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + alphaRows = await readRows(limitedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u1", "u2"]); + + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":22}'::jsonb`))); + alphaRows = await readRows(limitedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u1", rowData: { team: "alpha", value: 1 } }, + { rowIdentifier: "u2", rowData: { team: "alpha", value: 22 } }, + ]); + + await runStatements(fromTable.deleteRow("u1")); + alphaRows = await readRows(limitedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["u2", "u3"]); + }); + + test("limitTable trigger stream reconstructs the same final state as listRowsInGroup", async () => { + const { fromTable, groupedTable, limitedTable } = createLimitedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(limitedTable.init()); + registerLimitAuditTrigger(limitedTable, "limit_change"); + + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("a4", expr(`'{"team":"alpha","value":4}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.deleteRow("a1")); + await runStatements(fromTable.setRow("a5", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("a0", expr(`'{"team":"alpha","value":0}'::jsonb`))); + await runStatements(fromTable.deleteRow("a2")); + await runStatements(fromTable.setRow("a0", expr(`'{"team":"beta","value":100}'::jsonb`))); + + const auditRows = (await readMapTriggerAuditRows()) + .filter((row) => row.event === "limit_change"); + const reconstructed = new Map(); + for (const row of auditRows) { + const groupKey = row.groupKey as string | null; + const rowIdentifier = String(row.rowIdentifier); + const key = `${groupKey ?? "__NULL__"}:${rowIdentifier}`; + if (row.newRowData == null) { + reconstructed.delete(key); + } else { + reconstructed.set(key, { groupKey, rowIdentifier, rowData: row.newRowData }); + } + } + + const actualRows = (await readRows(limitedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: String(row.rowidentifier), + rowData: row.rowdata, + })); + const reconstructedRows = [...reconstructed.values()]; + const sortRows = (rows: Array<{ groupKey: string | null, rowIdentifier: string, rowData: unknown }>) => rows + .sort((a, b) => stringCompare( + `${a.groupKey ?? "__NULL__"}:${a.rowIdentifier}:${JSON.stringify(a.rowData)}`, + `${b.groupKey ?? "__NULL__"}:${b.rowIdentifier}:${JSON.stringify(b.rowData)}`, + )); + expect(sortRows(reconstructedRows)).toEqual(sortRows(actualRows)); + }); + + test("limitTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, limitedTable } = createLimitedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerLimitAuditTrigger(limitedTable, "limit_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + expect(await readBoolean(limitedTable.isInitialized())).toBe(false); + const groups = await readRows(limitedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups).toEqual([]); + const limitAuditRows = (await readMapTriggerAuditRows()).filter((row) => row.event === "limit_change"); + expect(limitAuditRows).toEqual([]); + }); + + test("concatTable virtually concatenates grouped inputs and prefixes row identifiers", async () => { + const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable(); + await runStatements(fromTableA.init()); + await runStatements(fromTableB.init()); + await runStatements(groupedTableA.init()); + await runStatements(groupedTableB.init()); + + await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTableA.setRow("a2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTableB.setRow("b1", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTableB.setRow("b2", expr(`'{"team":"gamma","value":4}'::jsonb`))); + + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false); + expect(await readRows(concatenatedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + await runStatements(concatenatedTable.init()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + + const groups = await readRows(concatenatedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta", "gamma"]); + + const alphaRows = await readRows(concatenatedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows + .map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })) + .sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))) + .toEqual([ + { rowIdentifier: "0:a1", rowData: { team: "alpha", value: 1 } }, + { rowIdentifier: "1:b1", rowData: { team: "alpha", value: 3 } }, + ]); + + const allRows = await readRows(concatenatedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRows + .map((row) => ({ + groupKey: row.groupkey, + rowIdentifier: row.rowidentifier, + rowData: row.rowdata, + })) + .sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))) + .toEqual([ + { groupKey: "alpha", rowIdentifier: "0:a1", rowData: { team: "alpha", value: 1 } }, + { groupKey: "alpha", rowIdentifier: "1:b1", rowData: { team: "alpha", value: 3 } }, + { groupKey: "beta", rowIdentifier: "0:a2", rowData: { team: "beta", value: 2 } }, + { groupKey: "gamma", rowIdentifier: "1:b2", rowData: { team: "gamma", value: 4 } }, + ]); + }); + + test("concatTable forwards prefixed trigger changes from each input table", async () => { + const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable(); + await runStatements(fromTableA.init()); + await runStatements(fromTableB.init()); + await runStatements(groupedTableA.init()); + await runStatements(groupedTableB.init()); + await runStatements(concatenatedTable.init()); + registerConcatAuditTrigger(concatenatedTable, "concat_change"); + + await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTableB.setRow("b1", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTableB.setRow("b1", expr(`'{"team":"gamma","value":5}'::jsonb`))); + await runStatements(fromTableA.deleteRow("a1")); + + const auditRows = (await readMapTriggerAuditRows()) + .filter((row) => row.event === "concat_change") + .map((row) => ({ + groupKey: row.groupKey, + rowIdentifier: row.rowIdentifier, + oldRowData: row.oldRowData, + newRowData: row.newRowData, + })); + expect(auditRows).toEqual([ + { groupKey: "alpha", rowIdentifier: "0:a1", oldRowData: null, newRowData: { team: "alpha", value: 1 } }, + { groupKey: "beta", rowIdentifier: "1:b1", oldRowData: null, newRowData: { team: "beta", value: 2 } }, + { groupKey: "beta", rowIdentifier: "1:b1", oldRowData: { team: "beta", value: 2 }, newRowData: null }, + { groupKey: "gamma", rowIdentifier: "1:b1", oldRowData: null, newRowData: { team: "gamma", value: 5 } }, + { groupKey: "alpha", rowIdentifier: "0:a1", oldRowData: { team: "alpha", value: 1 }, newRowData: null }, + ]); + }); + + test("concatTable stays virtual but requires its own metadata initialization", async () => { + const { fromTableA, fromTableB, groupedTableA, groupedTableB, concatenatedTable } = createConcatenatedTable(); + + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false); + + const beforeInitGroups = await readRows(concatenatedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(beforeInitGroups).toEqual([]); + + await runStatements(fromTableA.init()); + await runStatements(groupedTableA.init()); + await runStatements(fromTableA.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false); + + const oneSideOnlyRows = await readRows(concatenatedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(oneSideOnlyRows).toEqual([]); + + await runStatements(concatenatedTable.init()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + const rowsAfterConcatInit = await readRows(concatenatedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(rowsAfterConcatInit.map((row) => row.rowidentifier)).toEqual(["0:a1"]); + + await runStatements(fromTableB.init()); + await runStatements(groupedTableB.init()); + await runStatements(fromTableB.setRow("b1", expr(`'{"team":"beta","value":2}'::jsonb`))); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + + await runStatements(concatenatedTable.delete()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(false); + + const rowsAfterDelete = await readRows(concatenatedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(rowsAfterDelete).toEqual([]); + + await runStatements(concatenatedTable.init()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + + await runStatements(groupedTableB.delete()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + const rowsAfterInputDelete = await readRows(concatenatedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(rowsAfterInputDelete.map((row) => row.rowidentifier)).toEqual(["0:a1"]); + }); + + test("concatTable allows input tables with different sort comparators", async () => { + const fromTableAsc = declareStoredTable<{ value: number, team: string }>({ tableId: "users-concat-sort-asc" }); + const groupedTableAsc = declareGroupByTable({ + tableId: "users-concat-sort-asc-by-team", + fromTable: fromTableAsc, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const sortedTableAsc = declareSortTable({ + tableId: "users-concat-sort-asc-sorted", + fromTable: groupedTableAsc, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${a.sql}) #>> '{}')::int) - (((${b.sql}) #>> '{}')::int)`), + }); + + const fromTableDesc = declareStoredTable<{ value: number, team: string }>({ tableId: "users-concat-sort-desc" }); + const groupedTableDesc = declareGroupByTable({ + tableId: "users-concat-sort-desc-by-team", + fromTable: fromTableDesc, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const sortedTableDesc = declareSortTable({ + tableId: "users-concat-sort-desc-sorted", + fromTable: groupedTableDesc, + getSortKey: mapper(`(("rowData"->>'value')::int) AS "newSortKey"`), + compareSortKeys: (a, b) => expr(`(((${b.sql}) #>> '{}')::int) - (((${a.sql}) #>> '{}')::int)`), + }); + + const concatenatedTable = declareConcatTable({ + tableId: "users-by-team-concat-sort-mismatch", + tables: [sortedTableAsc, sortedTableDesc], + }); + + await runStatements(fromTableAsc.init()); + await runStatements(groupedTableAsc.init()); + await runStatements(sortedTableAsc.init()); + await runStatements(fromTableDesc.init()); + await runStatements(groupedTableDesc.init()); + await runStatements(sortedTableDesc.init()); + + await runStatements(fromTableAsc.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTableDesc.setRow("b1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + await runStatements(concatenatedTable.init()); + expect(await readBoolean(concatenatedTable.isInitialized())).toBe(true); + + const alphaRows = await readRows(concatenatedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["0:a1", "1:b1"]); + }); + + test("sortTable init backfills rows in computed sort order and stores metadata", async () => { + const { fromTable, groupedTable, sortedTable } = createSortedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":1}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + + expect(await readBoolean(sortedTable.isInitialized())).toBe(true); + const groups = await readRows(sortedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(sortedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowSortKey: row.rowsortkey, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a1", rowSortKey: 1, rowData: { team: "alpha", value: 1 } }, + { rowIdentifier: "a2", rowSortKey: 2, rowData: { team: "alpha", value: 2 } }, + { rowIdentifier: "a3", rowSortKey: 3, rowData: { team: "alpha", value: 3 } }, + ]); + + const metadataRows = await sql` + SELECT 1 + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ARRAY[ + to_jsonb('table'::text), + to_jsonb('external:users-by-team-sorted'::text), + to_jsonb('storage'::text), + to_jsonb('metadata'::text) + ]::jsonb[] + `; + expect(metadataRows).toHaveLength(1); + }); + + test("sortTable emits insert, update, move, and delete changes with computed sort keys", async () => { + const { fromTable, groupedTable, sortedTable } = createSortedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + registerSortAuditTrigger(sortedTable, "sort_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":0}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":1}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + const auditRows = (await readMapTriggerAuditRows()) + .filter((row) => row.event === "sort_change") + .map((row) => ({ + groupKey: row.groupKey, + rowIdentifier: row.rowIdentifier, + oldRowData: row.oldRowData, + newRowData: row.newRowData, + })); + expect(auditRows).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1", oldRowData: { rowSortKey: null, rowData: null }, newRowData: { rowSortKey: 3, rowData: { team: "alpha", value: 3 } } }, + { groupKey: "alpha", rowIdentifier: "u2", oldRowData: { rowSortKey: null, rowData: null }, newRowData: { rowSortKey: 1, rowData: { team: "alpha", value: 1 } } }, + { groupKey: "alpha", rowIdentifier: "u1", oldRowData: { rowSortKey: 3, rowData: { team: "alpha", value: 3 } }, newRowData: { rowSortKey: 0, rowData: { team: "alpha", value: 0 } } }, + { groupKey: "alpha", rowIdentifier: "u2", oldRowData: { rowSortKey: 1, rowData: { team: "alpha", value: 1 } }, newRowData: { rowSortKey: null, rowData: null } }, + { groupKey: "beta", rowIdentifier: "u2", oldRowData: { rowSortKey: null, rowData: null }, newRowData: { rowSortKey: 1, rowData: { team: "beta", value: 1 } } }, + { groupKey: "alpha", rowIdentifier: "u1", oldRowData: { rowSortKey: 0, rowData: { team: "alpha", value: 0 } }, newRowData: { rowSortKey: null, rowData: null } }, + ]); + }); + + test("sortTable listRowsInGroup supports sort key range filtering", async () => { + const { fromTable, groupedTable, sortedTable } = createSortedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u4", expr(`'{"team":"alpha","value":4}'::jsonb`))); + + const midRows = await readRows(sortedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: expr(`to_jsonb(2)`), + end: expr(`to_jsonb(4)`), + startInclusive: true, + endInclusive: false, + })); + expect(midRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowSortKey: row.rowsortkey }))).toEqual([ + { rowIdentifier: "u2", rowSortKey: 2 }, + { rowIdentifier: "u3", rowSortKey: 3 }, + ]); + }); + + test("sortTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, sortedTable } = createSortedTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerSortAuditTrigger(sortedTable, "sort_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + + expect(await readBoolean(sortedTable.isInitialized())).toBe(false); + expect((await readMapTriggerAuditRows()).filter((row) => row.event === "sort_change")).toEqual([]); + expect(await readRows(sortedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + }); + + test("lFoldTable init backfills flattened rows in deterministic sorted order", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":4}'::jsonb`))); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + + expect(await readBoolean(lFoldTable.isInitialized())).toBe(true); + const groups = await readRows(lFoldTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ + rowIdentifier: row.rowidentifier, + rowSortKey: row.rowsortkey, + rowData: row.rowdata, + }))).toEqual([ + { rowIdentifier: "a1:1", rowSortKey: 1, rowData: { kind: "running", runningTotal: 1, value: 1 } }, + { rowIdentifier: "a2:1", rowSortKey: 2, rowData: { kind: "running", runningTotal: 3, value: 2 } }, + { rowIdentifier: "a2:2", rowSortKey: 2, rowData: { kind: "even-marker", runningTotal: 3, value: 2 } }, + { rowIdentifier: "a3:1", rowSortKey: 2, rowData: { kind: "running", runningTotal: 5, value: 2 } }, + { rowIdentifier: "a3:2", rowSortKey: 2, rowData: { kind: "even-marker", runningTotal: 5, value: 2 } }, + ]); + }); + + test("lFoldTable recomputes only affected suffix and handles reorder/delete transitions", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":5}'::jsonb`))); + + const beforeTailUpdate = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(beforeTailUpdate.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a1:1", rowData: { kind: "running", runningTotal: 1, value: 1 } }, + { rowIdentifier: "a2:1", rowData: { kind: "running", runningTotal: 4, value: 3 } }, + { rowIdentifier: "a3:1", rowData: { kind: "running", runningTotal: 9, value: 5 } }, + ]); + + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":6}'::jsonb`))); + const afterTailUpdate = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(afterTailUpdate.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a1:1", rowData: { kind: "running", runningTotal: 1, value: 1 } }, + { rowIdentifier: "a2:1", rowData: { kind: "running", runningTotal: 4, value: 3 } }, + { rowIdentifier: "a3:1", rowData: { kind: "running", runningTotal: 10, value: 6 } }, + { rowIdentifier: "a3:2", rowData: { kind: "even-marker", runningTotal: 10, value: 6 } }, + ]); + + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":0}'::jsonb`))); + const afterMiddleMove = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(afterMiddleMove.map((row) => ({ rowIdentifier: row.rowidentifier, rowSortKey: row.rowsortkey, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a2:1", rowSortKey: 0, rowData: { kind: "running", runningTotal: 0, value: 0 } }, + { rowIdentifier: "a2:2", rowSortKey: 0, rowData: { kind: "even-marker", runningTotal: 0, value: 0 } }, + { rowIdentifier: "a1:1", rowSortKey: 1, rowData: { kind: "running", runningTotal: 1, value: 1 } }, + { rowIdentifier: "a3:1", rowSortKey: 6, rowData: { kind: "running", runningTotal: 7, value: 6 } }, + { rowIdentifier: "a3:2", rowSortKey: 6, rowData: { kind: "even-marker", runningTotal: 7, value: 6 } }, + ]); + + await runStatements(fromTable.deleteRow("a1")); + const afterDelete = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(afterDelete.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a2:1", rowData: { kind: "running", runningTotal: 0, value: 0 } }, + { rowIdentifier: "a2:2", rowData: { kind: "even-marker", runningTotal: 0, value: 0 } }, + { rowIdentifier: "a3:1", rowData: { kind: "running", runningTotal: 6, value: 6 } }, + { rowIdentifier: "a3:2", rowData: { kind: "even-marker", runningTotal: 6, value: 6 } }, + ]); + }); + + test("lFoldTable trigger stream reconstructs exact final table state", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + registerLFoldAuditTrigger(lFoldTable, "lfold_change"); + + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":4}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":6}'::jsonb`))); + await runStatements(fromTable.deleteRow("a1")); + + const auditRows = (await readMapTriggerAuditRows()).filter((row) => row.event === "lfold_change"); + const reconstructed = new Map(); + for (const row of auditRows) { + const groupKey = row.groupKey as string | null; + const rowIdentifier = String(row.rowIdentifier); + const key = `${groupKey ?? "__NULL__"}:${rowIdentifier}`; + const payload = row.newRowData as Record | null; + const newRowData = payload == null ? null : Reflect.get(payload, "rowData"); + const newRowSortKey = payload == null ? null : Reflect.get(payload, "rowSortKey"); + if (newRowData == null) { + reconstructed.delete(key); + } else { + reconstructed.set(key, { groupKey, rowIdentifier, rowSortKey: newRowSortKey, rowData: newRowData }); + } + } + + const actualRows = (await readRows(lFoldTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: String(row.rowidentifier), + rowSortKey: row.rowsortkey, + rowData: row.rowdata, + })); + const reconstructedRows = [...reconstructed.values()]; + const sortRows = (rows: Array<{ groupKey: string | null, rowIdentifier: string, rowSortKey: unknown, rowData: unknown }>) => rows + .sort((a, b) => stringCompare( + `${a.groupKey ?? "__NULL__"}:${a.rowIdentifier}:${JSON.stringify(a.rowSortKey)}:${JSON.stringify(a.rowData)}`, + `${b.groupKey ?? "__NULL__"}:${b.rowIdentifier}:${JSON.stringify(b.rowSortKey)}:${JSON.stringify(b.rowData)}`, + )); + expect(sortRows(reconstructedRows)).toEqual(sortRows(actualRows)); + }); + + test("lFoldTable uses rowIdentifier as deterministic tie-breaker for equal sort keys", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + + await runStatements(fromTable.setRow("z", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + const alphaRows = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a:1", rowData: { kind: "running", runningTotal: 2, value: 2 } }, + { rowIdentifier: "a:2", rowData: { kind: "even-marker", runningTotal: 2, value: 2 } }, + { rowIdentifier: "z:1", rowData: { kind: "running", runningTotal: 4, value: 2 } }, + { rowIdentifier: "z:2", rowData: { kind: "even-marker", runningTotal: 4, value: 2 } }, + ]); + }); + + test("lFoldTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + registerLFoldAuditTrigger(lFoldTable, "lfold_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + + expect(await readBoolean(lFoldTable.isInitialized())).toBe(false); + expect((await readMapTriggerAuditRows()).filter((row) => row.event === "lfold_change")).toEqual([]); + expect(await readRows(lFoldTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + }); + + test("timeFoldTable init emits rows and enqueues future reductions", async () => { + const { fromTable, groupedTable, timeFoldTable } = createTimeFoldTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("b1", expr(`'{"team":"beta","value":4}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + + expect(await readBoolean(timeFoldTable.isInitialized())).toBe(true); + const alphaRows = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ + rowIdentifier: String(Reflect.get(row, "rowidentifier") ?? Reflect.get(row, "rowIdentifier")), + rowData: row.rowdata, + })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "a1:1", rowData: { runningTotal: 2, value: 2, timestamp: null } }, + { rowIdentifier: "a2:1", rowData: { runningTotal: 3, value: 3, timestamp: null } }, + ]); + + const queuedRows = await readTimeFoldQueueRows(); + expect(queuedRows).toEqual([ + { rowIdentifier: "a1", groupKey: "alpha", stateAfter: 2, rowData: { team: "alpha", value: 2 } }, + { rowIdentifier: "a2", groupKey: "alpha", stateAfter: 3, rowData: { team: "alpha", value: 3 } }, + { rowIdentifier: "b1", groupKey: "beta", stateAfter: 4, rowData: { team: "beta", value: 4 } }, + ]); + }); + + test("timeFoldTable updates and deletes keep queue rows in sync", async () => { + const { fromTable, groupedTable, timeFoldTable } = createTimeFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + registerTimeFoldAuditTrigger(timeFoldTable, "timefold_change"); + + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":4}'::jsonb`))); + + const queueAfterUpdate = await readTimeFoldQueueRows(); + expect(queueAfterUpdate).toEqual([ + { rowIdentifier: "a1", groupKey: "alpha", stateAfter: 4, rowData: { team: "alpha", value: 4 } }, + ]); + + const auditRows = (await readMapTriggerAuditRows()).filter((row) => row.event === "timefold_change"); + expect(auditRows.map((row) => ({ + rowIdentifier: row.rowIdentifier, + oldRowData: row.oldRowData, + newRowData: row.newRowData, + }))).toEqual([ + { + rowIdentifier: "a1:1", + oldRowData: null, + newRowData: { runningTotal: 1, value: 1, timestamp: null }, + }, + { + rowIdentifier: "a1:1", + oldRowData: { runningTotal: 1, value: 1, timestamp: null }, + newRowData: { runningTotal: 4, value: 4, timestamp: null }, + }, + ]); + + await runStatements(fromTable.deleteRow("a1")); + const rowsAfterDelete = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(rowsAfterDelete).toEqual([]); + + const queueAfterDelete = await sql>` + SELECT COUNT(*)::int AS "count" + FROM "BulldozerTimeFoldQueue" + `; + const queueCountRow = queueAfterDelete[0]; + expect(queueCountRow.count).toBe(0); + }); + + test("timeFoldTable stays no-op while uninitialized", async () => { + const { fromTable, groupedTable, timeFoldTable } = createTimeFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + registerTimeFoldAuditTrigger(timeFoldTable, "timefold_uninitialized"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":7}'::jsonb`))); + + expect(await readBoolean(timeFoldTable.isInitialized())).toBe(false); + expect((await readMapTriggerAuditRows()).filter((row) => row.event === "timefold_uninitialized")).toEqual([]); + expect(await readRows(timeFoldTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + expect(await readTimeFoldQueueRows()).toEqual([]); + }); + + test("timeFoldTable reruns immediately when reducer timestamp is already due", async () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-timefold-immediate" }); + const groupedTable = declareGroupByTable({ + tableId: "users-timefold-immediate-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const timeFoldTable = declareTimeFoldTable({ + tableId: "users-timefold-immediate-folded", + fromTable: groupedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + CASE + WHEN "timestamp" IS NULL THEN 1 + ELSE 2 + END AS "newState", + jsonb_build_array( + jsonb_build_object( + 'phase', + CASE + WHEN "timestamp" IS NULL THEN 'initial' + ELSE 'rerun' + END, + 'value', (("oldRowData"->>'value')::int), + 'timestamp', + CASE + WHEN "timestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("timestamp") + END + ) + ) AS "newRowsData", + CASE + WHEN "timestamp" IS NULL THEN (now() - interval '1 minute') + ELSE NULL::timestamptz + END AS "nextTimestamp" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + + const alphaRows = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows).toHaveLength(2); + expect(alphaRows.map((row) => ({ + rowIdentifier: row.rowidentifier, + rowData: row.rowdata, + }))).toEqual([ + { + rowIdentifier: "a1:1", + rowData: { phase: "initial", value: 5, timestamp: null }, + }, + { + rowIdentifier: "a1:2", + rowData: expect.objectContaining({ phase: "rerun", value: 5 }), + }, + ]); + const rerunRow = alphaRows[1]; + expect(Reflect.get(rerunRow.rowdata as object, "timestamp")).not.toBeNull(); + expect(await readTimeFoldQueueRows()).toEqual([]); + }); + + test("timeFoldTable does not enqueue when reducer returns null nextTimestamp", async () => { + const fromTable = declareStoredTable<{ value: number, team: string }>({ tableId: "users-timefold-no-queue" }); + const groupedTable = declareGroupByTable({ + tableId: "users-timefold-no-queue-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const timeFoldTable = declareTimeFoldTable({ + tableId: "users-timefold-no-queue-folded", + fromTable: groupedTable, + initialState: expr(`'0'::jsonb`), + reducer: mapper(` + ("oldState") AS "newState", + jsonb_build_array( + jsonb_build_object( + 'value', (("oldRowData"->>'value')::int), + 'timestamp', 'null'::jsonb + ) + ) AS "newRowsData", + NULL::timestamptz AS "nextTimestamp" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":9}'::jsonb`))); + + const alphaRows = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "a1:1", rowData: { value: 9, timestamp: null } }, + ]); + expect(await readTimeFoldQueueRows()).toEqual([]); + }); + + test("timeFoldTable moving rows across groups replaces queued group entry", async () => { + const { fromTable, groupedTable, timeFoldTable } = createTimeFoldTable(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(timeFoldTable.init()); + + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a1", expr(`'{"team":null,"value":7}'::jsonb`))); + + const queueRows = await readTimeFoldQueueRows(); + expect(queueRows).toHaveLength(1); + const queueRow = queueRows[0]; + expect(queueRow.rowIdentifier).toBe("a1"); + expect(queueRow.groupKey).toBe(null); + expect(queueRow.rowData).toEqual({ team: null, value: 7 }); + expect(queueRow.stateAfter).toBeGreaterThan(0); + + const alphaRows = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows).toEqual([]); + const nullGroupRows = await readRows(timeFoldTable.listRowsInGroup({ + groupKey: expr(`'null'::jsonb`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(nullGroupRows).toHaveLength(1); + const nullGroupRow = nullGroupRows[0]; + expect(nullGroupRow.rowidentifier).toBe("a1:1"); + expect(nullGroupRow.rowdata).toMatchObject({ value: 7, timestamp: null }); + }); + + test("leftJoinTable init backfills matches and unmatched left rows per group", async () => { + const { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable } = createLeftJoinedTable(); + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u4", expr(`'{"team":"alpha","value":7}'::jsonb`))); + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":1,"label":"silver"}'::jsonb`))); + await runStatements(joinTable.setRow("r2", expr(`'{"team":"alpha","threshold":5,"label":"gold"}'::jsonb`))); + await runStatements(joinTable.setRow("r3", expr(`'{"team":"beta","threshold":2,"label":"vip"}'::jsonb`))); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + + expect(await readBoolean(leftJoinedTable.isInitialized())).toBe(true); + const groups = await readRows(leftJoinedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["alpha", "beta"]); + + const alphaRows = await readRows(leftJoinedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { + rowIdentifier: `["u1", "r2"]`, + rowData: { + leftRowData: { team: "alpha", value: 5 }, + rightRowData: { team: "alpha", threshold: 5, label: "gold" }, + }, + }, + { + rowIdentifier: `["u2", "r1"]`, + rowData: { + leftRowData: { team: "alpha", value: 1 }, + rightRowData: { team: "alpha", threshold: 1, label: "silver" }, + }, + }, + { + rowIdentifier: `["u4", null]`, + rowData: { + leftRowData: { team: "alpha", value: 7 }, + rightRowData: null, + }, + }, + ]); + + const betaRows = await readRows(leftJoinedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(betaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { + rowIdentifier: `["u3", "r3"]`, + rowData: { + leftRowData: { team: "beta", value: 2 }, + rightRowData: { team: "beta", threshold: 2, label: "vip" }, + }, + }, + ]); + }); + + test("leftJoinTable matches null join keys with IS NOT DISTINCT FROM semantics", async () => { + const fromTable = declareStoredTable<{ value: number | null, team: string | null }>({ tableId: "left-join-null-users" }); + const joinTable = declareStoredTable<{ threshold: number | null, team: string | null, label: string }>({ tableId: "left-join-null-rules" }); + const groupedFromTable = declareGroupByTable({ + tableId: "left-join-null-users-by-team", + fromTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const groupedJoinTable = declareGroupByTable({ + tableId: "left-join-null-rules-by-team", + fromTable: joinTable, + groupBy: mapper(`"rowData"->'team' AS "groupKey"`), + }); + const leftJoinedTable = declareLeftJoinTable({ + tableId: "left-join-null-users-rules", + leftTable: groupedFromTable, + rightTable: groupedJoinTable, + leftJoinKey: mapper(`"rowData"->'value' AS "joinKey"`), + rightJoinKey: mapper(`"rowData"->'threshold' AS "joinKey"`), + }); + + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + await runStatements(fromTable.setRow("u-null", expr(`'{"team":"alpha","value":null}'::jsonb`))); + await runStatements(fromTable.setRow("u-num", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(joinTable.setRow("r-null", expr(`'{"team":"alpha","threshold":null,"label":"null-match"}'::jsonb`))); + await runStatements(joinTable.setRow("r-num", expr(`'{"team":"alpha","threshold":3,"label":"num-match"}'::jsonb`))); + + const alphaRows = await readRows(leftJoinedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { + rowIdentifier: `["u-null", "r-null"]`, + rowData: { + leftRowData: { team: "alpha", value: null }, + rightRowData: { team: "alpha", threshold: null, label: "null-match" }, + }, + }, + { + rowIdentifier: `["u-num", "r-num"]`, + rowData: { + leftRowData: { team: "alpha", value: 3 }, + rightRowData: { team: "alpha", threshold: 3, label: "num-match" }, + }, + }, + ]); + }); + + test("leftJoinTable recomputes touched groups when either input table changes", async () => { + const { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable } = createLeftJoinedTable(); + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":2,"label":"silver"}'::jsonb`))); + await runStatements(joinTable.setRow("r2", expr(`'{"team":"alpha","threshold":4,"label":"gold"}'::jsonb`))); + await runStatements(joinTable.setRow("rb1", expr(`'{"team":"beta","threshold":3,"label":"beta-rule"}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":4}'::jsonb`))); + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":6,"label":"silver"}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":5}'::jsonb`))); + await runStatements(joinTable.deleteRow("rb1")); + await runStatements(fromTable.deleteRow("u3")); + await runStatements(fromTable.deleteRow("u2")); + + const groups = await readRows(leftJoinedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey)).toEqual(["beta"]); + + const betaRows = await readRows(leftJoinedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(betaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { + rowIdentifier: `["u1", null]`, + rowData: { + leftRowData: { team: "beta", value: 5 }, + rightRowData: null, + }, + }, + ]); + }); + + test("leftJoinTable listRowsInGroup is deterministically ordered by rowIdentifier", async () => { + const { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable } = createLeftJoinedTable(); + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(joinTable.setRow("r2", expr(`'{"team":"alpha","threshold":5,"label":"rule-2"}'::jsonb`))); + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":5,"label":"rule-1"}'::jsonb`))); + + const alphaRows = await readRows(leftJoinedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual([ + `["u1", "r1"]`, + `["u1", "r2"]`, + `["u2", "r1"]`, + `["u2", "r2"]`, + ]); + }); + + test("sortTable bulk init respects descending comparator", async () => { + const { fromTable, groupedTable, sortedTable } = createDescendingSortedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + + const alphaRows = await readRows(sortedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["a3", "a2", "a1"]); + }); + + test("limitTable honors source comparator for top-N", async () => { + const { fromTable, groupedTable, sortedTable, limitedTable } = createDescendingLimitedTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(limitedTable.init()); + + const alphaRows = await readRows(limitedTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["a3", "a2"]); + }); + + test("lFoldTable read order matches source comparator", async () => { + const { fromTable, groupedTable, sortedTable, lFoldTable } = createDescendingLFoldTable(); + await runStatements(fromTable.init()); + await runStatements(fromTable.setRow("a1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("a2", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("a3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(groupedTable.init()); + await runStatements(sortedTable.init()); + await runStatements(lFoldTable.init()); + + const alphaRows = await readRows(lFoldTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => row.rowidentifier)).toEqual(["a3:1", "a2:1", "a1:1"]); + }); + + test("leftJoinTable trigger stream reconstructs exact final table state", async () => { + const { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable } = createLeftJoinedTable(); + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + await runStatements(leftJoinedTable.init()); + registerLeftJoinAuditTrigger(leftJoinedTable, "left_join_change"); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"beta","value":7}'::jsonb`))); + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":3,"label":"silver"}'::jsonb`))); + await runStatements(joinTable.setRow("r2", expr(`'{"team":"alpha","threshold":5,"label":"gold"}'::jsonb`))); + await runStatements(joinTable.setRow("r3", expr(`'{"team":"beta","threshold":6,"label":"beta"}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"beta","value":8}'::jsonb`))); + await runStatements(joinTable.deleteRow("r2")); + await runStatements(fromTable.deleteRow("u3")); + + const auditRows = (await readMapTriggerAuditRows()).filter((row) => row.event === "left_join_change"); + const reconstructed = new Map(); + for (const row of auditRows) { + const groupKey = row.groupKey as string | null; + const rowIdentifier = String(row.rowIdentifier); + const key = `${groupKey ?? "__NULL__"}:${rowIdentifier}`; + if (row.newRowData == null) { + reconstructed.delete(key); + } else { + reconstructed.set(key, { groupKey, rowIdentifier, rowData: row.newRowData }); + } + } + + const actualRows = (await readRows(leftJoinedTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).map((row) => ({ + groupKey: row.groupkey as string | null, + rowIdentifier: String(row.rowidentifier), + rowData: row.rowdata, + })); + const reconstructedRows = [...reconstructed.values()]; + const sortRows = (rows: Array<{ groupKey: string | null, rowIdentifier: string, rowData: unknown }>) => rows + .sort((a, b) => stringCompare( + `${a.groupKey ?? "__NULL__"}:${a.rowIdentifier}:${JSON.stringify(a.rowData)}`, + `${b.groupKey ?? "__NULL__"}:${b.rowIdentifier}:${JSON.stringify(b.rowData)}`, + )); + expect(sortRows(reconstructedRows)).toEqual(sortRows(actualRows)); + }); + + test("leftJoinTable stays no-op while uninitialized", async () => { + const { fromTable, joinTable, groupedFromTable, groupedJoinTable, leftJoinedTable } = createLeftJoinedTable(); + await runStatements(fromTable.init()); + await runStatements(joinTable.init()); + await runStatements(groupedFromTable.init()); + await runStatements(groupedJoinTable.init()); + registerLeftJoinAuditTrigger(leftJoinedTable, "left_join_change"); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(joinTable.setRow("r1", expr(`'{"team":"alpha","threshold":2,"label":"silver"}'::jsonb`))); + + expect(await readBoolean(leftJoinedTable.isInitialized())).toBe(false); + expect((await readMapTriggerAuditRows()).filter((row) => row.event === "left_join_change")).toEqual([]); + expect(await readRows(leftJoinedTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }))).toEqual([]); + }); + + test("flatMap -> map -> groupBy composition stays consistent across updates", async () => { + const { fromTable, groupedTable, flatMappedTable, mappedAfterFlatMap, groupedByKind } = createFlatMapMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(flatMappedTable.init()); + await runStatements(mappedAfterFlatMap.init()); + await runStatements(groupedByKind.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":-1}'::jsonb`))); + + const groups = await readRows(groupedByKind.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["base", "double"]); + + const baseRows = await readRows(groupedByKind.listRowsInGroup({ + groupKey: expr(`to_jsonb('base'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(baseRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2:1:1", rowData: { team: "beta", kind: "base", mappedValuePlusOne: 103 } }, + ]); + + const doubleRows = await readRows(groupedByKind.listRowsInGroup({ + groupKey: expr(`to_jsonb('double'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(doubleRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2:2:1", rowData: { team: "beta", kind: "double", mappedValuePlusOne: 5 } }, + ]); + }); + + test("stacked map tables propagate updates across multiple mapping layers", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":7}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"alpha","value":4}'::jsonb`))); + + const groupsAfterMove = await readRows(mappedTableLevel2.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["alpha"]); + + const alphaRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(a.rowIdentifier, b.rowIdentifier))).toEqual([ + { rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } }, + { rowIdentifier: "u2:1:1", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } }, + ]); + + await runStatements(fromTable.deleteRow("u1")); + const alphaRowsAfterDelete = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsAfterDelete.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2:1:1", rowData: { team: "alpha", valueScaled: 28, bucket: "low" } }, + ]); + }); + + test("stacked map tables handle special row identifiers and null group transitions", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + + const specialIdentifier = "user/one:two space"; + await runStatements(fromTable.setRow(specialIdentifier, expr(`'{"team":null,"value":3}'::jsonb`))); + + const nullGroupRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`'null'::jsonb`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(nullGroupRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: `${specialIdentifier}:1:1`, rowData: { team: null, valueScaled: 26, bucket: "low" } }, + ]); + + await runStatements(fromTable.setRow(specialIdentifier, expr(`'{"team":"alpha","value":3}'::jsonb`))); + const groupsAfterMove = await readRows(mappedTableLevel2.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsAfterMove.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + + test("stacked map tables backfill correctly with staggered initialization order", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2 } = createStackedMappedTables(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":2}'::jsonb`))); + + await runStatements(mappedTableLevel1.init()); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"alpha","value":3}'::jsonb`))); + + await runStatements(mappedTableLevel2.init()); + const allRowsAfterInit = await readRows(mappedTableLevel2.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allRowsAfterInit.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "alpha", rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 22, bucket: "low" } }, + { groupKey: "alpha", rowIdentifier: "u3:1:1", rowData: { team: "alpha", valueScaled: 26, bucket: "low" } }, + { groupKey: "beta", rowIdentifier: "u2:1:1", rowData: { team: "beta", valueScaled: 24, bucket: "low" } }, + ]); + + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + const betaRows = await readRows(mappedTableLevel2.listRowsInGroup({ + groupKey: expr(`to_jsonb('beta'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(betaRows.map((row) => ({ rowIdentifier: row.rowidentifier, rowData: row.rowdata }))).toEqual([ + { rowIdentifier: "u2:1:1", rowData: { team: "beta", valueScaled: 60, bucket: "high" } }, + ]); + }); + + test("groupBy over a stacked map table stays consistent on mapped key transitions", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":2}'::jsonb`))); + + const initialGroups = await readRows(groupedByBucketTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(initialGroups.map((row) => row.groupkey).sort(stringCompare)).toEqual(["high", "low"]); + + const lowRows = await readRows(groupedByBucketTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('low'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(lowRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1:1:1", "u3:1:1"]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":30}'::jsonb`))); + await runStatements(fromTable.deleteRow("u3")); + + const finalGroups = await readRows(groupedByBucketTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(finalGroups.map((row) => row.groupkey)).toEqual(["high"]); + + const highRows = await readRows(groupedByBucketTable.listRowsInGroup({ + groupKey: expr(`to_jsonb('high'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(highRows.map((row) => row.rowidentifier).sort(stringCompare)).toEqual(["u1:1:1", "u2:1:1"]); + }); + + test("composed trigger fanout works for stacked map and downstream groupBy tables", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + mappedTableLevel2.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerMapTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("map_level_2_change"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + groupedByBucketTable.registerRowChangeTrigger((changesTable) => [ + sqlStatement` + INSERT INTO "BulldozerGroupTriggerAudit" ( + "event", + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + ) + SELECT + ${expr(sqlStringLiteral("bucket_group_change"))}, + "groupKey", + "rowIdentifier", + "oldRowData", + "newRowData" + FROM ${changesTable} + `, + ]); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":30}'::jsonb`))); + await runStatements(fromTable.deleteRow("u1")); + + expect(await readMapTriggerAuditRows()).toEqual([ + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1:1:1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + }, + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1:1:1", + oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + newRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + }, + { + event: "map_level_2_change", + groupKey: "alpha", + rowIdentifier: "u1:1:1", + oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + newRowData: null, + }, + ]); + expect(await readGroupTriggerAuditRows()).toEqual([ + { + event: "bucket_group_change", + groupKey: "low", + rowIdentifier: "u1:1:1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + }, + { + event: "bucket_group_change", + groupKey: "low", + rowIdentifier: "u1:1:1", + oldRowData: { team: "alpha", valueScaled: 22, bucket: "low" }, + newRowData: null, + }, + { + event: "bucket_group_change", + groupKey: "high", + rowIdentifier: "u1:1:1", + oldRowData: null, + newRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + }, + { + event: "bucket_group_change", + groupKey: "high", + rowIdentifier: "u1:1:1", + oldRowData: { team: "alpha", valueScaled: 80, bucket: "high" }, + newRowData: null, + }, + ]); + }); + + test("deep pipeline delete and re-init restores exact source truth", async () => { + const { fromTable, groupedTable, mappedTableLevel1, mappedTableLevel2, groupedByBucketTable } = createGroupMapGroupPipeline(); + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":1}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":20}'::jsonb`))); + await runStatements(fromTable.setRow("u3", expr(`'{"team":"gamma","value":2}'::jsonb`))); + + await runStatements(groupedByBucketTable.delete()); + await runStatements(mappedTableLevel2.delete()); + await runStatements(mappedTableLevel1.delete()); + + expect(await readBoolean(mappedTableLevel1.isInitialized())).toBe(false); + expect(await readBoolean(mappedTableLevel2.isInitialized())).toBe(false); + expect(await readBoolean(groupedByBucketTable.isInitialized())).toBe(false); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":5}'::jsonb`))); + await runStatements(fromTable.deleteRow("u2")); + await runStatements(fromTable.setRow("u4", expr(`'{"team":"delta","value":0}'::jsonb`))); + + await runStatements(mappedTableLevel1.init()); + await runStatements(mappedTableLevel2.init()); + await runStatements(groupedByBucketTable.init()); + + const allBucketRows = await readRows(groupedByBucketTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(allBucketRows.map((row) => ({ groupKey: row.groupkey, rowIdentifier: row.rowidentifier, rowData: row.rowdata })).sort((a, b) => stringCompare(`${a.groupKey}:${a.rowIdentifier}`, `${b.groupKey}:${b.rowIdentifier}`))).toEqual([ + { groupKey: "high", rowIdentifier: "u1:1:1", rowData: { team: "alpha", valueScaled: 30, bucket: "high" } }, + { groupKey: "low", rowIdentifier: "u3:1:1", rowData: { team: "gamma", valueScaled: 24, bucket: "low" } }, + { groupKey: "low", rowIdentifier: "u4:1:1", rowData: { team: "delta", valueScaled: 20, bucket: "low" } }, + ]); + }); + + test("parallel map tables on the same grouped source stay isolated", async () => { + const { fromTable, groupedTable } = createGroupedTable(); + const mapTableA = declareMapTable({ + tableId: "users-map-a", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + (("rowData"->>'value')::int + 100) AS "mappedValueA" + `), + }); + const mapTableB = declareMapTable({ + tableId: "users-map-b", + fromTable: groupedTable, + mapper: mapper(` + ("rowData"->'team') AS "team", + ((("rowData"->>'value')::int) * -1) AS "mappedValueB" + `), + }); + + await runStatements(fromTable.init()); + await runStatements(groupedTable.init()); + await runStatements(mapTableA.init()); + await runStatements(mapTableB.init()); + + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":3}'::jsonb`))); + await runStatements(fromTable.setRow("u2", expr(`'{"team":"beta","value":4}'::jsonb`))); + await runStatements(fromTable.setRow("u1", expr(`'{"team":"alpha","value":6}'::jsonb`))); + await runStatements(fromTable.deleteRow("u2")); + + const alphaRowsA = await readRows(mapTableA.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsA.map((row) => row.rowdata)).toEqual([{ team: "alpha", mappedValueA: 106 }]); + + const alphaRowsB = await readRows(mapTableB.listRowsInGroup({ + groupKey: expr(`to_jsonb('alpha'::text)`), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(alphaRowsB.map((row) => row.rowdata)).toEqual([{ team: "alpha", mappedValueB: -6 }]); + + const groupsA = await readRows(mapTableA.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + const groupsB = await readRows(mapTableB.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })); + expect(groupsA.map((row) => row.groupkey)).toEqual(["alpha"]); + expect(groupsB.map((row) => row.groupkey)).toEqual(["alpha"]); + }); + + test("toExecutableSqlTransaction handles empty statements", async () => { + await runStatements([]); + }); + + test("toQueryableSqlQuery returns executable SQL", async () => { + const table = declareStoredTable<{ value: number }>({ tableId: "users" }); + await runStatements(table.init()); + await runStatements(table.setRow("alpha", expr(`'{"value":1}'::jsonb`))); + + const query = table.listRowsInGroup({ + groupKey: expr("'null'::jsonb"), + start: expr("'null'::jsonb"), + end: expr("'null'::jsonb"), + startInclusive: true, + endInclusive: true, + }); + const rows = await sql.unsafe(toQueryableSqlQuery(query)); + expect(rows).toHaveLength(1); + expect(rows[0].rowdata).toEqual({ value: 1 }); + }); +}); diff --git a/apps/backend/src/lib/bulldozer/db/index.ts b/apps/backend/src/lib/bulldozer/db/index.ts new file mode 100644 index 0000000000..bbed03bb42 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/index.ts @@ -0,0 +1,106 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; + +import { BULLDOZER_SORT_HELPERS_SQL } from "./bulldozer-sort-helpers-sql"; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlQuery, SqlStatement, TableId } from "./utilities"; +import { quoteSqlIdentifier } from "./utilities"; + +// ====== Table implementations ====== +// IMPORTANT NOTE: For every new table implementation, we should also add tests (unit, fuzzing, & perf; including an entry in the "hundreds of thousands" perf test), an example in the example schema, and support in Bulldozer Studio. + +export type Table = { + tableId: TableId, + inputTables: Table[], + debugArgs: Record, + + // Query groups and rows + listGroups(options: { start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + /** + * Rows queried across all groups may include `groupKey`; rows queried for a specific `groupKey` + * may omit it. + */ + listRowsInGroup(options: { groupKey?: SqlExpression, start: SqlExpression | "start", end: SqlExpression | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery>, + + // Sorting and grouping + compareGroupKeys(a: SqlExpression, b: SqlExpression): SqlExpression, + compareSortKeys(a: SqlExpression, b: SqlExpression): SqlExpression, + + // Lifecycle/migration methods + /** Called when the table should be created on the storage engine. */ + init(): SqlStatement[], + /** Called when the table should be deleted from the storage engine. */ + delete(): SqlStatement[], + isInitialized(): SqlExpression, + + // Internal methods, used only by table constructors to create relationships between them + /** + * @param trigger A SQL statement that can reference the changes table with columns `groupKey: GK`, `rowIdentifier: RowIdentifier`, `oldRowSortKey: SK | null`, `newRowSortKey: SK | null`, `oldRowData: RowData | null`, `newRowData: RowData | null`. Note that this trigger should be a no-op if the table that created this trigger is not initialized. + */ + registerRowChangeTrigger(trigger: (changesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => SqlStatement[]): { deregister: () => void }, +}; + +export { declareConcatTable } from "./tables/concat-table"; +export { declareFilterTable } from "./tables/filter-table"; +export { declareFlatMapTable } from "./tables/flat-map-table"; +export { declareGroupByTable } from "./tables/group-by-table"; +export { declareLFoldTable } from "./tables/l-fold-table"; +export { declareLeftJoinTable } from "./tables/left-join-table"; +export { declareLimitTable } from "./tables/limit-table"; +export { declareMapTable } from "./tables/map-table"; +export { declareSortTable } from "./tables/sort-table"; +export { declareStoredTable } from "./tables/stored-table"; +export { declareTimeFoldTable } from "./tables/time-fold-table"; + +const BULLDOZER_LOCK_ID = 7857391; // random number to avoid conflicts with other applications + +export function toQueryableSqlQuery(query: SqlQuery): string { + return query.sql; +} +export function toExecutableSqlStatements(statements: SqlStatement[]): string { + const requiresSortHelpers = statements.some((statement) => statement.sql.includes("pg_temp.bulldozer_sort_")); + const requiresSequentialExecutor = requiresSortHelpers || statements.some((statement) => statement.requiresSequentialExecution === true); + if (!requiresSequentialExecutor) { + return deindent` + WITH __dummy_statement_1__ AS (SELECT 1), + ${statements.map(statement => deindent` + ${quoteSqlIdentifier(statement.outputName ?? `unnamed_statement_${generateSecureRandomString().slice(0, 8)}`).sql} AS ( + ${statement.sql} + ), + `).join("\n")} + __dummy_statement_2__ AS (SELECT 1) + SELECT 1; + `; + } + + const executableStatements = statements.map((statement) => { + if (statement.outputName == null) { + return `${statement.sql};`; + } + return deindent` + CREATE TEMP TABLE ${quoteSqlIdentifier(statement.outputName).sql} ON COMMIT DROP AS + WITH "__statement_output" AS ( + ${statement.sql} + ) + SELECT * FROM "__statement_output"; + `; + }).join("\n\n"); + return deindent` + ${requiresSortHelpers ? BULLDOZER_SORT_HELPERS_SQL : ""} + + ${executableStatements} + `; +} +export function toExecutableSqlTransaction(statements: SqlStatement[], options: { statementTimeout?: string } = {}): string { + return deindent` + BEGIN; + + SET LOCAL jit = off; + ${options.statementTimeout ? `SET LOCAL statement_timeout = '${options.statementTimeout}';` : ""} + + SELECT pg_advisory_xact_lock(${BULLDOZER_LOCK_ID}); + + ${toExecutableSqlStatements(statements)} + + COMMIT; + `; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/concat-table.ts b/apps/backend/src/lib/bulldozer/db/tables/concat-table.ts new file mode 100644 index 0000000000..1ae7393530 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/concat-table.ts @@ -0,0 +1,215 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + getTablePathSegments, + quoteSqlIdentifier, + quoteSqlJsonbLiteral, + quoteSqlStringLiteral, + singleNullSortKeyRangePredicate, + sqlArray, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString, +} from "../utilities"; + +export function declareConcatTable< + GK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + tables: Table[], +}): Table { + const tables = [...options.tables]; + const firstTable = tables[0] ?? (() => { + throw new StackAssertionError("declareConcatTable requires at least one input table", { tableId: options.tableId }); + })(); + const referenceCompareGroupKeysSql = firstTable.compareGroupKeys(sqlExpression`$1`, sqlExpression`$2`).sql; + for (const table of tables) { + const compareGroupKeysSql = table.compareGroupKeys(sqlExpression`$1`, sqlExpression`$2`).sql; + if (compareGroupKeysSql !== referenceCompareGroupKeysSql) { + throw new StackAssertionError("declareConcatTable requires group-comparator-compatible input tables", { + tableId: options.tableId, + tableDebugId: tableIdToDebugString(table.tableId), + }); + } + } + const triggers = new Map) => SqlStatement[]>(); + const rawExpression = (sql: string): SqlExpression => ({ type: "expression", sql }); + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const createConcatenatedRowIdentifierSql = (tableIndex: number, rowIdentifierSql: string) => + `${quoteSqlStringLiteral(`${tableIndex}:`).sql} || ${rowIdentifierSql}`; + const getInputInitializedSql = (table: Table) => table.isInitialized().sql; + const getUnionedListGroupsSql = (queryOptions: Parameters[0]) => { + return tables + .map((table) => deindent` + SELECT "sourceGroups"."groupkey" AS "groupKey" + FROM (${table.listGroups(queryOptions).sql}) AS "sourceGroups" + WHERE ${getInputInitializedSql(table)} + `) + .join("\nUNION ALL\n"); + }; + const getUnionedListRowsSql = (queryOptions: Parameters[0] & { allGroups: boolean }) => { + return tables.map((table, tableIndex) => { + if (queryOptions.allGroups) { + return deindent` + SELECT + "sourceRows"."groupkey" AS "groupKey", + ${createConcatenatedRowIdentifierSql(tableIndex, `"sourceRows"."rowidentifier"`)} AS "rowIdentifier", + 'null'::jsonb AS "rowSortKey", + "sourceRows"."rowdata" AS "rowData" + FROM (${table.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).sql}) AS "sourceRows" + WHERE ${getInputInitializedSql(table)} + `; + } + const groupKey = queryOptions.groupKey ?? (() => { + throw new StackAssertionError("declareConcatTable specific-group query requires a group key"); + })(); + return deindent` + SELECT + ${createConcatenatedRowIdentifierSql(tableIndex, `"sourceRows"."rowidentifier"`)} AS "rowIdentifier", + 'null'::jsonb AS "rowSortKey", + "sourceRows"."rowdata" AS "rowData" + FROM (${table.listRowsInGroup({ + groupKey, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).sql}) AS "sourceRows" + WHERE ${getInputInitializedSql(table)} + `; + }).join("\nUNION ALL\n"); + }; + const createInputTriggerStatements = ( + table: Table, + tableIndex: number, + changesTable: SqlExpression<{ __brand: "$SQL_Table" }>, + ) => { + const concatChangesTableName = `concat_changes_${generateSecureRandomString()}`; + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + ${rawExpression(createConcatenatedRowIdentifierSql(tableIndex, `"changes"."rowIdentifier"`))} AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData" + FROM ${changesTable} AS "changes" + WHERE ${isInitializedExpression} + AND ${rawExpression(getInputInitializedSql(table))} + `.toStatement(concatChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(concatChangesTableName))), + ]; + }; + let inputTriggerRegistrations: Array<{ deregister: () => void }> = []; + const ensureInputTriggerRegistrations = () => { + if (inputTriggerRegistrations.length > 0) return; + inputTriggerRegistrations = tables.map((table, tableIndex) => { + return table.registerRowChangeTrigger((changesTable) => { + return createInputTriggerStatements(table, tableIndex, changesTable); + }); + }); + }; + const deregisterInputTriggers = () => { + for (const registration of inputTriggerRegistrations) { + registration.deregister(); + } + inputTriggerRegistrations = []; + }; + + return { + tableId: options.tableId, + inputTables: tables, + debugArgs: { + operator: "concat", + tableId: tableIdToDebugString(options.tableId), + inputTableIds: tables.map((table) => tableIdToDebugString(table.tableId)), + }, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT DISTINCT "concatGroups"."groupKey" AS groupKey + FROM (${rawExpression(getUnionedListGroupsSql({ start, end, startInclusive, endInclusive }))}) AS "concatGroups" + WHERE ${isInitializedExpression} + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey != null ? sqlQuery` + SELECT + "concatRows"."rowIdentifier" AS rowIdentifier, + "concatRows"."rowSortKey" AS rowSortKey, + "concatRows"."rowData" AS rowData + FROM (${rawExpression(getUnionedListRowsSql({ + groupKey, + start, + end, + startInclusive, + endInclusive, + allGroups: false, + }))}) AS "concatRows" + WHERE ${isInitializedExpression} + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + SELECT + "concatRows"."groupKey" AS groupKey, + "concatRows"."rowIdentifier" AS rowIdentifier, + "concatRows"."rowSortKey" AS rowSortKey, + "concatRows"."rowData" AS rowData + FROM (${rawExpression(getUnionedListRowsSql({ + start, + end, + startInclusive, + endInclusive, + allGroups: true, + }))}) AS "concatRows" + WHERE ${isInitializedExpression} + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + compareGroupKeys: firstTable.compareGroupKeys, + compareSortKeys: () => sqlExpression`0`, + init: () => { + ensureInputTriggerRegistrations(); + return [sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb) + `]; + }, + delete: () => { + deregisterInputTriggers(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/filter-table.ts b/apps/backend/src/lib/bulldozer/db/tables/filter-table.ts new file mode 100644 index 0000000000..0c29bab72a --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/filter-table.ts @@ -0,0 +1,82 @@ +import { pick } from "@stackframe/stack-shared/dist/utils/objects"; +import type { Table } from ".."; +import type { Json, RowData, SqlPredicate, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + getTablePathSegments, + quoteSqlJsonbLiteral, + sqlArray, + sqlExpression, + sqlMapper, + sqlStatement, + tableIdToDebugString +} from "../utilities"; +import { declareFlatMapTable } from "./flat-map-table"; + +export function declareFilterTable< + GK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + filter: SqlPredicate, +}): Table { + const nestedFlatMapTable = declareFlatMapTable({ + tableId: { tableType: "internal", internalId: "filter", parent: options.tableId }, + fromTable: options.fromTable, + mapper: sqlMapper` + CASE + WHEN ${options.filter} + THEN jsonb_build_array("rowData") + ELSE '[]'::jsonb + END AS "rows" + `, + }); + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "filter", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + filterSql: options.filter.sql, + }, + init: () => [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb) + `, + ...nestedFlatMapTable.init(), + ], + delete: () => [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `], + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `, + ...pick(nestedFlatMapTable, [ + "compareGroupKeys", + "compareSortKeys", + "listGroups", + "listRowsInGroup", + "registerRowChangeTrigger", + ]), + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/flat-map-table.ts b/apps/backend/src/lib/bulldozer/db/tables/flat-map-table.ts new file mode 100644 index 0000000000..4d0cc78281 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/flat-map-table.ts @@ -0,0 +1,409 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + singleNullSortKeyRangePredicate, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; + +export function declareFlatMapTable< + GK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + mapper: SqlMapper, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const createExpandedRowIdentifier = (sourceRowIdentifier: SqlExpression, flatIndex: SqlExpression): SqlExpression => + sqlExpression`(${sourceRowIdentifier} || ':' || (${flatIndex}::text))`; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`; + const oldFlatRowsTableName = `old_flat_rows_${generateSecureRandomString()}`; + const newFlatRowsTableName = `new_flat_rows_${generateSecureRandomString()}`; + const flatMapChangesTableName = `flat_map_changes_${generateSecureRandomString()}`; + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "sourceRowIdentifier", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow", + "oldMapped"."rows" AS "oldMappedRows", + "newMapped"."rows" AS "newMappedRows" + FROM ${fromChangesTable} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."rows" AS "rows" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "oldMapped" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') + LEFT JOIN LATERAL ( + SELECT "mapped"."rows" AS "rows" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "newMapped" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') + WHERE ${isInitializedExpression} + `.toStatement(mappedChangesTableName), + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"changes"."sourceRowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN "changes"."hasOldRow" THEN ( + CASE + WHEN jsonb_typeof("changes"."oldMappedRows") = 'array' THEN "changes"."oldMappedRows" + ELSE '[]'::jsonb + END + ) + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(oldFlatRowsTableName), + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"changes"."sourceRowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN "changes"."hasNewRow" THEN ( + CASE + WHEN jsonb_typeof("changes"."newMappedRows") = 'array' THEN "changes"."newMappedRows" + ELSE '[]'::jsonb + END + ) + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(newFlatRowsTableName), + sqlStatement` + WITH "distinctGroups" AS ( + SELECT DISTINCT "groupKey" + FROM ${quoteSqlIdentifier(newFlatRowsTableName)} + ) + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT + ${getGroupKeyPath(sqlExpression`"distinctGroups"."groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM "distinctGroups" + + UNION ALL + + SELECT + ${getGroupRowsPath(sqlExpression`"distinctGroups"."groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM "distinctGroups" + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "changes" + WHERE "target"."keyPath" = ${getGroupRowPath( + sqlExpression`"changes"."groupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "rowData") + FROM ${quoteSqlIdentifier(newFlatRowsTableName)} + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "changes" + WHERE "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRow" + WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "deletingRow" + WHERE "deletingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey" + AND "groupRow"."keyPath" = ${getGroupRowPath( + sqlExpression`"deletingRow"."groupKey"`, + sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`, + )}::jsonb[] + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(newFlatRowsTableName)} AS "insertingRow" + WHERE "insertingRow"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey" + ) + `, + sqlQuery` + SELECT + COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey", + COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData", + CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData" + FROM ${quoteSqlIdentifier(oldFlatRowsTableName)} AS "oldRows" + FULL OUTER JOIN ${quoteSqlIdentifier(newFlatRowsTableName)} AS "newRows" + ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey" + AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier" + WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData" + `.toStatement(flatMapChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(flatMapChangesTableName))), + ]; + }; + let fromTableTriggerRegistration: null | { deregister: () => void } = null; + const ensureFromTableTriggerRegistration = () => { + if (fromTableTriggerRegistration != null) return; + fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + return createFromTableTriggerStatements(fromChangesTable); + }); + }; + const deregisterFromTableTrigger = () => { + fromTableTriggerRegistration?.deregister(); + fromTableTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "flatMap", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + mapperSql: options.mapper.sql, + }, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => { + ensureFromTableTriggerRegistration(); + const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`; + const fromRowsTableName = `from_rows_${generateSecureRandomString()}`; + const mappedRowsTableName = `mapped_rows_${generateSecureRandomString()}`; + const flatRowsTableName = `flat_rows_${generateSecureRandomString()}`; + + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.fromTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromGroupsTableName), + sqlQuery` + SELECT + "groups"."groupkey" AS "groupKey", + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupkey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + `.toStatement(fromRowsTableName), + sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "sourceRowIdentifier", + "mapped"."rows" AS "mappedRows" + FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows" + LEFT JOIN LATERAL ( + SELECT "mapped"."rows" AS "rows" + FROM ( + SELECT ${options.mapper} + FROM ( + SELECT + "rows"."rowIdentifier" AS "rowIdentifier", + "rows"."rowData" AS "rowData" + ) AS "mapperInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(mappedRowsTableName), + sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"rows"."sourceRowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(mappedRowsTableName)} AS "rows" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("rows"."mappedRows") = 'array' THEN "rows"."mappedRows" + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(flatRowsTableName), + sqlStatement` + WITH "distinctGroups" AS ( + SELECT DISTINCT "groupKey" + FROM ${quoteSqlIdentifier(flatRowsTableName)} + ) + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT + ${getGroupKeyPath(sqlExpression`"distinctGroups"."groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM "distinctGroups" + + UNION ALL + + SELECT + ${getGroupRowsPath(sqlExpression`"distinctGroups"."groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM "distinctGroups" + + UNION ALL + + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[] AS "keyPath", + jsonb_build_object('rowData', "rowData") AS "value" + FROM ${quoteSqlIdentifier(flatRowsTableName)} + ) AS "insertRows" + `, + ]; + }, + delete: () => { + deregisterFromTableTrigger(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/group-by-table.ts b/apps/backend/src/lib/bulldozer/db/tables/group-by-table.ts new file mode 100644 index 0000000000..6868bbfa62 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/group-by-table.ts @@ -0,0 +1,354 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + singleNullSortKeyRangePredicate, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; + +export function declareGroupByTable< + GK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + groupBy: SqlMapper<{ rowIdentifier: RowIdentifier, rowData: RD }, { groupKey: GK }>, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const compareGroupKeys = (a: SqlExpression, b: SqlExpression) => sqlExpression` + ((${a}) > (${b}))::int - ((${a}) < (${b}))::int + `; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`; + const groupedChangesTableName = `grouped_changes_${generateSecureRandomString()}`; + + return [ + sqlQuery` + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow", + "oldGroup"."groupKey" AS "oldGroupKey", + "newGroup"."groupKey" AS "newGroupKey" + FROM ${fromChangesTable} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "oldGroup" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "newGroup" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') + WHERE ${isInitializedExpression} + AND ( + NOT ( + "changes"."oldRowData" IS NOT NULL + AND jsonb_typeof("changes"."oldRowData") = 'object' + AND "changes"."newRowData" IS NOT NULL + AND jsonb_typeof("changes"."newRowData") = 'object' + ) + OR "changes"."oldRowData" IS DISTINCT FROM "changes"."newRowData" + OR "oldGroup"."groupKey" IS DISTINCT FROM "newGroup"."groupKey" + ) + `.toStatement(mappedChangesTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"newGroupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"newGroupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "target"."keyPath" = ${getGroupRowPath( + sqlExpression`"changes"."oldGroupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"newGroupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "newRowData") + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(mappedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRow" + WHERE "groupRow"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."oldGroupKey"`)}::jsonb[] + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "deletingRow" + WHERE "deletingRow"."hasOldRow" + AND "deletingRow"."oldGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey" + AND "groupRow"."keyPath" = ${getGroupRowPath( + sqlExpression`"deletingRow"."oldGroupKey"`, + sqlExpression`to_jsonb("deletingRow"."rowIdentifier"::text)`, + )}::jsonb[] + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(mappedChangesTableName)} AS "insertingRow" + WHERE "insertingRow"."hasNewRow" + AND "insertingRow"."newGroupKey" IS NOT DISTINCT FROM "changes"."oldGroupKey" + ) + `, + sqlQuery` + SELECT + "oldGroupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + "oldRowData" AS "oldRowData", + CASE + WHEN "hasNewRow" AND "oldGroupKey" IS NOT DISTINCT FROM "newGroupKey" THEN "newRowData" + ELSE 'null'::jsonb + END AS "newRowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasOldRow" + UNION ALL + SELECT + "newGroupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + 'null'::jsonb AS "oldRowData", + "newRowData" AS "newRowData" + FROM ${quoteSqlIdentifier(mappedChangesTableName)} + WHERE "hasNewRow" + AND (NOT "hasOldRow" OR "oldGroupKey" IS DISTINCT FROM "newGroupKey") + `.toStatement(groupedChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(groupedChangesTableName))), + ]; + }; + let fromTableTriggerRegistration: null | { deregister: () => void } = null; + const ensureFromTableTriggerRegistration = () => { + if (fromTableTriggerRegistration != null) return; + fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + return createFromTableTriggerStatements(fromChangesTable); + }); + }; + const deregisterFromTableTrigger = () => { + fromTableTriggerRegistration?.deregister(); + fromTableTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "groupBy", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + groupBySql: options.groupBy.sql, + }, + compareGroupKeys, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => { + ensureFromTableTriggerRegistration(); + const fromTableAllRowsTableName = `from_table_all_rows_${generateSecureRandomString()}`; + const fromTableRowsWithGroupKeyTableName = `from_table_rows_with_group_key_${generateSecureRandomString()}`; + + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.fromTable.listRowsInGroup({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromTableAllRowsTableName), + sqlQuery` + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData", + "mapped"."groupKey" AS "groupKey" + FROM ${quoteSqlIdentifier(fromTableAllRowsTableName)} AS "rows" + LEFT JOIN LATERAL ( + SELECT "mapped"."groupKey" + FROM ( + SELECT ${options.groupBy} + FROM ( + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + ) AS "groupByInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(fromTableRowsWithGroupKeyTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + UNION + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[] AS "keyPath", + jsonb_build_object('rowData', "rowData") AS "value" + FROM ${quoteSqlIdentifier(fromTableRowsWithGroupKeyTableName)} + ) AS "insertRows" + `, + ]; + }, + delete: () => { + deregisterFromTableTrigger(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + ORDER BY "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] ASC + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + -- Get all rows from all groups + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/l-fold-table.ts b/apps/backend/src/lib/bulldozer/db/tables/l-fold-table.ts new file mode 100644 index 0000000000..89239ada69 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/l-fold-table.ts @@ -0,0 +1,803 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + getTablePathSegments, + quoteSqlIdentifier, + quoteSqlJsonbLiteral, + sqlArray, + sqlExpression, + sqlMapper, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; +import { declareSortTable } from "./sort-table"; + +/** + * Materialized left-fold table. + * + * For each group, this table folds source rows in sort order (ties are deterministically broken by source + * `rowIdentifier`) and stores the reducer output as flattened rows. + * + * Reducer contract: + * - Input: `{ oldState, oldRowData }` + * - Output: `{ newState, newRowsData }` + * - `newState` is carried into the next row in the same group. + * - `newRowsData` is flattened into output rows for the current source row. + * + * Output details: + * - Output row sort key is the source row sort key. + * - Output row identifier is `${sourceRowIdentifier}:${index}` (1-based index in `newRowsData`). + * + * Incremental behavior and performance: + * - An internal sort table (treap-backed via `declareSortTable`) maintains source ordering. + * - On source changes, LFold recomputes only the affected suffix in each touched group. + * - If the first row changes, the full group is recomputed; if the last row changes, only the tail is. + * - Per touched group complexity is roughly `O(log n + affectedRows * reducerCost + affectedOutputRows)`. + */ +export function declareLFoldTable< + GK extends Json, + SK extends Json, + OldRD extends RowData, + NewRD extends RowData, + S extends Json, +>(options: { + tableId: TableId, + fromTable: Table, + initialState: SqlExpression, + reducer: SqlMapper<{ oldState: S, oldRowData: OldRD }, { newState: S, newRowsData: NewRD[] }>, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const fromTableOperator = ( + "operator" in options.fromTable.debugArgs + && typeof options.fromTable.debugArgs.operator === "string" + ) ? options.fromTable.debugArgs.operator : null; + const reusesInputSortTable = fromTableOperator === "sort"; + const sourceSortTableId: TableId = reusesInputSortTable ? options.fromTable.tableId : { + tableType: "internal", + internalId: "lfold-source-sort", + parent: options.tableId, + }; + const sourceSortTable: Table = reusesInputSortTable ? options.fromTable : declareSortTable({ + tableId: sourceSortTableId, + fromTable: options.fromTable, + getSortKey: sqlMapper` + "oldSortKey" AS "newSortKey" + `, + compareSortKeys: options.fromTable.compareSortKeys, + }); + const groupsPath = getStorageEnginePath(options.tableId, ["groups"]); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const getGroupStatesPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "states"]); + const getGroupStatePath = (groupKey: SqlExpression, sourceRowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "states", sourceRowIdentifier]); + const getSourceSortGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(sourceSortTableId, ["groups", groupKey, "rows"]); + const getSourceSortGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(sourceSortTableId, ["groups", groupKey, "rows", rowIdentifier]); + const createExpandedRowIdentifier = (sourceRowIdentifier: SqlExpression, flatIndex: SqlExpression): SqlExpression => + sqlExpression`(${sourceRowIdentifier} || ':' || (${flatIndex}::text))`; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const sortRangePredicate = (rowSortKey: SqlExpression, optionsForRange: { + start: SqlExpression | "start", + end: SqlExpression | "end", + startInclusive: boolean, + endInclusive: boolean, + }) => sqlExpression` + ${ + optionsForRange.start === "start" + ? sqlExpression`1 = 1` + : optionsForRange.startInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(rowSortKey, optionsForRange.start)} >= 0` + : sqlExpression`${options.fromTable.compareSortKeys(rowSortKey, optionsForRange.start)} > 0` + } + AND ${ + optionsForRange.end === "end" + ? sqlExpression`1 = 1` + : optionsForRange.endInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(rowSortKey, optionsForRange.end)} <= 0` + : sqlExpression`${options.fromTable.compareSortKeys(rowSortKey, optionsForRange.end)} < 0` + } + `; + + const createSourceSortTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`; + const boundaryCandidatesTableName = `boundary_candidates_${generateSecureRandomString()}`; + const earliestBoundaryCandidatesTableName = `earliest_boundary_candidates_${generateSecureRandomString()}`; + const touchedGroupsTableName = `touched_groups_${generateSecureRandomString()}`; + const currentSourceRowsTableName = `current_source_rows_${generateSecureRandomString()}`; + const affectedSourceRowsTableName = `affected_source_rows_${generateSecureRandomString()}`; + const firstAffectedRowsTableName = `first_affected_rows_${generateSecureRandomString()}`; + const rowsToClearTableName = `rows_to_clear_${generateSecureRandomString()}`; + const oldFoldRowsTableName = `old_fold_rows_${generateSecureRandomString()}`; + const recomputedSourceStatesTableName = `recomputed_source_states_${generateSecureRandomString()}`; + const newFoldRowsTableName = `new_fold_rows_${generateSecureRandomString()}`; + const lfoldChangesTableName = `lfold_changes_${generateSecureRandomString()}`; + + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowSortKey" AS "oldRowSortKey", + "changes"."newRowSortKey" AS "newRowSortKey", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + "changes"."hasOldRow" AS "hasOldRow", + "changes"."hasNewRow" AS "hasNewRow", + ( + ("changes"."hasOldRow" OR "changes"."hasNewRow") + AND ( + NOT ("changes"."hasOldRow" AND "changes"."hasNewRow") + OR "changes"."oldRowSortKey" IS DISTINCT FROM "changes"."newRowSortKey" + OR "changes"."oldRowData" IS DISTINCT FROM "changes"."newRowData" + ) + ) AS "shouldRecompute" + FROM ( + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowSortKey" AS "oldRowSortKey", + "changes"."newRowSortKey" AS "newRowSortKey", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow" + FROM ${fromChangesTable} AS "changes" + ) AS "changes" + WHERE ${isInitializedExpression} + `.toStatement(normalizedChangesTableName), + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."oldRowSortKey" AS "boundarySortKey", + "changes"."rowIdentifier" AS "boundaryRowIdentifier" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."shouldRecompute" AND "changes"."hasOldRow" + + UNION ALL + + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."newRowSortKey" AS "boundarySortKey", + "changes"."rowIdentifier" AS "boundaryRowIdentifier" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."shouldRecompute" AND "changes"."hasNewRow" + `.toStatement(boundaryCandidatesTableName), + sqlQuery` + SELECT + "candidate"."groupKey" AS "groupKey", + "candidate"."boundarySortKey" AS "boundarySortKey", + "candidate"."boundaryRowIdentifier" AS "boundaryRowIdentifier" + FROM ${quoteSqlIdentifier(boundaryCandidatesTableName)} AS "candidate" + WHERE NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(boundaryCandidatesTableName)} AS "other" + WHERE "other"."groupKey" IS NOT DISTINCT FROM "candidate"."groupKey" + AND ( + ${options.fromTable.compareSortKeys(sqlExpression`"other"."boundarySortKey"`, sqlExpression`"candidate"."boundarySortKey"`)} < 0 + OR ( + ${options.fromTable.compareSortKeys(sqlExpression`"other"."boundarySortKey"`, sqlExpression`"candidate"."boundarySortKey"`)} = 0 + AND "other"."boundaryRowIdentifier" < "candidate"."boundaryRowIdentifier" + ) + ) + ) + `.toStatement(earliestBoundaryCandidatesTableName), + sqlQuery` + SELECT DISTINCT "groupKey" + FROM ${quoteSqlIdentifier(earliestBoundaryCandidatesTableName)} + `.toStatement(touchedGroupsTableName), + sqlQuery` + SELECT + "groups"."groupKey" AS "groupKey", + ("sourceRows"."keyPath"[cardinality("sourceRows"."keyPath")] #>> '{}') AS "rowIdentifier", + "sourceRows"."value"->'rowSortKey' AS "rowSortKey", + "sourceRows"."value"->'rowData' AS "rowData", + "sourceRows"."value"->>'prevRowIdentifier' AS "prevRowIdentifier", + "sourceRows"."value"->>'nextRowIdentifier' AS "nextRowIdentifier" + FROM ${quoteSqlIdentifier(touchedGroupsTableName)} AS "groups" + INNER JOIN "BulldozerStorageEngine" AS "sourceRows" + ON "sourceRows"."keyPathParent" = ${getSourceSortGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + `.toStatement(currentSourceRowsTableName), + sqlQuery` + SELECT + "sourceRows"."groupKey" AS "groupKey", + "sourceRows"."rowIdentifier" AS "rowIdentifier", + "sourceRows"."rowSortKey" AS "rowSortKey", + "sourceRows"."rowData" AS "rowData", + "sourceRows"."prevRowIdentifier" AS "prevRowIdentifier", + "sourceRows"."nextRowIdentifier" AS "nextRowIdentifier" + FROM ${quoteSqlIdentifier(currentSourceRowsTableName)} AS "sourceRows" + INNER JOIN ${quoteSqlIdentifier(earliestBoundaryCandidatesTableName)} AS "boundary" + ON "boundary"."groupKey" IS NOT DISTINCT FROM "sourceRows"."groupKey" + WHERE + ${options.fromTable.compareSortKeys(sqlExpression`"sourceRows"."rowSortKey"`, sqlExpression`"boundary"."boundarySortKey"`)} > 0 + OR ( + ${options.fromTable.compareSortKeys(sqlExpression`"sourceRows"."rowSortKey"`, sqlExpression`"boundary"."boundarySortKey"`)} = 0 + AND "sourceRows"."rowIdentifier" >= "boundary"."boundaryRowIdentifier" + ) + `.toStatement(affectedSourceRowsTableName), + sqlQuery` + SELECT + "affectedRows"."groupKey" AS "groupKey", + "affectedRows"."rowIdentifier" AS "rowIdentifier", + "affectedRows"."rowSortKey" AS "rowSortKey", + "affectedRows"."rowData" AS "rowData", + "affectedRows"."prevRowIdentifier" AS "prevRowIdentifier", + "affectedRows"."nextRowIdentifier" AS "nextRowIdentifier" + FROM ${quoteSqlIdentifier(affectedSourceRowsTableName)} AS "affectedRows" + LEFT JOIN ${quoteSqlIdentifier(affectedSourceRowsTableName)} AS "affectedPrevRows" + ON "affectedPrevRows"."groupKey" IS NOT DISTINCT FROM "affectedRows"."groupKey" + AND "affectedPrevRows"."rowIdentifier" = "affectedRows"."prevRowIdentifier" + WHERE "affectedPrevRows"."rowIdentifier" IS NULL + `.toStatement(firstAffectedRowsTableName), + sqlQuery` + SELECT DISTINCT + "rows"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "rowIdentifier" + FROM ${quoteSqlIdentifier(affectedSourceRowsTableName)} AS "rows" + + UNION + + SELECT DISTINCT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."shouldRecompute" AND "changes"."hasOldRow" + `.toStatement(rowsToClearTableName), + sqlQuery` + SELECT + "rowsToClear"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"rowsToClear"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "stateRows"."value"->'rowSortKey' AS "rowSortKey", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(rowsToClearTableName)} AS "rowsToClear" + INNER JOIN "BulldozerStorageEngine" AS "stateRows" + ON "stateRows"."keyPath" = ${getGroupStatePath( + sqlExpression`"rowsToClear"."groupKey"`, + sqlExpression`to_jsonb("rowsToClear"."rowIdentifier"::text)`, + )}::jsonb[] + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("stateRows"."value"->'emittedRowsData') = 'array' THEN "stateRows"."value"->'emittedRowsData' + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(oldFoldRowsTableName), + sqlQuery` + WITH RECURSIVE "recomputedRows" AS ( + SELECT + "firstRows"."groupKey" AS "groupKey", + "firstRows"."rowIdentifier" AS "rowIdentifier", + "firstRows"."rowSortKey" AS "rowSortKey", + "firstRows"."rowData" AS "rowData", + "firstRows"."nextRowIdentifier" AS "nextRowIdentifier", + "seed"."oldState" AS "oldState", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData" + FROM ${quoteSqlIdentifier(firstAffectedRowsTableName)} AS "firstRows" + LEFT JOIN "BulldozerStorageEngine" AS "prevStateRows" + ON "firstRows"."prevRowIdentifier" IS NOT NULL + AND "prevStateRows"."keyPath" = ${getGroupStatePath( + sqlExpression`"firstRows"."groupKey"`, + sqlExpression`to_jsonb("firstRows"."prevRowIdentifier"::text)`, + )}::jsonb[] + CROSS JOIN LATERAL ( + SELECT + CASE + WHEN "firstRows"."prevRowIdentifier" IS NULL THEN to_jsonb(${options.initialState}) + ELSE COALESCE("prevStateRows"."value"->'stateAfter', to_jsonb(${options.initialState})) + END AS "oldState" + ) AS "seed" + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + "seed"."oldState" AS "oldState", + "firstRows"."rowData" AS "oldRowData" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + + UNION ALL + + SELECT + "recomputedRows"."groupKey" AS "groupKey", + ("nextSourceRows"."keyPath"[cardinality("nextSourceRows"."keyPath")] #>> '{}') AS "rowIdentifier", + "nextSourceRows"."value"->'rowSortKey' AS "rowSortKey", + "nextSourceRows"."value"->'rowData' AS "rowData", + "nextSourceRows"."value"->>'nextRowIdentifier' AS "nextRowIdentifier", + "recomputedRows"."newState" AS "oldState", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData" + FROM "recomputedRows" + INNER JOIN "BulldozerStorageEngine" AS "nextSourceRows" + ON "recomputedRows"."nextRowIdentifier" IS NOT NULL + AND "nextSourceRows"."keyPath" = ${getSourceSortGroupRowPath( + sqlExpression`"recomputedRows"."groupKey"`, + sqlExpression`to_jsonb("recomputedRows"."nextRowIdentifier"::text)`, + )}::jsonb[] + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + "recomputedRows"."newState" AS "oldState", + "nextSourceRows"."value"->'rowData' AS "oldRowData" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + ) + SELECT + "groupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + "rowSortKey" AS "rowSortKey", + "newState" AS "stateAfter", + "newRowsData" AS "emittedRowsData" + FROM "recomputedRows" + `.toStatement(recomputedSourceStatesTableName), + sqlQuery` + SELECT + "states"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"states"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "states"."rowSortKey" AS "rowSortKey", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} AS "states" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("states"."emittedRowsData") = 'array' THEN "states"."emittedRowsData" + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(newFoldRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupStatesPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "targetRows" + USING ${quoteSqlIdentifier(oldFoldRowsTableName)} AS "oldRows" + WHERE "targetRows"."keyPath" = ${getGroupRowPath( + sqlExpression`"oldRows"."groupKey"`, + sqlExpression`to_jsonb("oldRows"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "targetStates" + USING ${quoteSqlIdentifier(rowsToClearTableName)} AS "rowsToClear" + WHERE "targetStates"."keyPath" = ${getGroupStatePath( + sqlExpression`"rowsToClear"."groupKey"`, + sqlExpression`to_jsonb("rowsToClear"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupStatePath( + sqlExpression`"states"."groupKey"`, + sqlExpression`to_jsonb("states"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowSortKey', "states"."rowSortKey", + 'stateAfter', "states"."stateAfter", + 'emittedRowsData', "states"."emittedRowsData" + ) + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} AS "states" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"rows"."groupKey"`, + sqlExpression`to_jsonb("rows"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowSortKey', "rows"."rowSortKey", + 'rowData', "rows"."rowData" + ) + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} AS "rows" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPaths" + USING ${quoteSqlIdentifier(touchedGroupsTableName)} AS "groups" + WHERE "staleGroupPaths"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[], + ${getGroupStatesPath(sqlExpression`"groups"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "stateRows" + WHERE "stateRows"."keyPathParent" = ${getGroupStatesPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "foldRows" + WHERE "foldRows"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + ) + `, + sqlQuery` + SELECT + COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey", + COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier", + CASE WHEN "oldRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowSortKey" END AS "oldRowSortKey", + CASE WHEN "newRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowSortKey" END AS "newRowSortKey", + CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData", + CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData" + FROM ${quoteSqlIdentifier(oldFoldRowsTableName)} AS "oldRows" + FULL OUTER JOIN ${quoteSqlIdentifier(newFoldRowsTableName)} AS "newRows" + ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey" + AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier" + WHERE "oldRows"."rowSortKey" IS DISTINCT FROM "newRows"."rowSortKey" + OR "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData" + `.toStatement(lfoldChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(lfoldChangesTableName))), + ]; + }; + let sourceSortTriggerRegistration: null | { deregister: () => void } = null; + const ensureSourceSortTriggerRegistration = () => { + if (sourceSortTriggerRegistration != null) return; + sourceSortTriggerRegistration = sourceSortTable.registerRowChangeTrigger((fromChangesTable) => { + return createSourceSortTriggerStatements(fromChangesTable); + }); + }; + const deregisterSourceSortTrigger = () => { + sourceSortTriggerRegistration?.deregister(); + sourceSortTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "lfold", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + initialStateSql: options.initialState.sql, + reducerSql: options.reducer.sql, + }, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: options.fromTable.compareSortKeys, + init: () => { + ensureSourceSortTriggerRegistration(); + const firstSourceRowsTableName = `first_source_rows_${generateSecureRandomString()}`; + const recomputedSourceStatesTableName = `recomputed_source_states_${generateSecureRandomString()}`; + const newFoldRowsTableName = `new_fold_rows_${generateSecureRandomString()}`; + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${groupsPath}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + ...(reusesInputSortTable ? [] : sourceSortTable.init()), + sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS "groupKey", + ("sourceRows"."keyPath"[cardinality("sourceRows"."keyPath")] #>> '{}') AS "rowIdentifier", + "sourceRows"."value"->'rowSortKey' AS "rowSortKey", + "sourceRows"."value"->'rowData' AS "rowData", + "sourceRows"."value"->>'prevRowIdentifier' AS "prevRowIdentifier", + "sourceRows"."value"->>'nextRowIdentifier' AS "nextRowIdentifier" + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "sourceRows" + ON "sourceRows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(sourceSortTableId, ["groups"])}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND "sourceRows"."value"->>'prevRowIdentifier' IS NULL + `.toStatement(firstSourceRowsTableName), + sqlQuery` + WITH RECURSIVE "recomputedRows" AS ( + SELECT + "firstRows"."groupKey" AS "groupKey", + "firstRows"."rowIdentifier" AS "rowIdentifier", + "firstRows"."rowSortKey" AS "rowSortKey", + "firstRows"."rowData" AS "rowData", + "firstRows"."nextRowIdentifier" AS "nextRowIdentifier", + to_jsonb(${options.initialState}) AS "oldState", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData" + FROM ${quoteSqlIdentifier(firstSourceRowsTableName)} AS "firstRows" + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + to_jsonb(${options.initialState}) AS "oldState", + "firstRows"."rowData" AS "oldRowData" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + + UNION ALL + + SELECT + "recomputedRows"."groupKey" AS "groupKey", + ("nextSourceRows"."keyPath"[cardinality("nextSourceRows"."keyPath")] #>> '{}') AS "rowIdentifier", + "nextSourceRows"."value"->'rowSortKey' AS "rowSortKey", + "nextSourceRows"."value"->'rowData' AS "rowData", + "nextSourceRows"."value"->>'nextRowIdentifier' AS "nextRowIdentifier", + "recomputedRows"."newState" AS "oldState", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData" + FROM "recomputedRows" + INNER JOIN "BulldozerStorageEngine" AS "nextSourceRows" + ON "recomputedRows"."nextRowIdentifier" IS NOT NULL + AND "nextSourceRows"."keyPath" = ${getSourceSortGroupRowPath( + sqlExpression`"recomputedRows"."groupKey"`, + sqlExpression`to_jsonb("recomputedRows"."nextRowIdentifier"::text)`, + )}::jsonb[] + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + "recomputedRows"."newState" AS "oldState", + "nextSourceRows"."value"->'rowData' AS "oldRowData" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + ) + SELECT + "groupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + "rowSortKey" AS "rowSortKey", + "newState" AS "stateAfter", + "newRowsData" AS "emittedRowsData" + FROM "recomputedRows" + `.toStatement(recomputedSourceStatesTableName), + sqlQuery` + SELECT + "states"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"states"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "states"."rowSortKey" AS "rowSortKey", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} AS "states" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("states"."emittedRowsData") = 'array' THEN "states"."emittedRowsData" + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(newFoldRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupStatesPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupStatePath( + sqlExpression`"states"."groupKey"`, + sqlExpression`to_jsonb("states"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowSortKey', "states"."rowSortKey", + 'stateAfter', "states"."stateAfter", + 'emittedRowsData', "states"."emittedRowsData" + ) + FROM ${quoteSqlIdentifier(recomputedSourceStatesTableName)} AS "states" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"rows"."groupKey"`, + sqlExpression`to_jsonb("rows"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowSortKey', "rows"."rowSortKey", + 'rowData', "rows"."rowData" + ) + FROM ${quoteSqlIdentifier(newFoldRowsTableName)} AS "rows" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + ]; + }, + delete: () => { + deregisterSourceSortTrigger(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + WITH "orderedSourceRows" AS ( + SELECT + row_number() OVER () AS "rowOrder", + "sourceRows"."rowidentifier" AS "rowIdentifier" + FROM ( + ${sourceSortTable.listRowsInGroup({ + groupKey, + start, + end, + startInclusive, + endInclusive, + })} + ) AS "sourceRows" + ) + SELECT + ${createExpandedRowIdentifier( + sqlExpression`"orderedSourceRows"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS rowIdentifier, + "stateRows"."value"->'rowSortKey' AS rowSortKey, + "flatRow"."rowData" AS rowData + FROM "orderedSourceRows" + INNER JOIN "BulldozerStorageEngine" AS "stateRows" + ON "stateRows"."keyPath" = ${getGroupStatePath( + groupKey, + sqlExpression`to_jsonb("orderedSourceRows"."rowIdentifier"::text)`, + )}::jsonb[] + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("stateRows"."value"->'emittedRowsData') = 'array' THEN "stateRows"."value"->'emittedRowsData' + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + ORDER BY "orderedSourceRows"."rowOrder" ASC, "flatRow"."flatIndex" ASC + ` : sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + "rows"."value"->'rowSortKey' AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${sortRangePredicate(sqlExpression`"rows"."value"->'rowSortKey'`, { start, end, startInclusive, endInclusive })} + ORDER BY groupKey ASC, rowSortKey ASC, rowIdentifier ASC + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/left-join-table.ts b/apps/backend/src/lib/bulldozer/db/tables/left-join-table.ts new file mode 100644 index 0000000000..004401476f --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/left-join-table.ts @@ -0,0 +1,523 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + singleNullSortKeyRangePredicate, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; + +export function declareLeftJoinTable< + GK extends Json, + JK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + leftTable: Table, + rightTable: Table, + leftJoinKey: SqlMapper<{ rowIdentifier: RowIdentifier, rowData: OldRD }, { joinKey: JK }>, + rightJoinKey: SqlMapper<{ rowIdentifier: RowIdentifier, rowData: NewRD }, { joinKey: JK }>, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const rawExpression = (sql: string): SqlExpression => ({ type: "expression", sql }); + const groupsPath = getStorageEnginePath(options.tableId, ["groups"]); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const createJoinedRowIdentifier = ( + leftRowIdentifier: SqlExpression, + rightRowIdentifier: SqlExpression, + ): SqlExpression => sqlExpression` + ( + jsonb_build_array( + to_jsonb(${leftRowIdentifier}::text), + CASE + WHEN ${rightRowIdentifier} IS NULL THEN 'null'::jsonb + ELSE to_jsonb(${rightRowIdentifier}::text) + END + ) #>> '{}' + ) + `; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const createJoinedRowsStatement = (optionsForStatement: { + leftRowsTableName: string, + rightRowsTableName: string, + outputTableName: string, + }): SqlStatement => sqlQuery` + SELECT DISTINCT ON ("joinedRows"."groupKey", "joinedRows"."rowIdentifier") + "joinedRows"."groupKey" AS "groupKey", + "joinedRows"."rowIdentifier" AS "rowIdentifier", + "joinedRows"."rowData" AS "rowData" + FROM ( + SELECT + "leftRows"."groupKey" AS "groupKey", + ${createJoinedRowIdentifier( + sqlExpression`"leftRows"."leftRowIdentifier"`, + sqlExpression`"rightRows"."rightRowIdentifier"`, + )} AS "rowIdentifier", + jsonb_build_object( + 'leftRowData', "leftRows"."leftRowData", + 'rightRowData', "rightRows"."rightRowData" + ) AS "rowData" + FROM ${quoteSqlIdentifier(optionsForStatement.leftRowsTableName)} AS "leftRows" + LEFT JOIN ${quoteSqlIdentifier(optionsForStatement.rightRowsTableName)} AS "rightRows" + ON "rightRows"."groupKey" IS NOT DISTINCT FROM "leftRows"."groupKey" + AND "rightRows"."rightJoinKey" IS NOT DISTINCT FROM "leftRows"."leftJoinKey" + ) AS "joinedRows" + ORDER BY "joinedRows"."groupKey", "joinedRows"."rowIdentifier" + `.toStatement(optionsForStatement.outputTableName); + const createLeftRowsStatement = (optionsForRows: { + groupsTableName: string, + groupKeySql: string, + outputTableName: string, + }): SqlStatement => sqlQuery` + SELECT + ${rawExpression(optionsForRows.groupKeySql)} AS "groupKey", + "rows"."rowidentifier" AS "leftRowIdentifier", + "rows"."rowdata" AS "leftRowData", + "mapped"."joinKey" AS "leftJoinKey" + FROM ${quoteSqlIdentifier(optionsForRows.groupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.leftTable.listRowsInGroup({ + groupKey: rawExpression(optionsForRows.groupKeySql), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + LEFT JOIN LATERAL ( + SELECT "mapped"."joinKey" + FROM ( + SELECT ${options.leftJoinKey} + FROM ( + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + ) AS "joinKeyInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(optionsForRows.outputTableName); + const createRightRowsStatement = (optionsForRows: { + groupsTableName: string, + groupKeySql: string, + outputTableName: string, + }): SqlStatement => sqlQuery` + SELECT + ${rawExpression(optionsForRows.groupKeySql)} AS "groupKey", + "rows"."rowidentifier" AS "rightRowIdentifier", + "rows"."rowdata" AS "rightRowData", + "mapped"."joinKey" AS "rightJoinKey" + FROM ${quoteSqlIdentifier(optionsForRows.groupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.rightTable.listRowsInGroup({ + groupKey: rawExpression(optionsForRows.groupKeySql), + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + LEFT JOIN LATERAL ( + SELECT "mapped"."joinKey" + FROM ( + SELECT ${options.rightJoinKey} + FROM ( + SELECT + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "rowData" + ) AS "joinKeyInput" + ) AS "mapped" + ) AS "mapped" ON true + `.toStatement(optionsForRows.outputTableName); + + const registerInputTrigger = (optionsForTrigger: { + inputTable: Table, + changedSide: "left" | "right", + }) => { + return optionsForTrigger.inputTable.registerRowChangeTrigger((inputChangesTable) => { + const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`; + const affectedGroupsTableName = `affected_groups_${generateSecureRandomString()}`; + const oldLeftJoinRowsTableName = `old_left_join_rows_${generateSecureRandomString()}`; + const oldLeftRowsTableName = `old_left_rows_${generateSecureRandomString()}`; + const oldRightRowsTableName = `old_right_rows_${generateSecureRandomString()}`; + const newLeftRowsTableName = `new_left_rows_${generateSecureRandomString()}`; + const newRightRowsTableName = `new_right_rows_${generateSecureRandomString()}`; + const newLeftJoinRowsTableName = `new_left_join_rows_${generateSecureRandomString()}`; + const leftJoinChangesTableName = `left_join_changes_${generateSecureRandomString()}`; + + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow" + FROM ${inputChangesTable} AS "changes" + WHERE ${isInitializedExpression} + `.toStatement(normalizedChangesTableName), + sqlQuery` + SELECT DISTINCT "changes"."groupKey" AS "groupKey" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" OR "changes"."hasNewRow" + `.toStatement(affectedGroupsTableName), + sqlQuery` + SELECT + "groups"."groupKey" AS "groupKey", + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS "rowIdentifier", + "rows"."value"->'rowData' AS "rowData" + FROM ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + `.toStatement(oldLeftJoinRowsTableName), + createLeftRowsStatement({ + groupsTableName: affectedGroupsTableName, + groupKeySql: `"groups"."groupKey"`, + outputTableName: oldLeftRowsTableName, + }), + createRightRowsStatement({ + groupsTableName: affectedGroupsTableName, + groupKeySql: `"groups"."groupKey"`, + outputTableName: oldRightRowsTableName, + }), + optionsForTrigger.changedSide === "left" ? sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."leftRowIdentifier" AS "leftRowIdentifier", + "rows"."leftRowData" AS "leftRowData", + "rows"."leftJoinKey" AS "leftJoinKey" + FROM ${quoteSqlIdentifier(oldLeftRowsTableName)} AS "rows" + WHERE NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "changes"."groupKey" IS NOT DISTINCT FROM "rows"."groupKey" + AND "changes"."rowIdentifier" = "rows"."leftRowIdentifier" + ) + UNION ALL + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "leftRowIdentifier", + "changes"."newRowData" AS "leftRowData", + "mapped"."joinKey" AS "leftJoinKey" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."joinKey" + FROM ( + SELECT ${options.leftJoinKey} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "joinKeyInput" + ) AS "mapped" + ) AS "mapped" ON true + WHERE "changes"."hasNewRow" + `.toStatement(newLeftRowsTableName) : sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."leftRowIdentifier" AS "leftRowIdentifier", + "rows"."leftRowData" AS "leftRowData", + "rows"."leftJoinKey" AS "leftJoinKey" + FROM ${quoteSqlIdentifier(oldLeftRowsTableName)} AS "rows" + `.toStatement(newLeftRowsTableName), + optionsForTrigger.changedSide === "right" ? sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rightRowIdentifier" AS "rightRowIdentifier", + "rows"."rightRowData" AS "rightRowData", + "rows"."rightJoinKey" AS "rightJoinKey" + FROM ${quoteSqlIdentifier(oldRightRowsTableName)} AS "rows" + WHERE NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" + AND "changes"."groupKey" IS NOT DISTINCT FROM "rows"."groupKey" + AND "changes"."rowIdentifier" = "rows"."rightRowIdentifier" + ) + UNION ALL + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rightRowIdentifier", + "changes"."newRowData" AS "rightRowData", + "mapped"."joinKey" AS "rightJoinKey" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."joinKey" + FROM ( + SELECT ${options.rightJoinKey} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData" + ) AS "joinKeyInput" + ) AS "mapped" + ) AS "mapped" ON true + WHERE "changes"."hasNewRow" + `.toStatement(newRightRowsTableName) : sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rightRowIdentifier" AS "rightRowIdentifier", + "rows"."rightRowData" AS "rightRowData", + "rows"."rightJoinKey" AS "rightJoinKey" + FROM ${quoteSqlIdentifier(oldRightRowsTableName)} AS "rows" + `.toStatement(newRightRowsTableName), + createJoinedRowsStatement({ + leftRowsTableName: newLeftRowsTableName, + rightRowsTableName: newRightRowsTableName, + outputTableName: newLeftJoinRowsTableName, + }), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newLeftJoinRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newLeftJoinRowsTableName)} + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + WHERE "target"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "rowData") + FROM ${quoteSqlIdentifier(newLeftJoinRowsTableName)} + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + WHERE "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(newLeftJoinRowsTableName)} AS "newRows" + WHERE "newRows"."groupKey" IS NOT DISTINCT FROM "groups"."groupKey" + ) + `, + sqlQuery` + SELECT + COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey", + COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData", + CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData" + FROM ${quoteSqlIdentifier(oldLeftJoinRowsTableName)} AS "oldRows" + FULL OUTER JOIN ${quoteSqlIdentifier(newLeftJoinRowsTableName)} AS "newRows" + ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey" + AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier" + WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData" + `.toStatement(leftJoinChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(leftJoinChangesTableName))), + ]; + }); + }; + let inputTriggerRegistrations: Array<{ deregister: () => void }> = []; + const ensureInputTriggerRegistrations = () => { + if (inputTriggerRegistrations.length > 0) return; + inputTriggerRegistrations = [ + registerInputTrigger({ + inputTable: options.leftTable, + changedSide: "left", + }), + registerInputTrigger({ + inputTable: options.rightTable, + changedSide: "right", + }), + ]; + }; + const deregisterInputTriggers = () => { + for (const registration of inputTriggerRegistrations) { + registration.deregister(); + } + inputTriggerRegistrations = []; + }; + + return { + tableId: options.tableId, + inputTables: [options.leftTable, options.rightTable], + debugArgs: { + operator: "leftJoin", + tableId: tableIdToDebugString(options.tableId), + leftTableId: tableIdToDebugString(options.leftTable.tableId), + rightTableId: tableIdToDebugString(options.rightTable.tableId), + leftJoinKeySql: options.leftJoinKey.sql, + rightJoinKeySql: options.rightJoinKey.sql, + }, + compareGroupKeys: options.leftTable.compareGroupKeys, + compareSortKeys: () => sqlExpression`0`, + init: () => { + ensureInputTriggerRegistrations(); + const leftGroupsTableName = `left_groups_${generateSecureRandomString()}`; + const leftRowsTableName = `left_rows_${generateSecureRandomString()}`; + const rightRowsTableName = `right_rows_${generateSecureRandomString()}`; + const leftJoinedRowsTableName = `left_joined_rows_${generateSecureRandomString()}`; + + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${groupsPath}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.leftTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(leftGroupsTableName), + createLeftRowsStatement({ + groupsTableName: leftGroupsTableName, + groupKeySql: `"groups"."groupkey"`, + outputTableName: leftRowsTableName, + }), + createRightRowsStatement({ + groupsTableName: leftGroupsTableName, + groupKeySql: `"groups"."groupkey"`, + outputTableName: rightRowsTableName, + }), + createJoinedRowsStatement({ + leftRowsTableName, + rightRowsTableName, + outputTableName: leftJoinedRowsTableName, + }), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(leftJoinedRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(leftJoinedRowsTableName)} + UNION + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[] AS "keyPath", + jsonb_build_object('rowData', "rowData") AS "value" + FROM ${quoteSqlIdentifier(leftJoinedRowsTableName)} + ) AS "insertRows" + `, + ]; + }, + delete: () => { + deregisterInputTriggers(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.leftTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.leftTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.leftTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.leftTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey ? sqlQuery` + SELECT + ("row"."keyPath"[cardinality("row"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "row"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "row" + WHERE "row"."keyPathParent" = ${getGroupRowsPath(groupKey)}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ORDER BY rowIdentifier ASC + ` : sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ORDER BY groupKey ASC, rowIdentifier ASC + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/limit-table.ts b/apps/backend/src/lib/bulldozer/db/tables/limit-table.ts new file mode 100644 index 0000000000..4617800973 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/limit-table.ts @@ -0,0 +1,497 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, SqlExpression, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; + +export function declareLimitTable< + GK extends Json, + SK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + limit: SqlExpression, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const normalizedLimit = sqlExpression`GREATEST((${options.limit})::int, 0)`; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + + // TODO: Currently, we recompute the entire limit table when a particular group changes. In the future, we should use an ordered tree to do this incrementally + const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`; + const affectedGroupsTableName = `affected_groups_${generateSecureRandomString()}`; + const oldLimitedRowsTableName = `old_limited_rows_${generateSecureRandomString()}`; + const newLimitedRowsTableName = `new_limited_rows_${generateSecureRandomString()}`; + const limitChangesTableName = `limit_changes_${generateSecureRandomString()}`; + return [ + { + ...sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowSortKey" AS "oldRowSortKey", + "changes"."newRowSortKey" AS "newRowSortKey", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow" + FROM ${fromChangesTable} AS "changes" + WHERE ${isInitializedExpression} + `.toStatement(normalizedChangesTableName), + requiresSequentialExecution: true, + }, + sqlQuery` + SELECT DISTINCT "changes"."groupKey" AS "groupKey" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + WHERE "changes"."hasOldRow" OR "changes"."hasNewRow" + `.toStatement(affectedGroupsTableName), + sqlQuery` + SELECT + "groups"."groupKey" AS "groupKey", + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS "rowIdentifier", + "rows"."value"->'rowSortKey' AS "rowSortKey", + "rows"."value"->'rowData' AS "rowData" + FROM ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPath" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + `.toStatement(oldLimitedRowsTableName), + sqlQuery` + SELECT + "groups"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "rowIdentifier", + "rows"."rowSortKey" AS "rowSortKey", + "rows"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + WITH "sourceRows" AS ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupKey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "sourceRows" + ), + "sortKeyPresence" AS ( + SELECT EXISTS ( + SELECT 1 + FROM "sourceRows" + WHERE "rowsortkey" IS NOT NULL + AND "rowsortkey" <> 'null'::jsonb + ) AS "hasNonNullSortKey" + ) + SELECT + "selectedRows"."rowidentifier" AS "rowIdentifier", + "selectedRows"."rowsortkey" AS "rowSortKey", + "selectedRows"."rowdata" AS "rowData" + FROM ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM "sourceRows" + CROSS JOIN "sortKeyPresence" + WHERE "sortKeyPresence"."hasNonNullSortKey" + LIMIT ${normalizedLimit} + ) AS "selectedRows" + UNION ALL + SELECT + "selectedRows"."rowidentifier" AS "rowIdentifier", + "selectedRows"."rowsortkey" AS "rowSortKey", + "selectedRows"."rowdata" AS "rowData" + FROM ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM "sourceRows" + CROSS JOIN "sortKeyPresence" + WHERE NOT "sortKeyPresence"."hasNonNullSortKey" + ORDER BY "sourceRows"."rowidentifier" ASC + LIMIT ${normalizedLimit} + ) AS "selectedRows" + ) AS "rows" + `.toStatement(newLimitedRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newLimitedRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(newLimitedRowsTableName)} + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "target" + USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + WHERE "target"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowSortKey', "rowSortKey", + 'rowData', "rowData" + ) + FROM ${quoteSqlIdentifier(newLimitedRowsTableName)} + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPath" + USING ${quoteSqlIdentifier(affectedGroupsTableName)} AS "groups" + WHERE "staleGroupPath"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"groups"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"groups"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(newLimitedRowsTableName)} AS "newRows" + WHERE "newRows"."groupKey" IS NOT DISTINCT FROM "groups"."groupKey" + ) + `, + sqlQuery` + SELECT + COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey", + COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier", + CASE WHEN "oldRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowSortKey" END AS "oldRowSortKey", + CASE WHEN "newRows"."rowSortKey" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowSortKey" END AS "newRowSortKey", + CASE WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "oldRows"."rowData" END AS "oldRowData", + CASE WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb ELSE "newRows"."rowData" END AS "newRowData" + FROM ${quoteSqlIdentifier(oldLimitedRowsTableName)} AS "oldRows" + FULL OUTER JOIN ${quoteSqlIdentifier(newLimitedRowsTableName)} AS "newRows" + ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey" + AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier" + WHERE "oldRows"."rowSortKey" IS DISTINCT FROM "newRows"."rowSortKey" + OR "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData" + `.toStatement(limitChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(limitChangesTableName))), + ]; + }; + let fromTableTriggerRegistration: null | { deregister: () => void } = null; + const ensureFromTableTriggerRegistration = () => { + if (fromTableTriggerRegistration != null) return; + fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + return createFromTableTriggerStatements(fromChangesTable); + }); + }; + const deregisterFromTableTrigger = () => { + fromTableTriggerRegistration?.deregister(); + fromTableTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "limit", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + limitSql: options.limit.sql, + }, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: options.fromTable.compareSortKeys, + init: () => { + ensureFromTableTriggerRegistration(); + const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`; + const limitedRowsTableName = `limited_rows_${generateSecureRandomString()}`; + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["groups"])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `, + options.fromTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromGroupsTableName), + sqlQuery` + SELECT + "groups"."groupkey" AS "groupKey", + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowsortkey" AS "rowSortKey", + "rows"."rowdata" AS "rowData" + FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + WITH "sourceRows" AS ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupkey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "sourceRows" + ), + "sortKeyPresence" AS ( + SELECT EXISTS ( + SELECT 1 + FROM "sourceRows" + WHERE "rowsortkey" IS NOT NULL + AND "rowsortkey" <> 'null'::jsonb + ) AS "hasNonNullSortKey" + ) + SELECT + "selectedRows"."rowidentifier" AS "rowidentifier", + "selectedRows"."rowsortkey" AS "rowsortkey", + "selectedRows"."rowdata" AS "rowdata" + FROM ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM "sourceRows" + CROSS JOIN "sortKeyPresence" + WHERE "sortKeyPresence"."hasNonNullSortKey" + LIMIT ${normalizedLimit} + ) AS "selectedRows" + UNION ALL + SELECT + "selectedRows"."rowidentifier" AS "rowidentifier", + "selectedRows"."rowsortkey" AS "rowsortkey", + "selectedRows"."rowdata" AS "rowdata" + FROM ( + SELECT + "sourceRows"."rowidentifier" AS "rowidentifier", + "sourceRows"."rowsortkey" AS "rowsortkey", + "sourceRows"."rowdata" AS "rowdata" + FROM "sourceRows" + CROSS JOIN "sortKeyPresence" + WHERE NOT "sortKeyPresence"."hasNonNullSortKey" + ORDER BY "sourceRows"."rowidentifier" ASC + LIMIT ${normalizedLimit} + ) AS "selectedRows" + ) AS "rows" + `.toStatement(limitedRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(limitedRowsTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(limitedRowsTableName)} + UNION + SELECT + ${getGroupRowPath( + sqlExpression`"groupKey"`, + sqlExpression`to_jsonb("rowIdentifier"::text)`, + )}::jsonb[] AS "keyPath", + jsonb_build_object( + 'rowSortKey', "rowSortKey", + 'rowData', "rowData" + ) AS "value" + FROM ${quoteSqlIdentifier(limitedRowsTableName)} + ) AS "insertRows" + `, + ]; + }, + delete: () => { + deregisterFromTableTrigger(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey + ? sqlQuery` + WITH "limitedRows" AS ( + SELECT + ("row"."keyPath"[cardinality("row"."keyPath")] #>> '{}') AS "rowIdentifier", + "row"."value"->'rowSortKey' AS "rowSortKey", + "row"."value"->'rowData' AS "rowData" + FROM "BulldozerStorageEngine" AS "row" + WHERE "row"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"])}::jsonb[] + ), + "sortKeyPresence" AS ( + SELECT EXISTS ( + SELECT 1 + FROM "limitedRows" + WHERE "rowSortKey" IS NOT NULL + AND "rowSortKey" <> 'null'::jsonb + ) AS "hasNonNullSortKey" + ), + "selectedRows" AS ( + SELECT + "sourceRows"."rowidentifier" AS "rowIdentifier", + "sourceRows"."rowsortkey" AS "rowSortKey", + "sourceRows"."rowdata" AS "rowData", + 0::int AS "branchOrder", + row_number() OVER () AS "rowOrder" + FROM ( + ${options.fromTable.listRowsInGroup({ + groupKey, + start, + end, + startInclusive, + endInclusive, + })} + ) AS "sourceRows" + CROSS JOIN "sortKeyPresence" + WHERE "sortKeyPresence"."hasNonNullSortKey" + AND EXISTS ( + SELECT 1 + FROM "limitedRows" + WHERE "limitedRows"."rowIdentifier" = "sourceRows"."rowidentifier" + ) + + UNION ALL + + SELECT + "limitedRows"."rowIdentifier" AS "rowIdentifier", + "limitedRows"."rowSortKey" AS "rowSortKey", + "limitedRows"."rowData" AS "rowData", + 1::int AS "branchOrder", + row_number() OVER (ORDER BY "limitedRows"."rowIdentifier" ASC) AS "rowOrder" + FROM "limitedRows" + CROSS JOIN "sortKeyPresence" + WHERE NOT "sortKeyPresence"."hasNonNullSortKey" + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"limitedRows"."rowSortKey"`, start)} >= 0` + : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"limitedRows"."rowSortKey"`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"limitedRows"."rowSortKey"`, end)} <= 0` + : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"limitedRows"."rowSortKey"`, end)} < 0` + } + ) + SELECT + "selectedRows"."rowIdentifier" AS rowIdentifier, + "selectedRows"."rowSortKey" AS rowSortKey, + "selectedRows"."rowData" AS rowData + FROM "selectedRows" + ORDER BY "selectedRows"."branchOrder" ASC, "selectedRows"."rowOrder" ASC + ` + : sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + "rows"."value"->'rowSortKey' AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${getStorageEnginePath(options.tableId, ["groups"])}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, start)} >= 0` + : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, end)} <= 0` + : sqlExpression`${options.fromTable.compareSortKeys(sqlExpression`"rows"."value"->'rowSortKey'`, end)} < 0` + } + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/map-table.ts b/apps/backend/src/lib/bulldozer/db/tables/map-table.ts new file mode 100644 index 0000000000..87071a1295 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/map-table.ts @@ -0,0 +1,89 @@ +import { pick } from "@stackframe/stack-shared/dist/utils/objects"; +import type { Table } from ".."; +import type { Json, RowData, SqlMapper, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + getTablePathSegments, + quoteSqlJsonbLiteral, + sqlArray, + sqlExpression, + sqlMapper, + sqlStatement, + tableIdToDebugString +} from "../utilities"; +import { declareFlatMapTable } from "./flat-map-table"; + +export function declareMapTable< + GK extends Json, + OldRD extends RowData, + NewRD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + mapper: SqlMapper, +}): Table { + const nestedFlatMapTable = declareFlatMapTable({ + tableId: { tableType: "internal", internalId: "map", parent: options.tableId }, + fromTable: options.fromTable, + mapper: sqlMapper` + jsonb_build_array( + COALESCE( + ( + SELECT to_jsonb("mapped") + FROM ( + SELECT ${options.mapper} + ) AS "mapped" + ), + 'null'::jsonb + ) + ) AS "rows" + `, + }); + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "map", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + mapperSql: options.mapper.sql, + }, + init: () => [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb) + `, + ...nestedFlatMapTable.init(), + ], + delete: () => [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `], + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `, + ...pick(nestedFlatMapTable, [ + "compareGroupKeys", + "compareSortKeys", + "listGroups", + "listRowsInGroup", + "registerRowChangeTrigger", + ]), + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/sort-table.ts b/apps/backend/src/lib/bulldozer/db/tables/sort-table.ts new file mode 100644 index 0000000000..a7fcb9d802 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/sort-table.ts @@ -0,0 +1,418 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + quoteSqlStringLiteral, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString +} from "../utilities"; + +export function declareSortTable< + GK extends Json, + OldSK extends Json, + NewSK extends Json, + RD extends RowData, +>(options: { + tableId: TableId, + fromTable: Table, + getSortKey: SqlMapper<{ rowIdentifier: RowIdentifier, oldSortKey: OldSK, rowData: RD }, { newSortKey: NewSK }>, + compareSortKeys: (a: SqlExpression, b: SqlExpression) => SqlExpression, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const groupsPath = getStorageEnginePath(options.tableId, ["groups"]); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const getGroupMetadataPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "metadata"]); + const compareSortKeysSqlLiteral = quoteSqlStringLiteral(options.compareSortKeys(sqlExpression`$1`, sqlExpression`$2`).sql); + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const sortRangePredicate = (rowSortKey: SqlExpression, optionsForRange: { + start: SqlExpression | "start", + end: SqlExpression | "end", + startInclusive: boolean, + endInclusive: boolean, + }) => sqlExpression` + ${ + optionsForRange.start === "start" + ? sqlExpression`1 = 1` + : optionsForRange.startInclusive + ? sqlExpression`${options.compareSortKeys(rowSortKey, optionsForRange.start)} >= 0` + : sqlExpression`${options.compareSortKeys(rowSortKey, optionsForRange.start)} > 0` + } + AND ${ + optionsForRange.end === "end" + ? sqlExpression`1 = 1` + : optionsForRange.endInclusive + ? sqlExpression`${options.compareSortKeys(rowSortKey, optionsForRange.end)} <= 0` + : sqlExpression`${options.compareSortKeys(rowSortKey, optionsForRange.end)} < 0` + } + `; + const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`; + const sortChangesTableName = `sort_changes_${generateSecureRandomString()}`; + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + to_jsonb("oldSortKey"."newSortKey") AS "oldComputedSortKey", + to_jsonb("newSortKey"."newSortKey") AS "newComputedSortKey", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow" + FROM ${fromChangesTable} AS "changes" + LEFT JOIN LATERAL ( + SELECT "mapped"."newSortKey" + FROM ( + SELECT ${options.getSortKey} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowSortKey" AS "oldSortKey", + "changes"."oldRowData" AS "rowData" + ) AS "sortInput" + ) AS "mapped" + ) AS "oldSortKey" ON ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') + LEFT JOIN LATERAL ( + SELECT "mapped"."newSortKey" + FROM ( + SELECT ${options.getSortKey} + FROM ( + SELECT + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowSortKey" AS "oldSortKey", + "changes"."newRowData" AS "rowData" + ) AS "sortInput" + ) AS "mapped" + ) AS "newSortKey" ON ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') + WHERE ${isInitializedExpression} + `.toStatement(normalizedChangesTableName), + sqlStatement` + INSERT INTO pg_temp.bulldozer_side_effects ("note") + SELECT "effect"."note" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + CROSS JOIN LATERAL ( + SELECT pg_temp.bulldozer_sort_delete( + ${groupsPath}::jsonb[], + "changes"."groupKey", + ${compareSortKeysSqlLiteral}::text, + "changes"."rowIdentifier" + ) AS "note" + ) AS "effect" + WHERE "changes"."hasOldRow" + AND ( + NOT "changes"."hasNewRow" + OR "changes"."oldComputedSortKey" IS DISTINCT FROM "changes"."newComputedSortKey" + OR "changes"."oldRowData" IS DISTINCT FROM "changes"."newRowData" + ) + `, + sqlStatement` + INSERT INTO pg_temp.bulldozer_side_effects ("note") + SELECT "effect"."note" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} AS "changes" + CROSS JOIN LATERAL ( + SELECT pg_temp.bulldozer_sort_insert( + ${groupsPath}::jsonb[], + "changes"."groupKey", + ${compareSortKeysSqlLiteral}::text, + "changes"."rowIdentifier", + "changes"."newComputedSortKey", + "changes"."newRowData" + ) AS "note" + ) AS "effect" + WHERE "changes"."hasNewRow" + AND ( + NOT "changes"."hasOldRow" + OR "changes"."oldComputedSortKey" IS DISTINCT FROM "changes"."newComputedSortKey" + OR "changes"."oldRowData" IS DISTINCT FROM "changes"."newRowData" + ) + `, + sqlQuery` + SELECT + "groupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + CASE + WHEN "hasOldRow" THEN "oldComputedSortKey" + ELSE 'null'::jsonb + END AS "oldRowSortKey", + CASE + WHEN "hasNewRow" THEN "newComputedSortKey" + ELSE 'null'::jsonb + END AS "newRowSortKey", + "oldRowData" AS "oldRowData", + "newRowData" AS "newRowData" + FROM ${quoteSqlIdentifier(normalizedChangesTableName)} + WHERE ("hasOldRow" OR "hasNewRow") + AND ( + NOT ("hasOldRow" AND "hasNewRow") + OR "oldComputedSortKey" IS DISTINCT FROM "newComputedSortKey" + OR "oldRowData" IS DISTINCT FROM "newRowData" + ) + `.toStatement(sortChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(sortChangesTableName))), + ]; + }; + let fromTableTriggerRegistration: null | { deregister: () => void } = null; + const ensureFromTableTriggerRegistration = () => { + if (fromTableTriggerRegistration != null) return; + fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + return createFromTableTriggerStatements(fromChangesTable); + }); + }; + const deregisterFromTableTrigger = () => { + fromTableTriggerRegistration?.deregister(); + fromTableTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "sort", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + getSortKeySql: options.getSortKey.sql, + compareSortKeysSql: options.compareSortKeys(sqlExpression`$1`, sqlExpression`$2`).sql, + }, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: options.compareSortKeys, + init: () => { + ensureFromTableTriggerRegistration(); + const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`; + const fromRowsTableName = `from_rows_${generateSecureRandomString()}`; + const sortedRowsTableName = `sorted_rows_${generateSecureRandomString()}`; + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${groupsPath}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + ON CONFLICT ("keyPath") DO NOTHING + `, + options.fromTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromGroupsTableName), + sqlQuery` + SELECT + "groups"."groupkey" AS "groupKey", + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowsortkey" AS "oldSortKey", + "rows"."rowdata" AS "rowData" + FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupkey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + `.toStatement(fromRowsTableName), + sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "rowIdentifier", + "rows"."rowData" AS "rowData", + to_jsonb("sortKey"."newSortKey") AS "rowSortKey" + FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows" + CROSS JOIN LATERAL ( + SELECT "mapped"."newSortKey" + FROM ( + SELECT ${options.getSortKey} + FROM ( + SELECT + "rows"."rowIdentifier" AS "rowIdentifier", + "rows"."oldSortKey" AS "oldSortKey", + "rows"."rowData" AS "rowData" + ) AS "sortInput" + ) AS "mapped" + ) AS "sortKey" + `.toStatement(sortedRowsTableName), + sqlStatement` + INSERT INTO pg_temp.bulldozer_side_effects ("note") + SELECT pg_temp.bulldozer_sort_bulk_init_from_table( + ${groupsPath}::jsonb[], + ${quoteSqlStringLiteral(sortedRowsTableName)}::text, + ${compareSortKeysSqlLiteral}::text + ) + `, + ]; + }, + delete: () => { + deregisterFromTableTrigger(); + return [sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupMetadata" + ON "groupMetadata"."keyPath" = ${getGroupMetadataPath(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`)}::jsonb[] + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND COALESCE(("groupMetadata"."value"->>'rowCount')::int, 0) > 0 + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey != null ? sqlQuery` + WITH RECURSIVE "orderedRows" AS ( + SELECT + 0 AS "rowIndex", + ("startRow"."keyPath"[cardinality("startRow"."keyPath")] #>> '{}') AS "rowIdentifier", + "startRow"."value" AS "nodeValue" + FROM "BulldozerStorageEngine" AS "groupMetadata" + CROSS JOIN LATERAL ( + SELECT ${ + start === "start" + ? sqlExpression`"groupMetadata"."value"->>'headRowIdentifier'` + : sqlExpression`pg_temp.bulldozer_sort_find_successor(${groupsPath}::jsonb[], ${groupKey}, ${compareSortKeysSqlLiteral}::text, ''::text, ${start})` + } AS "startRowIdentifier" + ) AS "startLookup" + INNER JOIN "BulldozerStorageEngine" AS "startRow" + ON "startRow"."keyPath" = ${getGroupRowPath( + groupKey, + sqlExpression`to_jsonb("startLookup"."startRowIdentifier")`, + )}::jsonb[] + WHERE "groupMetadata"."keyPath" = ${getGroupMetadataPath(groupKey)}::jsonb[] + AND "startLookup"."startRowIdentifier" IS NOT NULL + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.compareSortKeys(sqlExpression`"startRow"."value"->'rowSortKey'`, end)} <= 0` + : sqlExpression`${options.compareSortKeys(sqlExpression`"startRow"."value"->'rowSortKey'`, end)} < 0` + } + + UNION ALL + + SELECT + "orderedRows"."rowIndex" + 1 AS "rowIndex", + ("nextRow"."keyPath"[cardinality("nextRow"."keyPath")] #>> '{}') AS "rowIdentifier", + "nextRow"."value" AS "nodeValue" + FROM "orderedRows" + INNER JOIN "BulldozerStorageEngine" AS "nextRow" + ON "orderedRows"."nodeValue"->>'nextRowIdentifier' IS NOT NULL + AND "nextRow"."keyPath" = ${getGroupRowPath(groupKey, sqlExpression`to_jsonb("orderedRows"."nodeValue"->>'nextRowIdentifier')`)}::jsonb[] + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.compareSortKeys(sqlExpression`"nextRow"."value"->'rowSortKey'`, end)} <= 0` + : sqlExpression`${options.compareSortKeys(sqlExpression`"nextRow"."value"->'rowSortKey'`, end)} < 0` + } + ) + SELECT + "orderedRows"."rowIdentifier" AS rowIdentifier, + "orderedRows"."nodeValue"->'rowSortKey' AS rowSortKey, + "orderedRows"."nodeValue"->'rowData' AS rowData + FROM "orderedRows" + WHERE ${sortRangePredicate(sqlExpression`"orderedRows"."nodeValue"->'rowSortKey'`, { start, end, startInclusive, endInclusive })} + ORDER BY "orderedRows"."rowIndex" ASC + ` : sqlQuery` + WITH RECURSIVE "groupMetadatas" AS ( + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS "groupKey", + "groupMetadata"."value" AS "groupMetadataValue" + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupMetadata" + ON "groupMetadata"."keyPath" = ${getGroupMetadataPath(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`)}::jsonb[] + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND COALESCE(("groupMetadata"."value"->>'rowCount')::int, 0) > 0 + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + ), + "orderedRows" AS ( + SELECT + "groupMetadatas"."groupKey" AS "groupKey", + 0 AS "rowIndex", + ("headRow"."keyPath"[cardinality("headRow"."keyPath")] #>> '{}') AS "rowIdentifier", + "headRow"."value" AS "nodeValue" + FROM "groupMetadatas" + INNER JOIN "BulldozerStorageEngine" AS "headRow" + ON ("groupMetadatas"."groupMetadataValue"->>'headRowIdentifier') IS NOT NULL + AND "headRow"."keyPath" = ${getGroupRowPath( + sqlExpression`"groupMetadatas"."groupKey"`, + sqlExpression`to_jsonb("groupMetadatas"."groupMetadataValue"->>'headRowIdentifier')`, + )}::jsonb[] + + UNION ALL + + SELECT + "orderedRows"."groupKey" AS "groupKey", + "orderedRows"."rowIndex" + 1 AS "rowIndex", + ("nextRow"."keyPath"[cardinality("nextRow"."keyPath")] #>> '{}') AS "rowIdentifier", + "nextRow"."value" AS "nodeValue" + FROM "orderedRows" + INNER JOIN "BulldozerStorageEngine" AS "nextRow" + ON "orderedRows"."nodeValue"->>'nextRowIdentifier' IS NOT NULL + AND "nextRow"."keyPath" = ${getGroupRowPath( + sqlExpression`"orderedRows"."groupKey"`, + sqlExpression`to_jsonb("orderedRows"."nodeValue"->>'nextRowIdentifier')`, + )}::jsonb[] + ) + SELECT + "orderedRows"."groupKey" AS groupKey, + "orderedRows"."rowIdentifier" AS rowIdentifier, + "orderedRows"."nodeValue"->'rowSortKey' AS rowSortKey, + "orderedRows"."nodeValue"->'rowData' AS rowData + FROM "orderedRows" + WHERE ${sortRangePredicate(sqlExpression`"orderedRows"."nodeValue"->'rowSortKey'`, { start, end, startInclusive, endInclusive })} + ORDER BY "orderedRows"."groupKey" ASC, "orderedRows"."rowIndex" ASC + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/stored-table.ts b/apps/backend/src/lib/bulldozer/db/tables/stored-table.ts new file mode 100644 index 0000000000..158c3a5a3d --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/stored-table.ts @@ -0,0 +1,145 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { RowData, RowIdentifier, SqlExpression, SqlStatement, TableId } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + quoteSqlStringLiteral, + singleNullSortKeyRangePredicate, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString, +} from "../utilities"; + +export function declareStoredTable(options: { + tableId: TableId, +}): Table & { + setRow(rowIdentifier: RowIdentifier, rowData: SqlExpression): SqlStatement[], + deleteRow(rowIdentifier: RowIdentifier): SqlStatement[], +} { + const triggers = new Map) => SqlStatement[]>(); + + // Note that this table has only one group and sort key (null), so all groups and rows are always returned by every filter. + return { + tableId: options.tableId, + inputTables: [], + debugArgs: { + operator: "stored", + tableId: tableIdToDebugString(options.tableId), + }, + compareGroupKeys: (a, b) => sqlExpression` 0 `, + compareSortKeys: (a, b) => sqlExpression` 0 `, + init: () => [sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["rows"])}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + `], + delete: () => [sqlStatement` + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getTablePath(options.tableId)}::jsonb[] + `], + isInitialized: () => sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT 'null'::jsonb AS groupKey + WHERE ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey == null ? sqlQuery` + SELECT + 'null'::jsonb AS groupKey, + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["rows"])}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ` : sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getStorageEnginePath(options.tableId, ["rows"])}::jsonb[] + AND ${groupKey} IS NOT DISTINCT FROM 'null'::jsonb + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + setRow: (rowIdentifier, rowData) => { + const oldRowsTableName = `old_rows_${generateSecureRandomString()}`; + const upsertedRowsTableName = `upserted_rows_${generateSecureRandomString()}`; + const changesTableName = `changes_${generateSecureRandomString()}`; + const rowIdentifierLiteral = quoteSqlStringLiteral(rowIdentifier); + const rowValue = sqlExpression` + jsonb_build_object( + 'rowData', ${rowData}::jsonb + ) + `; + return [ + sqlQuery` + SELECT "value"->'rowData' AS "oldRowData" + FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[] + `.toStatement(oldRowsTableName), + sqlQuery` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES ( + gen_random_uuid(), + ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[], + ${rowValue}::jsonb + ) + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = ${rowValue}::jsonb + RETURNING "value"->'rowData' AS "newRowData" + `.toStatement(upsertedRowsTableName), + sqlQuery` + SELECT + 'null'::jsonb AS "groupKey", + ${rowIdentifierLiteral}::text AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + ${quoteSqlIdentifier(oldRowsTableName)}."oldRowData" AS "oldRowData", + ${quoteSqlIdentifier(upsertedRowsTableName)}."newRowData" AS "newRowData" + FROM ${quoteSqlIdentifier(upsertedRowsTableName)} + LEFT JOIN ${quoteSqlIdentifier(oldRowsTableName)} ON true + `.toStatement(changesTableName), + ...[...triggers.values()].flatMap(trigger => trigger(quoteSqlIdentifier(changesTableName))) + ]; + }, + deleteRow: (rowIdentifier) => { + const deletedRowsTableName = `deleted_rows_${generateSecureRandomString()}`; + const changesTableName = `changes_${generateSecureRandomString()}`; + const rowIdentifierLiteral = quoteSqlStringLiteral(rowIdentifier); + return [ + sqlQuery` + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["rows", rowIdentifier])}::jsonb[] + RETURNING "value"->'rowData' AS "oldRowData" + `.toStatement(deletedRowsTableName), + sqlQuery` + SELECT + 'null'::jsonb AS "groupKey", + ${rowIdentifierLiteral}::text AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + ${quoteSqlIdentifier(deletedRowsTableName)}."oldRowData" AS "oldRowData", + 'null'::jsonb AS "newRowData" + FROM ${quoteSqlIdentifier(deletedRowsTableName)} + `.toStatement(changesTableName), + ...[...triggers.values()].flatMap(trigger => trigger(quoteSqlIdentifier(changesTableName))) + ]; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/tables/time-fold-table.ts b/apps/backend/src/lib/bulldozer/db/tables/time-fold-table.ts new file mode 100644 index 0000000000..ed06fdddb0 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/tables/time-fold-table.ts @@ -0,0 +1,562 @@ +import { generateSecureRandomString } from "@stackframe/stack-shared/dist/utils/crypto"; +import type { Table } from ".."; +import type { Json, RowData, RowIdentifier, SqlExpression, SqlMapper, SqlStatement, TableId, Timestamp } from "../utilities"; +import { + getStorageEnginePath, + getTablePath, + quoteSqlIdentifier, + quoteSqlStringLiteral, + singleNullSortKeyRangePredicate, + sqlExpression, + sqlQuery, + sqlStatement, + tableIdToDebugString, +} from "../utilities"; + +/** + * Materialized time-aware fold with queue-backed future reprocessing. + * + * For each input row, the reducer runs once with `timestamp = null`, then can optionally + * schedule follow-up runs by returning `nextTimestamp`. Due follow-ups rerun the reducer with + * `timestamp = previousNextTimestamp` and the latest state. + * + * Output semantics: + * - Timed reruns append newly emitted rows to previously emitted rows for that input row. + * - Source-row updates/deletes still recompute/reset that input row's emitted output. + * + * Determinism guidance: + * - Avoid non-deterministic SQL such as `now()` or random generators inside reducers when output + * correctness depends on those values. Re-initializing/replaying should produce the same results. + * - If randomness is needed (for example correlation IDs or light debugging metadata), treat it as + * best-effort auxiliary data and do not build correctness-critical logic on top of it. + * - Prefer deriving `nextTimestamp` from stable row fields (for example, an event timestamp on + * `oldRowData`) and from the reducer input `timestamp` itself. + */ +export function declareTimeFoldTable< + GK extends Json, + OldRD extends RowData, + NewRD extends RowData, + S extends Json, +>(options: { + tableId: TableId, + fromTable: Table, + initialState: SqlExpression, + reducer: SqlMapper<{ oldState: S, oldRowData: OldRD, timestamp: Timestamp | null }, { newState: S, newRowsData: NewRD[], nextTimestamp: Timestamp | null }>, +}): Table { + const triggers = new Map) => SqlStatement[]>(); + const reducerSqlLiteral = quoteSqlStringLiteral(options.reducer.sql); + const tableStoragePath = getStorageEnginePath(options.tableId, []); + const groupsPath = getStorageEnginePath(options.tableId, ["groups"]); + const getGroupKeyPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey]); + const getGroupRowsPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows"]); + const getGroupRowPath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "rows", rowIdentifier]); + const getGroupStatesPath = (groupKey: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "states"]); + const getGroupStatePath = (groupKey: SqlExpression, rowIdentifier: SqlExpression) => getStorageEnginePath(options.tableId, ["groups", groupKey, "states", rowIdentifier]); + const createExpandedRowIdentifier = (sourceRowIdentifier: SqlExpression, flatIndex: SqlExpression): SqlExpression => + sqlExpression`(${sourceRowIdentifier} || ':' || (${flatIndex}::text))`; + const isInitializedExpression = sqlExpression` + EXISTS ( + SELECT 1 FROM "BulldozerStorageEngine" + WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[] + ) + `; + const lastProcessedTimestampExpression = sqlExpression` + COALESCE( + ( + SELECT "lastProcessedAt" + FROM "BulldozerTimeFoldMetadata" + WHERE "key" = 'singleton' + ), + now() + ) + `; + const createApplyChangesStatements = (normalizedChangesTable: SqlExpression) => { + const oldStateRowsTableName = `old_state_rows_${generateSecureRandomString()}`; + const oldTimeFoldRowsTableName = `old_time_fold_rows_${generateSecureRandomString()}`; + const recomputedStatesTableName = `recomputed_states_${generateSecureRandomString()}`; + const newTimeFoldRowsTableName = `new_time_fold_rows_${generateSecureRandomString()}`; + const timeFoldChangesTableName = `time_fold_changes_${generateSecureRandomString()}`; + + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "stateRows"."value" AS "stateValue" + FROM ${normalizedChangesTable} AS "changes" + INNER JOIN "BulldozerStorageEngine" AS "stateRows" + ON "changes"."hasOldRow" + AND "stateRows"."keyPath" = ${getGroupStatePath( + sqlExpression`"changes"."groupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `.toStatement(oldStateRowsTableName), + sqlQuery` + SELECT + "states"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"states"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(oldStateRowsTableName)} AS "states" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("states"."stateValue"->'emittedRowsData') = 'array' THEN "states"."stateValue"->'emittedRowsData' + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(oldTimeFoldRowsTableName), + sqlQuery` + WITH RECURSIVE "stateChain" AS ( + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."newRowData" AS "rowData", + "lastProcessed"."lastProcessedAt" AS "lastProcessedAt", + 0 AS "depth", + to_jsonb(${options.initialState}) AS "oldState", + NULL::timestamptz AS "reducerTimestamp", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData", + "reduced"."newRowsData" AS "emittedRowsData", + "reduced"."nextTimestamp" AS "nextTimestamp" + FROM ${normalizedChangesTable} AS "changes" + CROSS JOIN LATERAL ( + SELECT ${lastProcessedTimestampExpression} AS "lastProcessedAt" + ) AS "lastProcessed" + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData", + CASE + WHEN "reducerRows"."nextTimestamp" IS NULL THEN NULL::timestamptz + ELSE ("reducerRows"."nextTimestamp")::timestamptz + END AS "nextTimestamp" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + to_jsonb(${options.initialState}) AS "oldState", + "changes"."newRowData" AS "oldRowData", + NULL::timestamptz AS "timestamp" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + WHERE "changes"."hasNewRow" + + UNION ALL + + SELECT + "stateChain"."groupKey" AS "groupKey", + "stateChain"."rowIdentifier" AS "rowIdentifier", + "stateChain"."rowData" AS "rowData", + "stateChain"."lastProcessedAt" AS "lastProcessedAt", + "stateChain"."depth" + 1 AS "depth", + "stateChain"."newState" AS "oldState", + "stateChain"."nextTimestamp" AS "reducerTimestamp", + "reduced"."newState" AS "newState", + "reduced"."newRowsData" AS "newRowsData", + ("stateChain"."emittedRowsData" || "reduced"."newRowsData") AS "emittedRowsData", + "reduced"."nextTimestamp" AS "nextTimestamp" + FROM "stateChain" + CROSS JOIN LATERAL ( + SELECT + to_jsonb("reducerRows"."newState") AS "newState", + CASE + WHEN jsonb_typeof(to_jsonb("reducerRows"."newRowsData")) = 'array' THEN to_jsonb("reducerRows"."newRowsData") + ELSE '[]'::jsonb + END AS "newRowsData", + CASE + WHEN "reducerRows"."nextTimestamp" IS NULL THEN NULL::timestamptz + ELSE ("reducerRows"."nextTimestamp")::timestamptz + END AS "nextTimestamp" + FROM ( + SELECT ${options.reducer} + FROM ( + SELECT + "stateChain"."newState" AS "oldState", + "stateChain"."rowData" AS "oldRowData", + "stateChain"."nextTimestamp" AS "timestamp" + ) AS "reducerInput" + ) AS "reducerRows" + ) AS "reduced" + WHERE "stateChain"."nextTimestamp" IS NOT NULL + AND "stateChain"."nextTimestamp" <= "stateChain"."lastProcessedAt" + AND "stateChain"."depth" < 10000 + ) + SELECT DISTINCT ON ("groupKey", "rowIdentifier") + "groupKey" AS "groupKey", + "rowIdentifier" AS "rowIdentifier", + "rowData" AS "rowData", + "lastProcessedAt" AS "lastProcessedAt", + "newState" AS "stateAfter", + "emittedRowsData" AS "emittedRowsData", + "nextTimestamp" AS "nextTimestamp" + FROM "stateChain" + ORDER BY "groupKey", "rowIdentifier", "depth" DESC + `.toStatement(recomputedStatesTableName), + sqlQuery` + SELECT + "states"."groupKey" AS "groupKey", + ${createExpandedRowIdentifier( + sqlExpression`"states"."rowIdentifier"`, + sqlExpression`"flatRow"."flatIndex"`, + )} AS "rowIdentifier", + "flatRow"."rowData" AS "rowData" + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} AS "states" + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof("states"."emittedRowsData") = 'array' THEN "states"."emittedRowsData" + ELSE '[]'::jsonb + END + ) WITH ORDINALITY AS "flatRow"("rowData", "flatIndex") + `.toStatement(newTimeFoldRowsTableName), + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + "insertRows"."keyPath", + "insertRows"."value" + FROM ( + SELECT DISTINCT + ${getGroupKeyPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupRowsPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} + UNION + SELECT DISTINCT + ${getGroupStatesPath(sqlExpression`"groupKey"`)}::jsonb[] AS "keyPath", + 'null'::jsonb AS "value" + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} + ) AS "insertRows" + ON CONFLICT ("keyPath") DO NOTHING + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "targetRows" + USING ${quoteSqlIdentifier(oldTimeFoldRowsTableName)} AS "oldRows" + WHERE "targetRows"."keyPath" = ${getGroupRowPath( + sqlExpression`"oldRows"."groupKey"`, + sqlExpression`to_jsonb("oldRows"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "targetStates" + USING ${normalizedChangesTable} AS "changes" + WHERE "changes"."hasOldRow" + AND "targetStates"."keyPath" = ${getGroupStatePath( + sqlExpression`"changes"."groupKey"`, + sqlExpression`to_jsonb("changes"."rowIdentifier"::text)`, + )}::jsonb[] + `, + sqlStatement` + DELETE FROM "BulldozerTimeFoldQueue" AS "queue" + USING ${normalizedChangesTable} AS "changes" + WHERE ("changes"."hasOldRow" OR "changes"."hasNewRow") + AND "queue"."tableStoragePath" = ${tableStoragePath}::jsonb[] + AND "queue"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey" + AND "queue"."rowIdentifier" = "changes"."rowIdentifier" + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupStatePath( + sqlExpression`"states"."groupKey"`, + sqlExpression`to_jsonb("states"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object( + 'rowData', "states"."rowData", + 'stateAfter', "states"."stateAfter", + 'emittedRowsData', "states"."emittedRowsData", + 'nextTimestamp', + CASE + WHEN "states"."nextTimestamp" IS NULL THEN 'null'::jsonb + ELSE to_jsonb("states"."nextTimestamp") + END + ) + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} AS "states" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + SELECT + gen_random_uuid(), + ${getGroupRowPath( + sqlExpression`"rows"."groupKey"`, + sqlExpression`to_jsonb("rows"."rowIdentifier"::text)`, + )}::jsonb[], + jsonb_build_object('rowData', "rows"."rowData") + FROM ${quoteSqlIdentifier(newTimeFoldRowsTableName)} AS "rows" + ON CONFLICT ("keyPath") DO UPDATE + SET "value" = EXCLUDED."value" + `, + sqlStatement` + DELETE FROM "BulldozerStorageEngine" AS "staleGroupPaths" + USING ${normalizedChangesTable} AS "changes" + WHERE "changes"."hasOldRow" + AND "staleGroupPaths"."keyPath" IN ( + ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[], + ${getGroupStatesPath(sqlExpression`"changes"."groupKey"`)}::jsonb[], + ${getGroupKeyPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "stateRows" + WHERE "stateRows"."keyPathParent" = ${getGroupStatesPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "timeFoldRows" + WHERE "timeFoldRows"."keyPathParent" = ${getGroupRowsPath(sqlExpression`"changes"."groupKey"`)}::jsonb[] + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} AS "newStates" + WHERE "newStates"."groupKey" IS NOT DISTINCT FROM "changes"."groupKey" + ) + `, + sqlStatement` + INSERT INTO "BulldozerTimeFoldQueue" ( + "id", + "tableStoragePath", + "groupKey", + "rowIdentifier", + "scheduledAt", + "stateAfter", + "rowData", + "reducerSql" + ) + SELECT + gen_random_uuid(), + ${tableStoragePath}::jsonb[], + "states"."groupKey", + "states"."rowIdentifier", + "states"."nextTimestamp", + "states"."stateAfter", + "states"."rowData", + ${reducerSqlLiteral} + FROM ${quoteSqlIdentifier(recomputedStatesTableName)} AS "states" + WHERE "states"."nextTimestamp" IS NOT NULL + AND "states"."nextTimestamp" > "states"."lastProcessedAt" + ON CONFLICT ("tableStoragePath", "groupKey", "rowIdentifier") DO UPDATE + SET + "scheduledAt" = EXCLUDED."scheduledAt", + "stateAfter" = EXCLUDED."stateAfter", + "rowData" = EXCLUDED."rowData", + "reducerSql" = EXCLUDED."reducerSql", + "updatedAt" = now() + `, + sqlQuery` + SELECT + COALESCE("newRows"."groupKey", "oldRows"."groupKey") AS "groupKey", + COALESCE("newRows"."rowIdentifier", "oldRows"."rowIdentifier") AS "rowIdentifier", + 'null'::jsonb AS "oldRowSortKey", + 'null'::jsonb AS "newRowSortKey", + CASE + WHEN "oldRows"."rowData" IS NULL THEN 'null'::jsonb + ELSE "oldRows"."rowData" + END AS "oldRowData", + CASE + WHEN "newRows"."rowData" IS NULL THEN 'null'::jsonb + ELSE "newRows"."rowData" + END AS "newRowData" + FROM ${quoteSqlIdentifier(oldTimeFoldRowsTableName)} AS "oldRows" + FULL OUTER JOIN ${quoteSqlIdentifier(newTimeFoldRowsTableName)} AS "newRows" + ON "oldRows"."groupKey" IS NOT DISTINCT FROM "newRows"."groupKey" + AND "oldRows"."rowIdentifier" = "newRows"."rowIdentifier" + WHERE "oldRows"."rowData" IS DISTINCT FROM "newRows"."rowData" + `.toStatement(timeFoldChangesTableName), + ...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(timeFoldChangesTableName))), + ]; + }; + const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => { + const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`; + return [ + sqlQuery` + SELECT + "changes"."groupKey" AS "groupKey", + "changes"."rowIdentifier" AS "rowIdentifier", + "changes"."oldRowData" AS "oldRowData", + "changes"."newRowData" AS "newRowData", + ("changes"."oldRowData" IS NOT NULL AND jsonb_typeof("changes"."oldRowData") = 'object') AS "hasOldRow", + ("changes"."newRowData" IS NOT NULL AND jsonb_typeof("changes"."newRowData") = 'object') AS "hasNewRow" + FROM ${fromChangesTable} AS "changes" + WHERE ${isInitializedExpression} + AND ( + NOT ( + "changes"."oldRowData" IS NOT NULL + AND jsonb_typeof("changes"."oldRowData") = 'object' + AND "changes"."newRowData" IS NOT NULL + AND jsonb_typeof("changes"."newRowData") = 'object' + ) + OR "changes"."oldRowData" IS DISTINCT FROM "changes"."newRowData" + ) + `.toStatement(normalizedChangesTableName), + ...createApplyChangesStatements(quoteSqlIdentifier(normalizedChangesTableName)), + ]; + }; + let fromTableTriggerRegistration: null | { deregister: () => void } = null; + const ensureFromTableTriggerRegistration = () => { + if (fromTableTriggerRegistration != null) return; + fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => { + return createFromTableTriggerStatements(fromChangesTable); + }); + }; + const deregisterFromTableTrigger = () => { + fromTableTriggerRegistration?.deregister(); + fromTableTriggerRegistration = null; + }; + + return { + tableId: options.tableId, + inputTables: [options.fromTable], + debugArgs: { + operator: "timefold", + tableId: tableIdToDebugString(options.tableId), + fromTableId: tableIdToDebugString(options.fromTable.tableId), + initialStateSql: options.initialState.sql, + reducerSql: options.reducer.sql, + }, + compareGroupKeys: options.fromTable.compareGroupKeys, + compareSortKeys: (_a, _b) => sqlExpression`0`, + init: () => { + ensureFromTableTriggerRegistration(); + const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`; + const fromRowsTableName = `from_rows_${generateSecureRandomString()}`; + const initChangesTableName = `init_changes_${generateSecureRandomString()}`; + return [ + sqlStatement` + INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value") + VALUES + (gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb), + (gen_random_uuid(), ${tableStoragePath}, 'null'::jsonb), + (gen_random_uuid(), ${groupsPath}, 'null'::jsonb), + (gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}, '{ "version": 1 }'::jsonb) + ON CONFLICT ("keyPath") DO NOTHING + `, + options.fromTable.listGroups({ + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + }).toStatement(fromGroupsTableName), + sqlQuery` + SELECT + "groups"."groupkey" AS "groupKey", + "rows"."rowidentifier" AS "rowIdentifier", + "rows"."rowdata" AS "newRowData" + FROM ${quoteSqlIdentifier(fromGroupsTableName)} AS "groups" + CROSS JOIN LATERAL ( + ${options.fromTable.listRowsInGroup({ + groupKey: sqlExpression`"groups"."groupkey"`, + start: "start", + end: "end", + startInclusive: true, + endInclusive: true, + })} + ) AS "rows" + `.toStatement(fromRowsTableName), + sqlQuery` + SELECT + "rows"."groupKey" AS "groupKey", + "rows"."rowIdentifier" AS "rowIdentifier", + 'null'::jsonb AS "oldRowData", + "rows"."newRowData" AS "newRowData", + false AS "hasOldRow", + true AS "hasNewRow" + FROM ${quoteSqlIdentifier(fromRowsTableName)} AS "rows" + `.toStatement(initChangesTableName), + ...createApplyChangesStatements(quoteSqlIdentifier(initChangesTableName)), + ]; + }, + delete: () => { + deregisterFromTableTrigger(); + return [ + sqlStatement` + DELETE FROM "BulldozerTimeFoldQueue" + WHERE "tableStoragePath" = ${tableStoragePath}::jsonb[] + `, + sqlStatement` + WITH RECURSIVE "pathsToDelete" AS ( + SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path" + UNION ALL + SELECT "BulldozerStorageEngine"."keyPath" AS "path" + FROM "BulldozerStorageEngine" + INNER JOIN "pathsToDelete" + ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path" + ) + DELETE FROM "BulldozerStorageEngine" + WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete") + `, + ]; + }, + isInitialized: () => isInitializedExpression, + listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery` + SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey + FROM "BulldozerStorageEngine" AS "groupPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND EXISTS ( + SELECT 1 + FROM "BulldozerStorageEngine" AS "groupRowsPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRow" + ON "groupRow"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + ) + AND ${ + start === "start" + ? sqlExpression`1 = 1` + : startInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} >= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, start)} > 0` + } + AND ${ + end === "end" + ? sqlExpression`1 = 1` + : endInclusive + ? sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} <= 0` + : sqlExpression`${options.fromTable.compareGroupKeys(sqlExpression`"groupPath"."keyPath"[cardinality("groupPath"."keyPath")]`, end)} < 0` + } + ORDER BY "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] ASC + `, + listRowsInGroup: ({ groupKey, start, end, startInclusive, endInclusive }) => groupKey != null ? sqlQuery` + SELECT + ("keyPath"[cardinality("keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" + WHERE "keyPathParent" = ${getGroupRowsPath(groupKey)}::jsonb[] + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ORDER BY rowIdentifier ASC + ` : sqlQuery` + SELECT + "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey, + ("rows"."keyPath"[cardinality("rows"."keyPath")] #>> '{}') AS rowIdentifier, + 'null'::jsonb AS rowSortKey, + "rows"."value"->'rowData' AS rowData + FROM "BulldozerStorageEngine" AS "groupPath" + INNER JOIN "BulldozerStorageEngine" AS "groupRowsPath" + ON "groupRowsPath"."keyPathParent" = "groupPath"."keyPath" + INNER JOIN "BulldozerStorageEngine" AS "rows" + ON "rows"."keyPathParent" = "groupRowsPath"."keyPath" + WHERE "groupPath"."keyPathParent" = ${groupsPath}::jsonb[] + AND "groupRowsPath"."keyPath"[cardinality("groupRowsPath"."keyPath")] = to_jsonb('rows'::text) + AND ${singleNullSortKeyRangePredicate({ start, end, startInclusive, endInclusive })} + ORDER BY groupKey ASC, rowIdentifier ASC + `, + registerRowChangeTrigger: (trigger) => { + const id = generateSecureRandomString(); + triggers.set(id, trigger); + return { deregister: () => triggers.delete(id) }; + }, + }; +} diff --git a/apps/backend/src/lib/bulldozer/db/utilities.ts b/apps/backend/src/lib/bulldozer/db/utilities.ts new file mode 100644 index 0000000000..ef427a5765 --- /dev/null +++ b/apps/backend/src/lib/bulldozer/db/utilities.ts @@ -0,0 +1,88 @@ +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { templateIdentity } from "@stackframe/stack-shared/dist/utils/strings"; + +const sqlTemplateLiteral = (type: T) => (strings: TemplateStringsArray, ...values: { sql: string }[]) => ({ type, sql: templateIdentity(strings, ...values.map(v => v.sql)) }); + +export type SqlStatement = { type: "statement", outputName?: string, sql: string, requiresSequentialExecution?: boolean }; +export const sqlStatement = sqlTemplateLiteral<"statement">("statement"); + +export type SqlQuery = void> = { type: "query", sql: string, toStatement(outputName?: string): SqlStatement }; +export const sqlQuery = (...args: Parameters>>) => { + return { + ...sqlTemplateLiteral<"query">("query")(...args), + toStatement(outputName?: string) { + return { type: "statement", outputName, sql: this.sql } as const; + } + }; +}; + +export type SqlExpression = { type: "expression", sql: string }; +export const sqlExpression = sqlTemplateLiteral<"expression">("expression"); + +export type Json = string | number | boolean | null | { [key: string]: Json } | Json[]; +export type RowData = Record; +export type Timestamp = string; +export type SqlMapper = { type: "mapper", sql: string }; // ex.: "row.id AS id, row.old_value + 1 AS new_value" +export const sqlMapper = sqlTemplateLiteral<"mapper">("mapper"); +export type SqlPredicate = { type: "predicate", sql: string }; // ex.: "user_id = 123" +export const sqlPredicate = sqlTemplateLiteral<"predicate">("predicate"); + +export const sqlArray = (exprs: (SqlExpression | SqlMapper)[]) => ({ type: "expression", sql: `ARRAY[${exprs.map(e => e.sql).join(", ")}]` } as const); + +export type RowIdentifier = string; +export type TableId = string | { "tableType": "internal", "internalId": string, "parent": null | TableId }; + +export function quoteSqlIdentifier(input: string): SqlExpression { + if (input.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) == null) { + throw new StackAssertionError("Invalid SQL identifier", { input }); + } + return { type: "expression", sql: `"${input}"` }; +} +export function quoteSqlStringLiteral(input: string): SqlExpression { + return { type: "expression", sql: `'${input.replaceAll("'", "''")}'` }; +} +export function quoteSqlJsonbLiteral(input: Json): SqlExpression { + return { type: "expression", sql: `${quoteSqlStringLiteral(JSON.stringify(input)).sql}::jsonb` }; +} +export function getTablePath(tableId: TableId): SqlExpression { + return sqlArray(getTablePathSegments(tableId)); +} +export function getStorageEnginePath(tableId: TableId, path: (string | SqlExpression | SqlMapper)[]): SqlExpression { + return sqlArray([ + ...getTablePathSegments(tableId), + quoteSqlJsonbLiteral("storage"), + ...path.map(p => typeof p === "string" ? quoteSqlJsonbLiteral(p) : p), + ]); +} +export function getTablePathSegments(tableId: TableId): SqlExpression[] { + const tableIdWithParents = []; + let currentTableId = tableId; + while (true) { + if (typeof currentTableId === "string") { + tableIdWithParents.push(`external:${currentTableId}`); + break; + } else { + tableIdWithParents.push(`internal:${currentTableId.internalId}`); + if (currentTableId.parent === null) break; + currentTableId = currentTableId.parent; + } + } + return [ + ...tableIdWithParents.reverse().flatMap(id => ["table", id]), + ].map(id => quoteSqlJsonbLiteral(id)); +} +export function tableIdToDebugString(tableId: TableId): string { + return typeof tableId === "string" + ? tableId + : JSON.stringify(tableId); +} +export function singleNullSortKeyRangePredicate(options: { + start: SqlExpression | "start", + end: SqlExpression | "end", + startInclusive: boolean, + endInclusive: boolean, +}): SqlExpression { + return (options.start === "start" || options.startInclusive) && (options.end === "end" || options.endInclusive) + ? sqlExpression`1 = 1` + : sqlExpression`1 = 0`; +} diff --git a/apps/backend/src/lib/tokens.tsx b/apps/backend/src/lib/tokens.tsx index e9c817f797..82bb8f4afe 100644 --- a/apps/backend/src/lib/tokens.tsx +++ b/apps/backend/src/lib/tokens.tsx @@ -314,7 +314,6 @@ export async function generateAccessTokenFromRefreshTokenIfValid(options: Refres email: user.primary_email, email_verified: user.primary_email_verified, selected_team_id: user.selected_team_id, - signed_up_at: Math.floor(user.signed_up_at_millis / 1000), is_anonymous: user.is_anonymous, is_restricted: user.is_restricted, restricted_reason: user.restricted_reason, diff --git a/apps/backend/src/prisma-client.tsx b/apps/backend/src/prisma-client.tsx index c06fe3a21a..2fef5229c7 100644 --- a/apps/backend/src/prisma-client.tsx +++ b/apps/backend/src/prisma-client.tsx @@ -1,5 +1,4 @@ import { Prisma, PrismaClient } from "@/generated/prisma/client"; -import { getStackServerApp } from "@/stack"; import { PrismaNeon } from "@prisma/adapter-neon"; import { PrismaPg } from '@prisma/adapter-pg'; import { readReplicas } from '@prisma/extension-read-replicas'; @@ -60,6 +59,7 @@ async function resolveNeonConnectionString(entry: string): Promise { if (!isUuid(entry)) { return entry; } + const { getStackServerApp } = await import("@/stack"); const store = await getStackServerApp().getDataVaultStore('neon-connection-strings'); const secret = "no client side encryption"; const value = await store.getValue(entry, { secret }); diff --git a/apps/backend/src/route-handlers/prisma-handler.tsx b/apps/backend/src/route-handlers/prisma-handler.tsx index 22d1121bb9..7e06c51323 100644 --- a/apps/backend/src/route-handlers/prisma-handler.tsx +++ b/apps/backend/src/route-handlers/prisma-handler.tsx @@ -12,14 +12,15 @@ type GetResult = [T] extends [never] ? R : T; type AllPrismaModelNames = Prisma.TypeMap["meta"]["modelProps"]; +type ModelOperations = Prisma.TypeMap["model"][Capitalize]["operations"]; type WhereUnique = Prisma.TypeMap["model"][Capitalize]["operations"]["findUniqueOrThrow"]["args"]["where"]; type WhereMany = Prisma.TypeMap["model"][Capitalize]["operations"]["findMany"]["args"]["where"]; type Where = { [K in keyof WhereMany as (K extends keyof WhereUnique ? K : never)]: WhereMany[K] }; type Include = (Prisma.TypeMap["model"][Capitalize]["operations"]["findMany"]["args"] & { include?: unknown })["include"]; type BaseFields = Where & Partial>; type PRead, I extends Include> = GetResult]["payload"], { where: W, include: I }, "findUniqueOrThrow">; -type PUpdate = Prisma.TypeMap["model"][Capitalize]["operations"]["update"]["args"]["data"]; -type PCreate = Prisma.TypeMap["model"][Capitalize]["operations"]["create"]["args"]["data"]; +type PUpdate = ModelOperations extends { update: { args: { data: infer D } } } ? D : never; +type PCreate = ModelOperations extends { create: { args: { data: infer D } } } ? D : never; type PEitherWrite = (PCreate | PUpdate) & Partial & PUpdate, unknown>>; type CRead> = T extends { Admin: { Read: infer R } } ? R : never; @@ -141,13 +142,18 @@ export function createPrismaCrudHandlers< }; }), onCreate: wrapper(false, async (data, context) => { - const prisma = await (globalPrismaClient[prismaModelName].create as any)({ + const prismaModel = globalPrismaClient[prismaModelName]; + const createMethod = Reflect.get(prismaModel, "create"); + if (typeof createMethod !== "function") { + throw new Error(`Prisma model ${prismaModelName} does not support create()`); + } + const prisma = await Reflect.apply(createMethod, prismaModel, [{ include: await options.include(context), data: { ...await options.baseFields(context), ...await crudToPrisma(data, { ...context, type: 'create' }), }, - }); + }]); // TODO pass the same transaction to onCreate as the one that creates the user row // we should probably do this with all functions and pass a transaction around in the context await options.onCreate?.(prisma, context); diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html index 828ddfec1b..b65ecb695e 100644 --- a/apps/dev-launchpad/public/index.html +++ b/apps/dev-launchpad/public/index.html @@ -324,6 +324,16 @@

Background services

importance: 1, img: "https://www.svgrepo.com/show/374002/replication.svg", }, + { + name: "Bulldozer Studio", + portSuffix: "39", + description: [ + "Bulldozer table graph and editor", + "Includes raw storage debug browser", + ], + importance: 1, + img: "https://www.svgrepo.com/show/349299/database.svg", + }, { name: "JS example", portSuffix: "19", diff --git a/claude/CLAUDE-KNOWLEDGE.md b/claude/CLAUDE-KNOWLEDGE.md index 0f5fa27dad..1f94fe42b1 100644 --- a/claude/CLAUDE-KNOWLEDGE.md +++ b/claude/CLAUDE-KNOWLEDGE.md @@ -79,6 +79,9 @@ A: Run lint from `apps/dashboard` directly (for example `pnpm lint -- "src/app/( Q: How should unsubscribe-link e2e tests avoid breakage from email theme/layout changes? A: In `apps/e2e/tests/backend/endpoints/api/v1/unsubscribe-link.test.ts`, avoid snapshotting the entire rendered HTML for transactional emails; assert stable behavior instead (email content present and `/api/v1/emails/unsubscribe-link` absent) so cosmetic wrapper/style changes do not fail the test. +Q: Why is the JIT disabled for Bulldozer DB mutations with only a few rows? +A: PostgreSQL JIT can dominate runtime for Bulldozer's giant single-statement CTE transactions. In a `group -> map -> map -> group` mutation with only 31 SQL statements and ~8 source rows, `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, VERBOSE)` showed ~1.4ms planning, ~1598.9ms execution, and ~1597.5ms of JIT time (`Optimization` ~836ms, `Emission` ~740ms) while the actual plan nodes were sub-millisecond. Disabling JIT locally for Bulldozer transactions with `SET LOCAL jit = off;` in `toExecutableSqlTransaction()` dropped the same query to ~0.63ms execution and brought the stacked fuzz case from ~41s to ~0.34s. + Q: How do cross-domain auth handoffs avoid creating extra refresh-token sessions? A: The cross-domain authorize route must carry the current `refreshTokenId` through authorization-code exchange and OAuth token issuance must reuse that ID. Keep `afterCallbackRedirectUrl` URL-only and persist refresh-token linkage in `ProjectUserAuthorizationCode.grantedRefreshTokenId`; then return that as `user.refreshTokenId` in `getAuthorizationCode` so token issuance can reuse the same refresh-token row with ownership checks. @@ -142,6 +145,21 @@ A: Only write `onboardingStatus` when the `Project.onboardingStatus` column exis Q: Where is the private sign-up risk engine generated entrypoint in backend now? A: The generator script writes `apps/backend/src/private/implementation.generated.ts` (not `src/generated/private-sign-up-risk-engine.ts`), and backend runtime imports should target `@/private/implementation.generated`. +Q: When do Bulldozer transactions need sequential temp-table execution instead of the default giant CTE executor? +A: Most Bulldozer operators should keep using the original giant-CTE executor because it is faster and matches existing semantics. `declareSortTable` is the exception for its bulk-init path: it needs real temp tables so a transaction-local PL/pgSQL helper can read intermediate sorted rows by table name. The safe split is to keep the CTE executor by default and switch to sequential execution only when a statement uses `pg_temp.bulldozer_sort_bulk_init_from_table`. Also, plain side-effecting `SELECT pg_temp.helper(...)` CTEs are not reliable unless they are wrapped in a data-modifying statement such as `INSERT INTO pg_temp.bulldozer_side_effects ...`. + +Q: Why can initializing a Bulldozer operator with an internal child table fail with `BulldozerStorageEngine_keyPathParent_fkey`? +A: Internal child table paths add an extra `"table"` segment (for example `.../table/external:parent/table/internal:child`). The parent operator must insert the intermediate `.../table` keyPath before running the child table `init()`. Without that node, inserting the child root path violates the storage engine parent foreign key. + +Q: Why can a multi-input operator like `declareLeftJoinTable` read stale upstream rows inside trigger execution? +A: Bulldozer executes most statement batches as one giant CTE transaction for speed. Within that single statement snapshot, downstream reads of `BulldozerStorageEngine` may not see upstream writes from earlier CTEs unless data is passed explicitly. For left-join trigger correctness (especially when one input depends on the other), force sequential execution for those statements (for example with a sentinel checked in `toExecutableSqlStatements`) or derive new input state directly from change tables instead of re-reading storage. + +Q: How can `declareLeftJoinTable` avoid accidental scans over unrelated `BulldozerStorageEngine` rows? +A: Avoid all-groups `listRowsInGroup` scans in `init()`. First list left-table groups, then fetch left/right rows per group via `CROSS JOIN LATERAL listRowsInGroup({ groupKey })`. For all-groups read paths, traverse from table-local group nodes using `keyPathParent = ` equality joins (`groupPath -> groupRowsPath -> rows`) instead of prefix-slice predicates like `keyPathParent[1:cardinality(...)] = ...`. + +Q: What query shape should Bulldozer use to list all rows without scaling with entire `BulldozerStorageEngine` size? +A: For table-scoped all-groups reads, use equality-join traversal rooted at that table's groups path: `groupPath (keyPathParent = groupsPath) -> groupRowsPath (keyPathParent = groupPath.keyPath and leaf = 'rows') -> rows (keyPathParent = groupRowsPath.keyPath)`. Avoid prefix slicing on `keyPathParent` (`[1:cardinality(...)] = ...`), which can force broad scans over unrelated tables. + Q: Why did EventTracker throw `Reflect.get called on non-object` in JS cookie tests? A: Partial browser mocks can expose `window` without a real `history` object. Calling `Reflect.get(historyObject, "pushState")` throws before type checks. Use normal guarded access (`Object.getOwnPropertyDescriptor(window, "history")?.value`) plus type guards for `pushState`/`replaceState`, and patch/restore methods directly without `Reflect`. @@ -154,5 +172,59 @@ A: In `packages/template/src/components-page/stack-handler-client.tsx`, parse ha Q: What is the current `app.urls` contract after deprecating runtime URL mutation? A: `app.urls` is now static (`getUrls(...)` only) and no longer injects runtime `after_auth_return_to` / `stack_cross_domain_*` params from `window.location`. For navigation flows, examples and consumers should use `redirectToXyz()` methods instead (for example `redirectToSignIn()` / `redirectToSignOut()`), while tests for hosted flows should assert dynamic params on actual redirect methods, not on `app.urls`. +Q: What is the fastest safe way to delete a Bulldozer table subtree from `BulldozerStorageEngine`? +A: Delete only the table root `keyPath` and rely on the existing `keyPathParent -> keyPath ON DELETE CASCADE` FK to remove descendants. This avoids recursive CTE path enumeration and significantly speeds up large deletes while preserving semantics. + +Q: How should `declareLimitTable.listRowsInGroup` implement the all-groups read path for performance? +A: Read directly from the materialized limit table subtree (`groups -> rows` via `keyPathParent` equality joins) and apply range predicates on stored `rowSortKey`, instead of scanning upstream source rows and semi-joining with `EXISTS` on each row. This keeps behavior but removes an avoidable full-source scan. + Q: How should user signup time be exposed in JWT claims before production rollout? A: Use `signed_up_at` (OIDC-style naming) in access tokens and encode it as Unix seconds in `apps/backend/src/lib/tokens.tsx` (`Math.floor(user.signed_up_at_millis / 1000)`). Since this is pre-prod, the payload schema can require `signed_up_at` directly without a backward-compat optional shim. + +Q: Why did adding `signed_up_at` to the access token payload break backend typecheck? +A: `AccessTokenPayload` currently does not include `signed_up_at`. In `apps/backend/src/lib/tokens.tsx`, `payload` is typed as `Omit`, so extra fields fail with `TS2353`. Until the schema/type is updated consistently, keep `signed_up_at` out of the payload object. + +Q: How should Bulldozer Studio mutation endpoints be hardened? +A: In `apps/backend/scripts/run-bulldozer-studio.ts`, enforce loopback-only requests, require a per-instance mutation token header (for all POST routes), bound request body size before buffering/JSON parse, and ensure raw writes use the same advisory transaction lock as other table mutations. For raw upsert correctness, insert missing parent key paths before upserting the leaf node. + +Q: What is the new `declareLeftJoinTable` API contract and why was it changed? +A: `declareLeftJoinTable` now takes `leftJoinKey` and `rightJoinKey` SQL mappers (each producing a `joinKey`) instead of an arbitrary `on` predicate. Join rows are matched when `leftJoinKey IS NOT DISTINCT FROM rightJoinKey` within the same group. This removes custom non-equality predicates, enables planner-friendly equality joins, and keeps null-key matching explicit (`IS NOT DISTINCT FROM`). + +Q: What does `listRowsInGroup` return regarding `groupKey` and what pitfall was fixed? +A: In Bulldozer, all-groups row queries can include `groupKey`, while specific-group queries may omit it. A bug in `declareStoredTable.listRowsInGroup` ignored the provided `groupKey` and did not expose `groupKey` for all-groups reads. It now returns `'null'::jsonb AS groupKey` for all-groups reads and correctly filters specific-group reads to only the null group (`groupKey IS NOT DISTINCT FROM 'null'::jsonb`). + +Q: How should Bulldozer materialized operators manage upstream trigger registrations across init/delete? +A: Register upstream row-change triggers lazily in `init()` (via an idempotent `ensure...Registration` helper), store deregistration handles, and call those `deregister()` functions in `delete()`. This avoids leaked/no-op trigger callbacks after table teardown while still allowing re-initialization to re-register subscriptions. + +Q: How can we test trigger registration lifecycle behavior without depending on database row changes? +A: In `apps/backend/src/lib/bulldozer/db/index.test.ts`, wrap input tables with an instrumentation helper that intercepts `registerRowChangeTrigger`, counts `register`/`deregister` calls, and tracks active registrations. Then assert `init()` registers exactly once per input, repeated `init()` is idempotent, `delete()` deregisters, and re-`init()` re-registers. + +Q: Why can `declareConcatTable` ignore input sort comparator differences? +A: `declareConcatTable` always emits `rowSortKey = null` and uses `compareSortKeys: () => 0` itself, so input sort-order semantics are not part of concat output behavior. It should only enforce group-key comparator compatibility, not sort comparator compatibility. + +Q: How should flaky subset-iteration perf assertions be stabilized? +A: In `apps/backend/src/lib/bulldozer/db/index.perf.test.ts`, keep a warmup query, then measure multiple timed runs (for example 5) and assert on average latency instead of a single run. Log average, standard deviation, variance, min, and max so regressions still show up while reducing one-off outlier failures. + +Q: What if multi-run average still flakes because of one or two large outliers? +A: Use robust stats for thresholds: keep logging full `avg/stddev/variance/min/max`, but assert subset-iteration performance on `trimmedAverage` (drop one min/max sample when there are 5 runs). This preserves sensitivity to sustained regressions while tolerating transient host contention spikes during concurrent test-file execution. + +Q: How should `declareTimeFoldTable` row identifiers and SQL aliases behave? +A: `declareTimeFoldTable` emits expanded output identifiers with a flat-row suffix (for example `sourceRowId:1` even when one row is emitted), matching other fold-style operators. For query outputs, use unquoted aliases (`AS groupKey`, `AS rowIdentifier`, etc.) if later clauses reference them (`ORDER BY groupKey, rowIdentifier`) to avoid case-sensitive alias lookup errors in Postgres. + +Q: Why can Bulldozer Studio show initialized derived tables that do not react to new stored-table mutations? +A: Trigger registrations are in-memory and are established in table `init()`. If the DB already has initialized derived tables from a previous Studio process, a fresh Studio process can report `initialized: true` from storage while lacking active trigger subscriptions. In `run-bulldozer-studio.ts`, rebind initialized derived tables at startup by deleting and re-initializing them in dependency order so subscriptions are re-registered. + +Q: How can I inspect the `declareTimeFoldTable` scheduler state in Bulldozer Studio? +A: Use the new `⏱️ Timefold` mode in `apps/backend/scripts/run-bulldozer-studio.ts`. It calls `/api/timefold/debug`, which reports whether `BulldozerTimeFoldQueue` and `BulldozerTimeFoldMetadata` exist, the metadata `lastProcessedAt` value, and up to 500 queued rows (including `scheduledAt`, `stateAfter`, `rowData`, and `reducerSql`) ordered by scheduled execution time. + +Q: Why can timefold queue rows remain overdue even though the reducer function exists? +A: The migration creates the queue processor function regardless of `pg_cron`, but `pg_cron` setup is best-effort and can be skipped (for example if `cron.job` is unavailable). In that state, `BulldozerTimeFoldQueue` grows while `lastProcessedAt` stops moving until `public.bulldozer_timefold_process_queue()` is called manually or `pg_cron` is installed/configured correctly. + +Q: How do we ensure `pg_cron` is actually available in local dev Postgres? +A: In `docker/dev-postgres-with-extensions/Dockerfile`, install `postgresql-15-cron`, add `pg_cron` to `shared_preload_libraries`, set `cron.database_name='stackframe'`, and create the extension during init (`CREATE EXTENSION pg_cron;`). After `pnpm run restart-deps`, `to_regclass('cron.job')` should be non-null and `cron.job_run_details` should show the `bulldozer-timefold-worker` running every second. + +Q: How does Bulldozer Studio "init all" work? +A: `apps/backend/scripts/run-bulldozer-studio.ts` now exposes `POST /api/tables/init-all`, which initializes only non-initialized tables in topological dependency order derived from table snapshots. The toolbar has a `πŸš€ init all` button that calls this endpoint and refreshes schema/details afterward. + +Q: What are safe reducer practices for `declareTimeFoldTable`, and how do timed reruns affect outputs? +A: Timefold reducers should avoid non-deterministic values (`now()`, random) for output-driving logic; prefer stable row timestamps and prior reducer timestamps so replay/re-init stays deterministic. Timed reruns now append newly emitted rows on top of existing emitted rows for a source row (instead of replacing prior timed outputs), while source updates/deletes still recompute/reset that source row’s materialized outputs. diff --git a/docker/dev-postgres-with-extensions/Dockerfile b/docker/dev-postgres-with-extensions/Dockerfile index b9d9a2f98c..0bfe255845 100644 --- a/docker/dev-postgres-with-extensions/Dockerfile +++ b/docker/dev-postgres-with-extensions/Dockerfile @@ -4,7 +4,8 @@ RUN apt-get update && apt-get install -y \ git \ build-essential \ libpq-dev \ - postgresql-server-dev-15 + postgresql-server-dev-15 \ + postgresql-15-cron # Install HypoPG RUN git clone https://github.com/HypoPG/hypopg.git /hypopg @@ -16,6 +17,7 @@ RUN cd /index_advisor && make install # Write initialization SQL RUN echo "CREATE EXTENSION pg_stat_statements;" >> /docker-entrypoint-initdb.d/init.sql +RUN echo "CREATE EXTENSION pg_cron;" >> /docker-entrypoint-initdb.d/init.sql RUN echo "CREATE EXTENSION hypopg;" >> /docker-entrypoint-initdb.d/init.sql RUN echo "CREATE EXTENSION index_advisor;" >> /docker-entrypoint-initdb.d/init.sql RUN echo "CREATE ROLE anon;" >> /docker-entrypoint-initdb.d/init.sql @@ -46,7 +48,8 @@ ENTRYPOINT ["sh", "-c", "\ \ # Start Postgres with replication enabled and extensions \ exec docker-entrypoint.sh postgres \ - -c shared_preload_libraries='pg_stat_statements' \ + -c shared_preload_libraries='pg_stat_statements,pg_cron' \ + -c cron.database_name='stackframe' \ -c pg_stat_statements.track=all \ -c wal_level=logical \ -c max_wal_senders=5 \ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 327333ea85..15eddec930 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -228,6 +228,9 @@ importers: dotenv-cli: specifier: ^7.3.0 version: 7.4.1 + elkjs: + specifier: ^0.11.1 + version: 0.11.1 emailable: specifier: ^3.1.1 version: 3.1.1 @@ -11469,6 +11472,9 @@ packages: electron-to-chromium@1.5.89: resolution: {integrity: sha512-okLMJSmbI+XHr8aG+wCK+VPH+d38sHMED6/q1CTsCNkqfdOZL3k2ThWnh44HL6bJKj9cabPCSVLDv9ynsIm8qg==} + elkjs@0.11.1: + resolution: {integrity: sha512-zxxR9k+rx5ktMwT/FwyLdPCrq7xN6e4VGGHH8hA01vVYKjTFik7nHOxBnAYtrgYUB1RpAiLvA1/U2YraWxyKKg==} + elliptic@6.5.7: resolution: {integrity: sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q==} @@ -28630,6 +28636,8 @@ snapshots: electron-to-chromium@1.5.89: {} + elkjs@0.11.1: {} + elliptic@6.5.7: dependencies: bn.js: 4.12.0 @@ -29173,7 +29181,7 @@ snapshots: debug: 4.4.3 enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) fast-glob: 3.3.3 get-tsconfig: 4.8.1 is-bun-module: 1.2.1 @@ -29216,7 +29224,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: @@ -29294,7 +29302,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.15.1 is-glob: 4.0.3