feat: update XML parsing functions to return promises; refactor imports for dynamic loading

Author: pelikhan

packages/core/src/chat.ts

@@ -54,7 +54,6 @@ import { resolveTokenEncoder } from "./encoders.js";
 import { approximateTokens, truncateTextToTokens } from "./tokens.js";
 import { computeFileEdits } from "./fileedits.js";
 import { HTMLEscape } from "./htmlescape.js";
-import { XMLTryParse } from "./xml.js";
 import {
   computePerplexity,
   computeStructuralUncertainty,
@@ -1419,7 +1418,7 @@ export function tracePromptResult(
   if (text) {
     const language = JSON5TryParse(text)
       ? "json"
-      : XMLTryParse(text)
+      : /^</.test(text)
         ? "xml"
         : /^(-|\*|#+|```)\s/im.test(text)
           ? "markdown"

packages/core/src/data.ts

@@ -58,7 +58,7 @@ export async function dataTryParse(
     else if (TOML_REGEX.test(filename)) data = TOMLTryParse(content);
     else if (JSON5_REGEX.test(filename)) data = JSON5TryParse(content, { repair: true });
     else if (YAML_REGEX.test(filename)) data = YAMLTryParse(content);
-    else if (XML_REGEX.test(filename)) data = XMLTryParse(content, options);
+    else if (XML_REGEX.test(filename)) data = await XMLTryParse(content, options);
     else if (JSONL_REGEX.test(filename)) data = JSONLTryParse(content);
     else if (MD_REGEX.test(filename) || MDX_REGEX.test(filename))
       data = YAMLTryParse(splitMarkdown(content).frontmatter);

packages/core/src/docx.ts

@@ -17,7 +17,6 @@ import { ensureDir } from "./fs.js";
 import { measure } from "./performance.js";
 import { dotGenaiscriptPath } from "./workdir.js";
 import type { DocxParseOptions, WorkspaceFile } from "./types.js";
-import { extractRawText, convertToHtml } from "mammoth";
 
 async function computeHashFolder(
   filename: string,
@@ -74,6 +73,8 @@ export async function DOCXTryParse(
   try {
     const input = content ? { buffer: Buffer.from(content) } : { path: host.resolvePath(filename) };
 
+    const { extractRawText, convertToHtml } = await import("mammoth");
+
     let text: string;
     if (format === "html" || format === "markdown") {
       const results = await convertToHtml(input);

packages/core/src/ffmpeg.ts

@@ -17,7 +17,7 @@ import { writeFile, readFile } from "node:fs/promises";
 import { errorMessage, serializeError } from "./error.js";
 import { fromBase64 } from "./base64.js";
 import { fileTypeFromBuffer } from "./filetype.js";
-import { appendFile, readdir, stat } from "node:fs/promises";
+import { appendFile, readdir } from "node:fs/promises";
 import prettyBytes from "pretty-bytes";
 import { filenameOrFileToFilename } from "./unwrappers.js";
 import { roundWithPrecision } from "./precision.js";
@@ -37,7 +37,6 @@ import type {
   VideoProbeResult,
   WorkspaceFile,
 } from "./types.js";
-import cmd from "fluent-ffmpeg";
 
 const ffmpegLimit = pLimit(1);
 const WILD_CARD = "%06d";
@@ -53,6 +52,7 @@ interface FFmpegCommandResult {
 }
 
 async function ffmpegCommand(options?: { timeout?: number }) {
+  const cmd = (await import("fluent-ffmpeg")).default;
   return cmd(options);
 }
 
@@ -233,7 +233,7 @@ export class FFmepgClient implements Ffmpeg {
     }
     const res = await this.run(
       filename,
-      async (cmd, fopts) => {
+      async (cmd) => {
         cmd.noVideo();
         if (transcription) {
           // https://community.openai.com/t/whisper-api-increase-file-limit-25-mb/566754

packages/core/src/math.ts

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 import { TraceOptions } from "./trace.js";
-import { evaluate } from "mathjs";
 
 /**
  * Asynchronously evaluates a mathematical expression.
@@ -28,6 +27,8 @@ export async function MathTryEvaluate(
     // Return defaultValue if expression is empty
     if (!expr) return defaultValue;
 
+    const { evaluate } = await import("mathjs");
+
     // Evaluate the expression and return the result
     const res = evaluate(expr, scope);
     return res;

packages/core/src/parsers.ts

@@ -94,10 +94,10 @@ export function createParsers(): Parsers {
         YAMLTryParse(filenameOrFileToContent(text), options?.defaultValue),
         options,
       ),
-    XML: (text, options) => {
+    XML: async (text, options) => {
       const { defaultValue, ...rest } = options || {};
       return tryValidateJSONWithSchema(
-        XMLTryParse(filenameOrFileToContent(text), defaultValue, rest),
+        await XMLTryParse(filenameOrFileToContent(text), defaultValue, rest),
         options,
       );
     },

packages/core/src/types.ts

@@ -3265,7 +3265,7 @@ export interface XMLObject {
    * Parses an XML payload to an object
    * @param text
    */
-  parse(text: string | WorkspaceFile, options?: XMLParseOptions): any;
+  parse(text: string | WorkspaceFile, options?: XMLParseOptions): Promise<any>;
 }
 
 export interface JSONSchemaUtilities {

packages/core/src/workspace.ts

@@ -135,7 +135,7 @@ export function createWorkspaceFileSystem(): Omit<WorkspaceFileSystem, "grep" |
     },
     readXML: async (f: string | Awaitable<WorkspaceFile>, options?: XMLParseOptions) => {
       const file = await fs.readText(f);
-      const res = XMLTryParse(file.content, options);
+      const res = await XMLTryParse(file.content, options);
       return tryValidateJSONWithSchema(res, options);
     },
     readCSV: async <T extends object>(

packages/core/src/xlsx.ts

@@ -4,7 +4,6 @@
 // Import the logInfo function for logging purposes
 import { logInfo } from "./util.js";
 import type { ParseXLSXOptions, WorkbookSheet } from "./types.js";
-import { read, utils } from "xlsx";
 
 /**
  * Parses XLSX data into an array of workbook sheets.
@@ -20,6 +19,7 @@ export async function XLSXParse(
   // Destructure options to separate sheet-specific option
   const { sheet, ...rest } = options || {};
   // Dynamically import 'xlsx' library's read and utils modules
+  const { read, utils } = await import("xlsx");
   // Read the workbook from the data with 'array' type
   const workbook = read(data, { type: "array" });
   // Filter and map the sheet names to WorkbookSheet objects

packages/core/src/xml.ts

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { XMLParser } from "fast-xml-parser";
 import { unfence } from "./unwrappers.js";
 import { filenameOrFileToContent } from "./unwrappers.js";
 import type { WorkspaceFile, XMLParseOptions } from "./types.js";
@@ -16,14 +15,14 @@ const dbg = genaiscriptDebug("xml");
  * @param options - Optional configuration for the XML parser
  * @returns The parsed XML object or defaultValue if an error occurs
  */
-export function XMLTryParse(
+export async function XMLTryParse(
   text: string | WorkspaceFile,
   defaultValue?: unknown,
   options?: XMLParseOptions,
 ) {
   try {
     // Try parsing the text and return the result or defaultValue
-    return XMLParse(text, options) ?? defaultValue;
+    return (await XMLParse(text, options)) ?? defaultValue;
   } catch (e) {
     // Return the default value if parsing fails
     dbg(`error: %s`, e?.message);
@@ -38,11 +37,13 @@ export function XMLTryParse(
  * @param options - Configuration options for the XML parser. These options are merged with the default parser settings.
  * @returns The parsed XML object.
  */
-export function XMLParse(text: string | WorkspaceFile, options?: XMLParseOptions) {
+export async function XMLParse(text: string | WorkspaceFile, options?: XMLParseOptions) {
   text = filenameOrFileToContent(text);
   // Remove specific markers from the XML string for cleaner processing
   const cleaned = unfence(text, "xml");
 
+  const { XMLParser } = await import("fast-xml-parser");
+
   // Create a new XMLParser instance with the specified options
   const parser = new XMLParser({
     ignoreAttributes: false, // Do not ignore XML attributes