support package:module (#1372)

Author: pelikhan

.github/workflows/build.yml

@@ -36,3 +36,5 @@ jobs:
               run: yarn test:core
             - name: unit tests
               run: yarn test:samples
+            - name: unit tests (2)
+              run: yarn test:modulesamples

package.json

@@ -41,6 +41,7 @@
         "debug:mcp": "npx --yes @modelcontextprotocol/inspector node packages/cli/built/genaiscript.cjs mcp --groups mcp --cwd packages/sample",
         "test:core": "cd packages/core && yarn test",
         "test:samples": "cd packages/sample && yarn test",
+        "test:modulesamples": "cd packages/modulesample && yarn test",
         "test:cli": "node packages/cli/built/genaiscript.cjs run code-annotator packages/sample/src/counting.py -l Test -o .genaiscript/tmp/tests/cli -ot .genaiscript/tmp/tests/cli/outtrace.md -oa .genaiscript/tmp/tests/cli/diags.sarif",
         "test:live": "node packages/cli/built/genaiscript.cjs run code-annotator packages/sample/src/counting.py -l Test -o .genaiscript/tmp/tests/cli --retry 1 --temperature 0.5 ",
         "test:front-matter": "node packages/cli/built/genaiscript.cjs run front-matter SUPPORT.md ",
@@ -56,6 +57,7 @@
         "test:fix": "node packages/cli/built/genaiscript.cjs scripts fix",
         "test:infomodel": "node packages/cli/built/genaiscript.cjs scripts model",
         "test:phi3": "node packages/cli/built/genaiscript.cjs run summarize-ollama-phi3 packages/sample/src/rag/markdown.md",
+        "genai:module": "cd packages/modulesample && node ../cli/built/genaiscript.cjs run",
         "run:script": "cd packages/sample/ && yarn run:script",
         "run:script:debug": "yarn compile-debug && cd packages/sample/ && yarn run:script",
         "cache:clear": "cd packages/sample/ && yarn cache:clear",

packages/core/src/evalprompt.ts

@@ -1,3 +1,6 @@
+import debug from "debug"
+const dbg = debug("genaiscript:evalprompt")
+
 import { host } from "./host"
 import MagicString from "magic-string"
 
@@ -21,9 +24,11 @@ export async function evalPrompt(
     let src: string = [prefix, jsSource, suffix].join("")
     // source map
     if (r.filename && sourceMaps) {
+        dbg("creating source map")
         const s = new MagicString(jsSource)
         s.prepend(prefix)
         s.append(suffix)
+        dbg(`resolving path for ${r.filename}`)
         const source = host.path.resolve(r.filename)
         const map = s.generateMap({
             source,
@@ -33,11 +38,13 @@ export async function evalPrompt(
         const mapURL: string = map.toUrl()
         // split keywords as so that JS engine does not try to load "mapUrl"
         src += "\n//# source" + "MappingURL=" + mapURL
+        dbg("appending sourceURL to source")
         src += "\n//# source" + "URL=" + source
     }
 
     // in principle we could cache this function (but would have to do that based on hashed body or sth)
     // but probably little point
     const fn = (0, eval)(src)
+    dbg(`eval ${r.filename}`)
     return await fn(...Object.values(ctx))
 }

packages/core/src/expander.ts

@@ -29,6 +29,7 @@ import { normalizeFloat, normalizeInt } from "./cleaners"
 import { mergeEnvVarsWithSystem } from "./vars"
 import { installGlobalPromptContext } from "./globals"
 import { mark } from "./performance"
+import { tryReadJSON } from "./fs"
 
 export async function callExpander(
     prj: Project,
@@ -62,10 +63,14 @@ export async function callExpander(
         logs += msg + "\n"
     }
 
+    // package.json { type: "module" }
+    const pkg = await tryReadJSON("package.json")
+    const isModule = pkg?.type === "module"
+    dbg(`package.json type: ${pkg?.type || ""}`)
     try {
         if (
             r.filename &&
-            !JS_REGEX.test(r.filename) &&
+            (isModule || !JS_REGEX.test(r.filename)) &&
             !PROMPTY_REGEX.test(r.filename)
         )
             await importPrompt(ctx, r, { logCb, trace })

packages/core/src/importprompt.ts

@@ -1,3 +1,6 @@
+import debug from "debug"
+const dbg = debug("genaiscript:importprompt")
+
 import { host } from "./host"
 import { logError } from "./util"
 import { TraceOptions } from "./trace"
@@ -12,10 +15,13 @@ export async function importFile<T = void>(
     } & TraceOptions
 ): Promise<T> {
     const { trace, onImported } = options || {}
-    if (!filename) throw new Error("filename is required")
+    if (!filename) {
+        throw new Error("filename is required")
+    }
 
     let unregister: () => void = undefined
     try {
+        dbg(`resolving module path for filename: ${filename}`)
         const modulePath = pathToFileURL(
             host.path.isAbsolute(filename)
                 ? filename
@@ -25,6 +31,7 @@ export async function importFile<T = void>(
             import.meta.url ??
             pathToFileURL(__filename ?? host.projectFolder()).toString()
 
+        dbg(`importing module from path: ${modulePath}`)
         const onImport = (file: string) => {
             // trace?.itemValue("📦 import", fileURLToPath(file))
         }
@@ -41,6 +48,7 @@ export async function importFile<T = void>(
 
         return result
     } catch (err) {
+        dbg("module imported failed")
         unregister?.()
         logError(err)
         trace?.error(err)
@@ -57,15 +65,19 @@ export async function importPrompt(
 ) {
     mark("prompt.import")
     const { filename } = r
+    dbg(`importing file: ${filename}`)
     return await importFile(filename, {
         ...(options || {}),
         onImported: async (module) => {
             const main = module.default
-            if (typeof main === "function") await main(ctx0)
-            else if (r.isSystem)
+            if (typeof main === "function") {
+                dbg(`found default export as function, calling`)
+                await main(ctx0)
+            } else if (r.isSystem) {
                 throw new Error(
                     "system prompt using esm JavaScript (mjs, mts) must have a default function."
                 )
+            }
         },
     })
 }

packages/modulesample/genaiscript/.gitignore

@@ -0,0 +1,3 @@
+genaiscript.d.ts
+tsconfig.json
+jsconfig.json

packages/modulesample/genaiscript/poem-js-module.genai.js

@@ -0,0 +1,4 @@
+import { parse } from "ini"
+script({ model: "echo" })
+const v = parse(`A = B`)
+$`write a short poem about ${v}`

packages/modulesample/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "genaiscript-modulesample",
+    "type": "module",
+    "scripts": {
+        "genai": "node ../cli/built/genaiscript.cjs run",
+        "test": "yarn genai poem-js-module"
+    }
+}