[GR-73863] Support lazy loading of source texts on DSL interpreter

PullRequest: graalpython/4302

Author: msimacek

graalpython/AGENTS.md

@@ -19,6 +19,37 @@ Main implementation tree: Java (Truffle interpreter), C (CPython C-API compatibi
 - This subtree contains both “source of truth” code and vendored/upstream-ish imports; keep patches minimal in `lib-python/` and large C module imports.
 - Some large headers/databases (unicode tables) are generated; avoid editing them by hand unless you also update the generator pipeline.
 
+## RUNNING
+
+- Python code can be executed with GraalPy using `mx python`, invoked just like normal `python` command. The project
+  *must* be first built with `mx python-jvm` or you will execute stale code. Note that `mx python-jvm` just builds, it
+  doesn't take arguments nor execute code.
+
+## TESTING
+
+- There are multiple kinds of tests:
+    - GraalPy Python tests
+        - Our own tests in `com.oracle.graal.python.test/src/tests/`
+            - Executed with `mx graalpytest test_file_name`
+            - New test should normally be added here, unless they need to be in Java
+    - CPython tests, also called tagged tests
+        - Tests copied from upstream CPython in `lib-python/3/tests`. Should not be modified unless specifically
+          requested. If modified, modifications should be marked with a `# GraalPy change` comment above the changed
+          part.
+        - Executed with `mx graalpytest --tagged test_file_name`
+        - Uses a "tagging" system where only a subset of tests specified in tag files is normally executed. The `--all`
+          flag makes it ignore the tags and execute all tests.
+    - JUnit tests
+        - In `com.oracle.graal.python.test/src` and `com.oracle.graal.python.test.integration/src`
+        - Used primarily for testing features exposed to Java, such as embedding, instrumentation or interop.
+        - The tests need to be built with `mx build` prior to execution. The `mx unittest com.example.TestName` command
+          can be used to run individual tests.
+- The `mx graalpytest` command accepts pytest‑style test selectors (e.g., `test_mod.py::TestClass::test_method`) but is
+  **not** a full pytest implementation. Standard pytest command‑line flags such as `-k`, `-m`, `-v`, `--maxfail` are not
+  supported.
+- Important: The test commands don't automatically rebuild the project. It is your reponsibility to rebuild the project
+  using `mx python-jvm` after making changes prior to running tests otherwise the tests will run stale code.
+
 ## ANTI-PATTERNS
 - Don’t use `mxbuild/**` outputs to understand behavior; always navigate `.../src/...` trees.
 - C-API: never mix `PyMem_*` / `PyObject_*` allocators with platform `malloc` family.

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java

@@ -550,6 +550,15 @@ public void testInspectJavaArray() throws Throwable {
 
     @Test
     public void testSourceFileURI() throws Throwable {
+        testSourceFileURIImpl(false);
+    }
+
+    @Test
+    public void testSourceFileURIBytecode() throws Throwable {
+        testSourceFileURIImpl(true);
+    }
+
+    private void testSourceFileURIImpl(boolean runFromBytecode) throws Throwable {
         if (System.getProperty("os.name").toLowerCase().contains("mac")) {
             // on the mac machines we run with symlinked directories and such, and it's annoying to
             // cater for that
@@ -565,6 +574,11 @@ public void testSourceFileURI() throws Throwable {
                             "sys.path.insert(0, '" + tempDir.toString() + "')\n" +
                             "import imported\n" +
                             "imported.sum(2, 3)\n").getBytes());
+
+            if (runFromBytecode) {
+                compileToBytecode(importedFile, importingFile);
+            }
+
             Source source = Source.newBuilder("python", importingFile.toFile()).build();
             try (DebuggerSession session = tester.startSession()) {
                 Breakpoint breakpoint = Breakpoint.newBuilder(importingFile.toUri()).lineIs(4).build();
@@ -602,6 +616,16 @@ public void testSourceFileURI() throws Throwable {
         }
     }
 
+    private void compileToBytecode(Path... files) {
+        StringBuilder sourceCode = new StringBuilder("import py_compile\n");
+        for (Path file : files) {
+            sourceCode.append("py_compile.compile(r\"").append(file).append("\")\n");
+        }
+        Source compileSource = Source.newBuilder("python", sourceCode.toString(), "compile_source_uri.py").buildLiteral();
+        tester.startEval(compileSource);
+        tester.expectDone();
+    }
+
     @Test
     public void testInlineEvaluationBreakpointBuiltin() throws Throwable {
         final Source source = Source.newBuilder("python", """

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -27,7 +27,6 @@
 
 import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_WIN32;
 import static com.oracle.graal.python.nodes.BuiltinNames.T__SIGNAL;
-import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION;
 import static com.oracle.graal.python.nodes.StringLiterals.T_PY_EXTENSION;
 import static com.oracle.graal.python.nodes.truffle.TruffleStringMigrationHelpers.isJavaString;
 import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR;
@@ -110,7 +109,6 @@
 import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.graal.python.runtime.object.PFactory;
 import com.oracle.graal.python.util.Function;
-import com.oracle.graal.python.util.LazySource;
 import com.oracle.graal.python.util.PythonUtils;
 import com.oracle.graal.python.util.Supplier;
 import com.oracle.truffle.api.Assumption;
@@ -367,6 +365,12 @@ public boolean isSingleContext() {
      */
     private final ConcurrentHashMap<Object, Source> sourceCache = new ConcurrentHashMap<>();
 
+    /*
+     * A map from sources without content to either a Source object with content or a TruffleFile
+     * that can be used to construct such object.
+     */
+    private final WeakHashMap<Source, Object> originalSources = new WeakHashMap<>();
+
     @Idempotent
     public static PythonLanguage get(Node node) {
         return REFERENCE.get(node);
@@ -548,54 +552,17 @@ protected CallTarget parse(ParsingRequest request) {
         return parse(context, source, inputType, topLevel, optimize, interactiveTerminal, argumentNames, futureFeatures);
     }
 
-    public static RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) {
-        boolean internal = shouldMarkSourceInternal(context);
-        SourceBuilder builder = null;
-        // The original file path should be passed as the name
-        String name = source.getName();
-        if (name != null && !name.isEmpty()) {
-            builder = sourceForOriginalFile(context, code, internal, name);
-            if (builder == null) {
-                if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) {
-                    String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length());
-                    String fs = context.getEnv().getFileNameSeparator();
-                    String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION;
-                    builder = sourceForOriginalFile(context, code, internal, path);
-                    if (builder == null) {
-                        path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py";
-                        builder = sourceForOriginalFile(context, code, internal, path);
-                    }
-                }
-            }
-        }
-        if (builder == null) {
-            builder = Source.newBuilder(source).internal(internal).content(Source.CONTENT_NONE);
-        }
+    public RootCallTarget callTargetFromBytecode(Source source, CodeUnit code) {
         RootNode rootNode;
-        LazySource lazySource = new LazySource(builder);
-
         if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
-            // TODO lazily load source in bytecode DSL interpreter too
-            rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, lazySource.getSource());
+            rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(this, source);
         } else {
-            rootNode = PBytecodeRootNode.create(context.getLanguage(), (BytecodeCodeUnit) code, lazySource, internal);
+            rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source, source.isInternal());
         }
 
         return PythonUtils.getOrCreateCallTarget(rootNode);
     }
 
-    private static SourceBuilder sourceForOriginalFile(PythonContext context, CodeUnit code, boolean internal, String path) {
-        try {
-            TruffleFile file = context.getEnv().getPublicTruffleFile(path);
-            if (!file.isReadable()) {
-                return null;
-            }
-            return Source.newBuilder(PythonLanguage.ID, file).name(code.name.toJavaStringUncached()).internal(internal);
-        } catch (SecurityException | UnsupportedOperationException | InvalidPathException e) {
-            return null;
-        }
-    }
-
     public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames,
                     EnumSet<FutureFeature> futureFeatures) {
         return parse(context, source, type, topLevel, optimize, interactiveTerminal, false, argumentNames, futureFeatures);
@@ -677,7 +644,7 @@ private RootNode compileForBytecodeInterpreter(ModTy mod, Source source, int opt
         Compiler compiler = new Compiler(parserCallbacks);
         CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures);
         BytecodeCodeUnit co = cu.assemble();
-        return PBytecodeRootNode.create(this, co, new LazySource(source), source.isInternal(), parserCallbacks);
+        return PBytecodeRootNode.create(this, co, source, source.isInternal(), parserCallbacks);
     }
 
     private RootNode compileForBytecodeDSLInterpreter(ModTy mod, Source source, int optimize,
@@ -957,7 +924,7 @@ private static Source newSource(PythonContext context, SourceBuilder srcBuilder)
         return srcBuilder.build();
     }
 
-    private static boolean shouldMarkSourceInternal(PythonContext ctxt) {
+    public static boolean shouldMarkSourceInternal(PythonContext ctxt) {
         return !ctxt.isCoreInitialized() && !ctxt.getLanguage().getEngineOption(PythonOptions.ExposeInternalSources);
     }
 
@@ -1259,6 +1226,36 @@ public Source getOrCreateSource(Function<Object, Source> rootNodeFunction, Objec
         return sourceCache.computeIfAbsent(key, rootNodeFunction);
     }
 
+    public Source getOrCreateSourceWithContent(Source sourceWithoutContent) {
+        if (sourceWithoutContent.hasCharacters()) {
+            return sourceWithoutContent;
+        }
+        synchronized (originalSources) {
+            Object original = originalSources.get(sourceWithoutContent);
+            if (original instanceof Source originalSource) {
+                return originalSource;
+            }
+            if (original instanceof TruffleFile originalFile) {
+                Source source;
+                try {
+                    source = Source.newBuilder(ID, originalFile).name(sourceWithoutContent.getName()).internal(sourceWithoutContent.isInternal()).mimeType(MIME_TYPE).build();
+                } catch (IOException | SecurityException | UnsupportedOperationException | IllegalArgumentException e) {
+                    source = sourceWithoutContent;
+                }
+                originalSources.put(sourceWithoutContent, source);
+                return source;
+            }
+            assert original == null;
+            return sourceWithoutContent;
+        }
+    }
+
+    public void registerOriginalFile(Source sourceWithoutContent, TruffleFile originalFile) {
+        synchronized (originalSources) {
+            originalSources.put(sourceWithoutContent, originalFile);
+        }
+    }
+
     public static PythonOS getPythonOS() {
         if (PythonOS.internalCurrent == PythonOS.PLATFORM_ANY) {
             if (ImageInfo.inImageBuildtimeCode()) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -507,7 +507,7 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath,
             try {
                 // get_data
                 TruffleString strBytecodePath = PyObjectStrAsTruffleStringNode.executeUncached(bytecodePath);
-                TruffleFile bytecodeFile = context.getEnv().getPublicTruffleFile(strBytecodePath.toJavaStringUncached());
+                TruffleFile bytecodeFile = context.getPublicTruffleFileRelaxed(strBytecodePath);
                 byte[] bytes = bytecodeFile.readAllBytes();
                 // _classify_pyc
                 if (bytes.length < 16 || !Arrays.equals(bytes, 0, 4, MAGIC_NUMBER_BYTES, 0, 4)) {
@@ -560,7 +560,16 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath,
                     Object message = PyObjectCallMethodObjArgs.executeUncached(MESSAGE, T_FORMAT, bytecodePath, sourcePath);
                     CallNode.executeUncached(context.lookupBuiltinModule(T__BOOTSTRAP).getAttribute(T__VERBOSE_MESSAGE), message);
                 }
-                return MarshalModuleBuiltins.fromBytecodeFile(context, bytecodeFile, bytes, 16, bytes.length - 16, cacheKey);
+                TruffleFile sourceFile = null;
+                if (sourcePath != PNone.NONE) {
+                    try {
+                        TruffleString strSourcePath = PyObjectStrAsTruffleStringNode.executeUncached(sourcePath);
+                        sourceFile = context.getPublicTruffleFileRelaxed(strSourcePath);
+                    } catch (SecurityException | UnsupportedOperationException | IllegalArgumentException ignored) {
+                        // Fall back to Marshal's empty source.
+                    }
+                }
+                return MarshalModuleBuiltins.fromBytecodeFile(context.getLanguage(), bytecodeFile, sourceFile, bytes, 16, bytes.length - 16, cacheKey);
             } catch (MarshalModuleBuiltins.Marshal.MarshalError me) {
                 throw PRaiseNode.raiseStatic(inliningTarget, me.type, me.message, me.arguments);
             } catch (IOException | SecurityException | UnsupportedOperationException | IllegalArgumentException e) {