More carefully encapsulate the compiler cache within NativeCompiler.

We want for NativeCompiler to provide a simple, clean abstraction for
a code-compilation service. You should be able to push code in, ask whether
you've pushed code before, and get TypedCallTargets and function pointers
out, and not really need to worry about the compiler cache.

This change moves the compiler cache fully behind the native compiler,
and tries to ensure that the python converter doesn't itself need to
track the functions that its compiled in the past. Instead, we're moving
towards it being stateless across invocations (and we'll probably do
the same thing for the native-to-llvm converter as well) since its
substantially simpler to think about.

As part of this change, we modify the compiler cache to acknowledge that
modules inside might not be loadable and that as a result, it might think
that a symbol is available when it is not. In this pathway, the compiler
cache tracks all modules that define a symbol, as well as the first module
that it loads that defines a symbol, and if its tries to load all possible
modules that provide a given symbol and fails, then returns without loading
the symbol. NativeCompiler doesn't return a positive indication that it
has a symbol until it has actually loaded the symbol.

Eventually we may not need this because we'll be loading modules partially.

Author: braxtonmckee

typed_python/compiler/native_compiler/binary_shared_object.py

@@ -40,6 +40,8 @@ def __init__(self, binaryForm, functionTypes, globalVariableDefinitions):
         """
         Args:
             binaryForm - a bytes object containing the actual compiled code for the module
+            functionTypes - a map from linkerName to native_ast.Type.Function containing the native
+                signatures of the functions
             globalVariableDefinitions - a map from name to GlobalVariableDefinition
         """
         self.binaryForm = binaryForm

typed_python/compiler/native_compiler/compiler_cache.py

@@ -19,7 +19,7 @@
 from typed_python.compiler.native_compiler.binary_shared_object import BinarySharedObject
 
 from typed_python.SerializationContext import SerializationContext
-from typed_python import Dict, ListOf
+from typed_python import Dict, ListOf, Set
 
 
 def ensureDirExists(cacheDir):
@@ -53,32 +53,65 @@ def __init__(self, cacheDir):
         ensureDirExists(cacheDir)
 
         self.loadedModules = Dict(str, LoadedModule)()
-        self.nameToModuleHash = Dict(str, str)()
 
+        # for each symbol, the first module we loaded that has that symbol
+        self.symbolToLoadedModuleHash = Dict(str, str)()
+
+        # for each module that we loaded or might load, the contents
+        self.symbolToModuleHashes = Dict(str, Set(str))()
+        self.moduleHashToSymbols = Dict(str, Set(str))()
+
+        # modules we might be able to load
         self.modulesMarkedValid = set()
+
+        # modules we definitely can't load
         self.modulesMarkedInvalid = set()
 
         for moduleHash in os.listdir(self.cacheDir):
             if len(moduleHash) == 40:
                 self.loadNameManifestFromStoredModuleByHash(moduleHash)
 
-    def hasSymbol(self, linkName):
-        return linkName in self.nameToModuleHash
+    def hasSymbol(self, symbol):
+        """Do we have this symbol defined somewhere?
 
-    def markModuleHashInvalid(self, hashstr):
-        with open(os.path.join(self.cacheDir, hashstr, "marked_invalid"), "w"):
-            pass
+        Note that this can change: if we attempt to laod a symbol and fail,
+        then it may no longer be defined anywhere. To really know if you have
+        a symbol, you have to load it.
+        """
+        return symbol in self.symbolToModuleHashes
 
-    def loadForSymbol(self, linkName):
-        moduleHash = self.nameToModuleHash[linkName]
+    def markModuleHashInvalid(self, moduleHash):
+        """Mark this module unloadable on disk and remove its symbols."""
+        with open(os.path.join(self.cacheDir, moduleHash, "marked_invalid"), "w"):
+            pass
 
-        nameToTypedCallTarget = {}
-        nameToNativeFunctionType = {}
+        # remove any symbols that we can't see anymore
+        for symbol in self.moduleHashToSymbols.pop(moduleHash, Set(str)()):
+            hashes = self.symbolToModuleHashes[symbol]
+            hashes.discard(moduleHash)
+            if not hashes:
+                del self.symbolToModuleHashes[symbol]
 
-        if not self.loadModuleByHash(moduleHash, nameToTypedCallTarget, nameToNativeFunctionType):
+    def loadForSymbol(self, symbol):
+        # check if this symbol is already loaded
+        if symbol in self.symbolToLoadedModuleHash:
             return None
 
-        return nameToTypedCallTarget, nameToNativeFunctionType
+        while symbol in self.symbolToModuleHashes:
+            moduleHash = list(self.symbolToModuleHashes[symbol])[0]
+
+            nameToTypedCallTarget = {}
+            nameToNativeFunctionType = {}
+
+            if self.loadModuleByHash(moduleHash, nameToTypedCallTarget, nameToNativeFunctionType):
+                return nameToTypedCallTarget, nameToNativeFunctionType
+            else:
+                assert (
+                    # either we can't load this symbol at all anymore
+                    symbol not in self.symbolToModuleHashes
+                    # or confirm we can't try to load this again
+                    or moduleHash not in self.symbolToModuleHashes[symbol]
+                )
 
     def loadModuleByHash(self, moduleHash, nameToTypedCallTarget, nameToNativeFunctionType):
         """Load a module by name.
@@ -134,6 +167,10 @@ def loadModuleByHash(self, moduleHash, nameToTypedCallTarget, nameToNativeFuncti
         nameToTypedCallTarget.update(callTargets)
         nameToNativeFunctionType.update(functionNameToNativeType)
 
+        for symbol in functionNameToNativeType:
+            if symbol not in self.symbolToLoadedModuleHash:
+                self.symbolToLoadedModuleHash[symbol] = moduleHash
+
         return True
 
     def addModule(self, binarySharedObject, nameToTypedCallTarget, linkDependencies):
@@ -149,16 +186,19 @@ def addModule(self, binarySharedObject, nameToTypedCallTarget, linkDependencies)
         dependentHashes = set()
 
         for name in linkDependencies:
-            dependentHashes.add(self.nameToModuleHash[name])
+            dependentHashes.add(self.symbolToLoadedModuleHash[name])
 
         path, hashToUse = self.writeModuleToDisk(binarySharedObject, nameToTypedCallTarget, dependentHashes)
 
         self.loadedModules[hashToUse] = (
             binarySharedObject.loadFromPath(os.path.join(path, "module.so"))
         )
 
-        for n in binarySharedObject.definedSymbols:
-            self.nameToModuleHash[n] = hashToUse
+        for symbol in binarySharedObject.definedSymbols:
+            if symbol not in self.symbolToLoadedModuleHash:
+                self.symbolToLoadedModuleHash[symbol] = hashToUse
+            self.symbolToModuleHashes.setdefault(symbol).add(hashToUse)
+        self.moduleHashToSymbols[hashToUse] = Set(str)(binarySharedObject.definedSymbols)
 
     def loadNameManifestFromStoredModuleByHash(self, moduleHash):
         if moduleHash in self.modulesMarkedValid:
@@ -185,9 +225,11 @@ def loadNameManifestFromStoredModuleByHash(self, moduleHash):
                 return False
 
         with open(os.path.join(targetDir, "name_manifest.dat"), "rb") as f:
-            self.nameToModuleHash.update(
-                SerializationContext().deserialize(f.read(), Dict(str, str))
-            )
+            manifest = SerializationContext().deserialize(f.read(), Set(str))
+
+            for symbolName in manifest:
+                self.symbolToModuleHashes.setdefault(symbolName).add(moduleHash)
+            self.moduleHashToSymbols[moduleHash] = manifest
 
         self.modulesMarkedValid.add(moduleHash)
 
@@ -225,12 +267,10 @@ def writeModuleToDisk(self, binarySharedObject, nameToTypedCallTarget, submodule
 
         # write the manifest. Every TP process using the cache will have to
         # load the manifest every time, so we try to use compiled code to load it
-        manifest = Dict(str, str)()
-        for n in binarySharedObject.functionTypes:
-            manifest[n] = hashToUse
+        manifest = Set(str)(binarySharedObject.functionTypes)
 
         with open(os.path.join(tempTargetDir, "name_manifest.dat"), "wb") as f:
-            f.write(SerializationContext().serialize(manifest, Dict(str, str)))
+            f.write(SerializationContext().serialize(manifest, Set(str)))
 
         with open(os.path.join(tempTargetDir, "name_manifest.txt"), "w") as f:
             for sourceName in manifest:
@@ -262,7 +302,7 @@ def writeModuleToDisk(self, binarySharedObject, nameToTypedCallTarget, submodule
         return targetDir, hashToUse
 
     def function_pointer_by_name(self, linkName):
-        moduleHash = self.nameToModuleHash.get(linkName)
+        moduleHash = self.symbolToLoadedModuleHash.get(linkName)
         if moduleHash is None:
             raise Exception("Can't find a module for " + linkName)
 

typed_python/compiler/native_compiler/compiler_cache_test.py

@@ -359,7 +359,7 @@ def test_compiler_cache_handles_changed_types():
 
         # if we try to use 'f', it should work even though we no longer have
         # a defniition for 'g2'
-        assert evaluateExprInFreshProcess(VERSION2, 'x.f(1)', compilerCacheDir) == 1
+        assert evaluateExprInFreshProcess(VERSION2, 'x.f(1)', compilerCacheDir, printComments=True) == 1
         assert len(os.listdir(compilerCacheDir)) == 2
 
         badCt = 0

typed_python/compiler/native_compiler/native_compiler.py

@@ -12,6 +12,8 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+from typed_python import OneOf
+
 import llvmlite.binding as llvm
 import typed_python.compiler.native_compiler.native_ast as native_ast
 import typed_python.compiler.native_compiler.native_ast_to_llvm as native_ast_to_llvm
@@ -20,36 +22,86 @@
 from typed_python.compiler.native_compiler.loaded_module import LoadedModule
 from typed_python.compiler.native_compiler.native_function_pointer import NativeFunctionPointer
 from typed_python.compiler.native_compiler.binary_shared_object import BinarySharedObject
+from typed_python.compiler.native_compiler.typed_call_target import TypedCallTarget
 
 
 class NativeCompiler:
     """"Engine for compiling bundles of native_ast.Function objects into NativeFunctionPointers.
 
     This class is responsible for
+
         * telling clients what named functions have been defined and what their types are.
         * compiling functions into a runnable form using llvm
         * performing any runtime-based performance optimizations
         * maintaining the compiler cache
 
-    Note that this class is NOT threadsafe and clients are expected to serialize their
-    access through Runtime.
+    Note that this class is NOT threadsafe
     """
     def __init__(self, inlineThreshold):
         self.compilerCache = None
+        self.hasEverHadFunctionsAdded = False
+
         self.engine, self.module_pass_manager = create_execution_engine(inlineThreshold)
         self.converter = native_ast_to_llvm.Converter()
-        self.functions_by_name = {}
         self.inlineThreshold = inlineThreshold
         self.verbose = False
         self.optimize = True
 
+        # map from linkName: str -> NativeFunctionPointer
+        # this is only populated if we don't have a compiler cache
+        self.linkNameToFunctionPtr = {}
+
+        # map from linkName: str -> TypedCallTarget.
+        # this contains every typed call target that we have loaded
+        # but not the ones that are available in the compiler cache but that
+        # have not yet been loaded
+        self._allTypedCallTargets = {}
+        self._allFunctionsDefined = set()
+
+    def isFunctionDefined(self, linkName: str) -> bool:
+        """Is this function known to the compiler?"""
+        if self.compilerCache is None:
+            return linkName in self.linkNameToFunctionPtr
+        else:
+            if linkName in self._allFunctionsDefined:
+                return True
+
+            if not self.compilerCache.hasSymbol(linkName):
+                return False
+
+            # the compiler cache has the symbol but we can't currently be sure
+            # that we can load it - so load it. If it fails, it won't tell us
+            # that it can load that symbol a second time.
+            self._loadFromCache(linkName)
+
+            return linkName in self._allFunctionsDefined
+
+    def typedCallTargetFor(self, linkName: str) -> OneOf(None, TypedCallTarget):
+        """If this function is known, its TypedCallTarget, or None.
+
+        If this function was added without a TypedCallTarget (say, because its
+        a destructor or some other untyped function) then this will be None.
+        """
+        if self.compilerCache is None:
+            # if we have no compiler cache, then _allTypedCallTargets will
+            # contain everything we've ever seen
+            return self._allTypedCallTargets.get(linkName)
+        else:
+            if linkName in self._allTypedCallTargets:
+                return self._allTypedCallTargets[linkName]
+
+            if linkName not in self._allFunctionsDefined:
+                if self.compilerCache.hasSymbol(linkName):
+                    self._loadFromCache(linkName)
+
+            return self._allTypedCallTargets.get(linkName)
+
     def initializeCompilerCache(self, compilerCacheDir):
         """Indicate that we should use a compiler cache from disk at 'compilerCacheDir'."""
-        self.compilerCache = CompilerCache(compilerCacheDir)
+        if self.hasEverHadFunctionsAdded:
+            raise Exception("Can't set the compiler cache if we've added functions.")
 
-    def markExternal(self, functionNameToType):
-        """Provide type signatures for a set of external functions."""
-        self.converter.markExternal(functionNameToType)
+        self.compilerCache = CompilerCache(compilerCacheDir)
 
     def mark_converter_verbose(self):
         self.converter.verbose = True
@@ -69,51 +121,67 @@ def addFunctions(
 
         Once a function has been added, we can request a NativeFunctionPointer for it.
         """
+        self.hasEverHadFunctionsAdded = True
+
         if self.compilerCache is None:
+            self._allTypedCallTargets.update(typedCallTargets)
+            self._allFunctionsDefined.update(functionDefinitions)
+
             loadedModule = self._buildModule(functionDefinitions)
             loadedModule.linkGlobalVariables()
         else:
             binary = self._buildSharedObject(functionDefinitions)
-
             self.compilerCache.addModule(
                 binary,
                 typedCallTargets,
                 externallyUsed
             )
 
-    def functionPointerByName(self, linkerName) -> NativeFunctionPointer:
+    def functionPointerByName(self, linkName) -> NativeFunctionPointer:
         """Find a NativeFunctionPointer for a given link-time name.
 
         Args:
-            linkerName (str) - the name of the compiled symbol we want.
+            linkName (str) - the name of the compiled symbol we want.
 
         Returns:
             a NativeFunctionPointer or None if the function has never been defined.
         """
         if self.compilerCache is not None:
             # the compiler cache has every shared object and can load them
-            return self.compilerCache.function_pointer_by_name(linkerName)
+            if linkName in self.linkNameToFunctionPtr:
+                return self.linkNameToFunctionPtr[linkName]
+
+            if not self.compilerCache.hasSymbol(linkName):
+                return None
+
+            self.compilerCache.loadForSymbol(linkName)
+
+            funcPtr = self.compilerCache.function_pointer_by_name(linkName)
+
+            assert funcPtr is not None
+
+            self.linkNameToFunctionPtr[linkName] = funcPtr
+
+            return funcPtr
 
         # the llvm compiler is just building shared objects, but the
         # compiler cache has all the pointers.
-        return self.functions_by_name.get(linkerName)
+        return self.linkNameToFunctionPtr.get(linkName)
 
-    def loadFromCache(self, linkName):
+    def _loadFromCache(self, linkName):
         """Attempt to load a cached copy of 'linkName' and all reachable code.
 
-        If it isn't defined, or has already been defined, return None. If we're loading it
-        for the first time, return a pair
-
-            (typedCallTargets, nativeTypes)
+        The compilerCache must exist and agree that this function exists.
+        """
+        assert self.compilerCache and self.compilerCache.hasSymbol(linkName)
 
-        where typedCallTargets is a map from linkName to TypedCallTarget, and nativeTypes is
-        a map from linkName to native_ast.Type.Function giving the native implementation type.
+        callTargetsAndTypes = self.compilerCache.loadForSymbol(linkName)
 
-        WARNING: this will return None if you already called 'functionPointerByName' on it
-        """
-        if self.compilerCache:
-            if self.compilerCache.hasSymbol(linkName):
-                return self.compilerCache.loadForSymbol(linkName)
+        if callTargetsAndTypes is not None:
+            newTypedCallTargets, newNativeFunctionTypes = callTargetsAndTypes
+            self.converter.markExternal(newNativeFunctionTypes)
+            self._allTypedCallTargets.update(newTypedCallTargets)
+            self._allFunctionsDefined.update(newNativeFunctionTypes)
 
     def _buildSharedObject(self, functions):
         """Add native definitions and return a BinarySharedObject representing the compiled code."""
@@ -184,7 +252,7 @@ def _buildModule(self, functions):
             native_function_pointers[fname] = NativeFunctionPointer(
                 fname, func_ptr, input_types, output_type
             )
-            self.functions_by_name[fname] = native_function_pointers[fname]
+            self.linkNameToFunctionPtr[fname] = native_function_pointers[fname]
 
         native_function_pointers[module.GET_GLOBAL_VARIABLES_NAME] = (
             NativeFunctionPointer(