Merge branch 'main' into Py314u

jan-janssen · web-flow · commit df6a761a3fee · 2026-01-04T16:31:27.000+01:00
diff --git a/.ci_support/environment-docs.yml b/.ci_support/environment-docs.yml
@@ -10,8 +10,8 @@ dependencies:
 - openmpi
 - sphinx
 - sphinx_rtd_theme
-- cloudpickle =3.1.1
-- h5py =3.15.0
+- cloudpickle =3.1.2
+- h5py =3.15.1
 - jupyter-book =1.0.0
 - mpi4py =4.1.1
 - python =3.12
diff --git a/.ci_support/environment-integration.yml b/.ci_support/environment-integration.yml
@@ -1,22 +1,23 @@
 channels:
 - conda-forge
 dependencies:
-- h5py =3.15.0
+- h5py =3.15.1
 - jupyter
 - mpi4py =4.1.1
 - numpy =2.3.3
 - openmpi
 - papermill
-- atomistics =0.3.0
-- cloudpickle =3.1.1
-- flux-core =0.64.0
+- atomistics =0.3.4
+- cloudpickle =3.1.2
+- flux-core =0.81.0
+- flux-pmix =0.6.0
 - gpaw =25.7.0
-- hatchling =1.27.0
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
 - ipython =9.0.2
 - matplotlib =3.10.6
 - networkx =3.4.2
 - pygraphviz =1.14
-- pysqa =0.3.2
+- pysqa =0.3.3
 - pyzmq =27.1.0
 - qe =7.5
diff --git a/.ci_support/environment-mini.yml b/.ci_support/environment-mini.yml
@@ -4,6 +4,6 @@ dependencies:
 - python
 - numpy =2.3.3
 - cloudpickle =3.1.0
-- hatchling =1.27.0
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
 - pyzmq =27.1.0
diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml
@@ -4,13 +4,13 @@ dependencies:
 - python
 - numpy =2.3.3
 - mpich
-- cloudpickle =3.1.1
-- h5py =3.15.0
-- hatchling =1.27.0
+- cloudpickle =3.1.2
+- h5py =3.15.1
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
 - ipython =9.0.2
 - mpi4py =4.1.1
 - networkx =3.4.2
 - pygraphviz =1.14
-- pysqa =0.3.2
+- pysqa =0.3.3
 - pyzmq =27.1.0
diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml
@@ -4,13 +4,13 @@ dependencies:
 - python
 - numpy =2.3.3
 - openmpi
-- cloudpickle =3.1.1
-- h5py =3.15.0
-- hatchling =1.27.0
+- cloudpickle =3.1.2
+- h5py =3.15.1
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
 - ipython =9.0.2
 - mpi4py =4.1.1
 - networkx =3.4.2
 - pygraphviz =1.14
-- pysqa =0.3.2
+- pysqa =0.3.3
 - pyzmq =27.1.0
diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml
@@ -4,9 +4,9 @@ dependencies:
 - python
 - numpy =2.3.3
 - msmpi
-- cloudpickle =3.1.1
-- h5py =3.15.0
-- hatchling =1.27.0
+- cloudpickle =3.1.2
+- h5py =3.15.1
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
 - ipython =9.0.2
 - mpi4py =4.1.1
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -198,11 +198,11 @@ jobs:
       shell: bash -l {0}
       timeout-minutes: 5
       run: |
-        echo -e '- coverage\n- flux-core =0.79.0'>> .ci_support/environment-mpich.yml
+        echo -e '- coverage\n- flux-core =0.81.0'>> .ci_support/environment-mpich.yml
         cat .ci_support/environment-mpich.yml
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: '3.12'
+        python-version: '3.13'
         miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: "true"
@@ -229,11 +229,11 @@ jobs:
       shell: bash -l {0}
       timeout-minutes: 5
       run: |
-        echo -e '- coverage\n- flux-core =0.79.0\n- flux-pmix=0.6.0' >> .ci_support/environment-openmpi.yml
+        echo -e '- coverage\n- flux-core =0.81.0\n- flux-pmix=0.6.0' >> .ci_support/environment-openmpi.yml
         cat .ci_support/environment-openmpi.yml
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: '3.12'
+        python-version: '3.13'
         miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: "true"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,10 +1,10 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.0
+    rev: v0.14.10
     hooks:
       - id: ruff
         name: ruff lint
         args: ["--fix"]
-        files: ^executorlib/
+        files: ^src/executorlib/
       - id: ruff-format
         name: ruff format
diff --git a/README.md b/README.md
@@ -134,6 +134,7 @@ as hierarchical job scheduler within the allocations.
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
   * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
+  * [Advanced Scheduling](https://executorlib.readthedocs.io/en/latest/1-single-node.html#advanced-scheduling)
   * [Testing and Debugging](https://executorlib.readthedocs.io/en/latest/1-single-node.html#testing-and-debugging)
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
diff --git a/binder/environment.yml b/binder/environment.yml
@@ -4,16 +4,16 @@ dependencies:
 - python
 - numpy =2.3.3
 - openmpi
-- cloudpickle =3.1.1
-- flux-core =0.79.0
+- cloudpickle =3.1.2
+- flux-core =0.81.0
 - flux-pmix =0.6.0
-- hatchling =1.27.0
+- hatchling =1.28.0
 - hatch-vcs =0.5.0
-- h5py =3.15.0
+- h5py =3.15.1
 - ipython =9.0.2
 - matplotlib =3.10.0
 - mpi4py =4.1.1
 - networkx =3.4.2
 - pygraphviz =1.14
-- pysqa =0.3.2
+- pysqa =0.3.3
 - pyzmq =27.1.0
diff --git a/notebooks/1-single-node.ipynb b/notebooks/1-single-node.ipynb
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,8 +1,8 @@
 [build-system]
 requires = [
-    "hatchling==1.27.0",
+    "hatchling==1.28.0",
     "hatch-vcs==0.5.0",
-    "cloudpickle==3.1.1",
+    "cloudpickle==3.1.2",
     "pyzmq==27.1.0",
 ]
 build-backend = "hatchling.build"
@@ -30,7 +30,7 @@ classifiers = [
     "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
-    "cloudpickle==3.1.1",
+    "cloudpickle==3.1.2",
     "pyzmq==27.1.0",
 ]
 dynamic = ["version"]
@@ -41,7 +41,7 @@ Documentation = "https://executorlib.readthedocs.io"
 Repository = "https://github.com/pyiron/executorlib"
 
 [project.optional-dependencies]
-cache = ["h5py==3.15.0"]
+cache = ["h5py==3.15.1"]
 graph = [
     "pygraphviz==1.14",
     "networkx==3.4.2",
@@ -53,13 +53,13 @@ graphnotebook = [
 ]
 mpi = ["mpi4py==4.1.1"]
 cluster = [
-    "pysqa==0.3.2",
-    "h5py==3.15.0",
+    "pysqa==0.3.3",
+    "h5py==3.15.1",
 ]
 all = [
     "mpi4py==4.1.1",
-    "pysqa==0.3.2",
-    "h5py==3.15.0",
+    "pysqa==0.3.3",
+    "h5py==3.15.1",
     "pygraphviz==1.14",
     "networkx==3.4.2",
     "ipython==9.0.2",
diff --git a/src/executorlib/executor/base.py b/src/executorlib/executor/base.py
@@ -21,6 +21,7 @@ class BaseExecutor(FutureExecutor, ABC):
 
     def __init__(self, executor: TaskSchedulerBase):
         self._task_scheduler = executor
+        self._is_active = True
 
     @property
     def max_workers(self) -> Optional[int]:
@@ -99,9 +100,49 @@ def submit(  # type: ignore
         Returns:
             Future: A Future representing the given call.
         """
-        return self._task_scheduler.submit(
-            *([fn] + list(args)), resource_dict=resource_dict, **kwargs
-        )
+        if self._is_active:
+            return self._task_scheduler.submit(
+                *([fn] + list(args)), resource_dict=resource_dict, **kwargs
+            )
+        else:
+            raise RuntimeError("cannot schedule new futures after shutdown")
+
+    def map(
+        self,
+        fn: Callable,
+        *iterables,
+        timeout: Optional[float] = None,
+        chunksize: int = 1,
+    ):
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: The size of the chunks the iterable will be broken into
+                before being passed to a child process. This argument is only
+                used by ProcessPoolExecutor; it is ignored by
+                ThreadPoolExecutor.
+
+        Returns:
+            An iterator equivalent to: map(func, *iterables) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If fn(*args) raises for any values.
+        """
+        if self._is_active:
+            return self._task_scheduler.map(
+                *([fn] + list(iterables)),
+                timeout=timeout,
+                chunksize=chunksize,
+            )
+        else:
+            raise RuntimeError("cannot schedule new futures after shutdown")
 
     def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
         """
@@ -119,6 +160,7 @@ def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
                 cancelled.
         """
         self._task_scheduler.shutdown(wait=wait, cancel_futures=cancel_futures)
+        self._is_active = False
 
     def __len__(self) -> int:
         """
@@ -143,3 +185,4 @@ def __exit__(self, *args, **kwargs) -> None:
         Exit method called when exiting the context manager.
         """
         self._task_scheduler.__exit__(*args, **kwargs)
+        self._is_active = False
diff --git a/src/executorlib/standalone/serialize.py b/src/executorlib/standalone/serialize.py
@@ -75,7 +75,7 @@ def serialize_funct(
                 "kwargs": fn_kwargs,
             }
         )
-        task_key = fn.__name__ + _get_hash(binary=binary_all)
+        task_key = _get_function_name(fn=fn) + _get_hash(binary=binary_all)
     data = {
         "fn": fn,
         "args": fn_args,
@@ -99,3 +99,10 @@ def _get_hash(binary: bytes) -> str:
     # Remove specification of jupyter kernel from hash to be deterministic
     binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)
     return str(hashlib.md5(binary_no_ipykernel).hexdigest())
+
+
+def _get_function_name(fn: Callable) -> str:
+    if hasattr(fn, "__name__"):
+        return fn.__name__
+    else:
+        return str(fn.__class__).split("'")[-2].split(".")[-1]
diff --git a/src/executorlib/task_scheduler/base.py b/src/executorlib/task_scheduler/base.py
@@ -143,6 +143,43 @@ def submit(  # type: ignore
             )
         return f
 
+    def map(
+        self,
+        fn: Callable,
+        *iterables,
+        timeout: Optional[float] = None,
+        chunksize: int = 1,
+    ):
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: The size of the chunks the iterable will be broken into
+                before being passed to a child process. This argument is only
+                used by ProcessPoolExecutor; it is ignored by
+                ThreadPoolExecutor.
+
+        Returns:
+            An iterator equivalent to: map(func, *iterables) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If fn(*args) raises for any values.
+        """
+        if isinstance(iterables, (list, tuple)) and any(
+            isinstance(i, Future) for i in iterables
+        ):
+            iterables = tuple(
+                i.result() if isinstance(i, Future) else i for i in iterables
+            )
+
+        return super().map(fn, *iterables, timeout=timeout, chunksize=chunksize)
+
     def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
         """
         Clean-up the resources associated with the Executor.
diff --git a/tests/test_singlenodeexecutor_cache.py b/tests/test_singlenodeexecutor_cache.py
@@ -17,6 +17,11 @@ def get_error(a):
     raise ValueError(a)
 
 
+class AddClass:
+    def __call__(self, a, b):
+        return a+b
+
+
 @unittest.skipIf(
     skip_h5py_test, "h5py is not installed, so the h5io tests are skipped."
 )
@@ -34,6 +39,21 @@ def test_cache_data(self):
             sum([sum(c["input_args"][0]) for c in cache_lst]), sum(result_lst)
         )
 
+    def test_cache_data_class(self):
+        cache_directory = os.path.abspath("executorlib_cache")
+        with SingleNodeExecutor(cache_directory=cache_directory) as exe:
+            self.assertTrue(exe)
+            cloudpickle_register(ind=1)
+            add_instance = AddClass()
+            future_lst = [exe.submit(add_instance, a=i, b=i) for i in range(1, 4)]
+            result_lst = [f.result() for f in future_lst]
+
+        cache_lst = get_cache_data(cache_directory=cache_directory)
+        self.assertEqual(sum([c["output"] for c in cache_lst]), sum(result_lst))
+        self.assertEqual(
+            sum([sum([c["input_kwargs"]["a"], c["input_kwargs"]["b"]]) for c in cache_lst]), sum(result_lst)
+        )
+
     def test_cache_key(self):
         cache_directory = os.path.abspath("executorlib_cache")
         with SingleNodeExecutor(cache_directory=cache_directory) as exe:
diff --git a/tests/test_singlenodeexecutor_mpi.py b/tests/test_singlenodeexecutor_mpi.py
diff --git a/tests/test_singlenodeexecutor_noblock.py b/tests/test_singlenodeexecutor_noblock.py
diff --git a/tests/test_standalone_serialize.py b/tests/test_standalone_serialize.py
