Allow reopened proxies to fill caches after read-only open

Author: FrancescAlted

src/blosc2/proxy.py

@@ -192,6 +192,11 @@ class Proxy(blosc2.Operand):
 
     This can be used to cache chunks of a regular data container which follows the
     :ref:`ProxySource` or :ref:`ProxyNDSource` interfaces.
+
+    If a persisted proxy is reopened in read mode, lazy cache fills still need to
+    update the local cache.  In that case the cache file is reopened internally
+    in append mode on demand when :meth:`fetch`, :meth:`afetch`, or
+    :meth:`__getitem__` needs to populate missing chunks.
     """
 
     def __init__(
@@ -209,6 +214,9 @@ def __init__(
         mode: str, optional
             "a" means read/write (create if it doesn't exist); "w" means create
             (overwrite if it exists). Default is "a".
+            When a persisted proxy is reopened through :func:`blosc2.open` with
+            ``mode="r"``, later lazy cache fills may reopen the cache file
+            internally in append mode so missing chunks can be written locally.
         kwargs: dict, optional
             Keyword arguments supported:
 
@@ -259,11 +267,21 @@ def __init__(
                 )
                 self._cache.fill_special(self.src.nbytes // self.src.typesize, blosc2.SpecialValue.UNINIT)
         self._schunk_cache = getattr(self._cache, "schunk", self._cache)
+        if self.urlpath is None:
+            self.urlpath = getattr(self._schunk_cache, "urlpath", None)
         vlmeta = kwargs.get("vlmeta")
         if vlmeta:
             for key in vlmeta:
                 self._schunk_cache.vlmeta[key] = vlmeta[key]
 
+    def _ensure_writable_cache(self) -> None:
+        """Reopen a persisted cache writable when lazy fetch needs to fill chunks."""
+        cache_urlpath = getattr(self._schunk_cache, "urlpath", None)
+        if cache_urlpath is None or getattr(self._schunk_cache, "mode", None) != "r":
+            return
+        self._cache = blosc2.blosc2_ext.open(cache_urlpath, "a", 0)
+        self._schunk_cache = getattr(self._cache, "schunk", self._cache)
+
     def fetch(self, item: slice | list[slice] | None = ()) -> blosc2.NDArray | blosc2.schunk.SChunk:
         """
         Get the container used as cache with the requested data updated.
@@ -279,6 +297,13 @@ def fetch(self, item: slice | list[slice] | None = ()) -> blosc2.NDArray | blosc
         out: :ref:`NDArray` or :ref:`SChunk`
             The local container used to cache the already requested data.
 
+        Notes
+        -----
+        If the proxy cache was reopened read-only from disk, this method may
+        reopen that cache internally in append mode before filling missing
+        chunks.  This preserves the logical read-only open of the proxy source
+        while still allowing the local cache to be populated lazily.
+
         Examples
         --------
         >>> import numpy as np
@@ -292,6 +317,7 @@ def fetch(self, item: slice | list[slice] | None = ()) -> blosc2.NDArray | blosc
         [2 3]
         [4 5]]
         """
+        self._ensure_writable_cache()
         if item == ():
             # Full realization
             for info in self._schunk_cache.iterchunks_info():
@@ -327,6 +353,9 @@ async def afetch(self, item: slice | list[slice] | None = ()) -> blosc2.NDArray
         -----
         This method is only available if the :ref:`ProxySource` or :ref:`ProxyNDSource`
         have an async `aget_chunk` method.
+        If the proxy cache was reopened read-only from disk, this method may
+        reopen that cache internally in append mode before filling missing
+        chunks.
 
         Examples
         --------
@@ -384,6 +413,7 @@ async def afetch(self, item: slice | list[slice] | None = ()) -> blosc2.NDArray
         """
         if not callable(getattr(self.src, "aget_chunk", None)):
             raise NotImplementedError("afetch is only available if the source has an aget_chunk method")
+        self._ensure_writable_cache()
         if item == ():
             # Full realization
             for info in self._schunk_cache.iterchunks_info():

src/blosc2/schunk.py

@@ -1721,6 +1721,9 @@ def open(
       it will return the Python-Blosc2 container used to cache the data which
       can be a :ref:`SChunk` or a :ref:`NDArray` and may not have all the data
       initialized (e.g. if the user has not accessed to it yet).
+      When such a persisted proxy is opened with ``mode="r"``, later lazy cache
+      fills may reopen the local cache internally in append mode so missing
+      chunks can still be written.
 
     * When opening a :ref:`LazyExpr` keep in mind the note above regarding operands.
 

tests/ndarray/test_proxy.py

@@ -105,6 +105,17 @@ def test_open(urlpath, shape, chunks, blocks, slices, dtype):
     blosc2.remove_urlpath(proxy_urlpath)
 
 
+def test_open_read_mode_allows_proxy_cache_fill(tmp_path):
+    src_urlpath = str(tmp_path / "src.b2nd")
+    proxy_urlpath = str(tmp_path / "proxy.b2nd")
+
+    a = blosc2.asarray(np.arange(25, dtype=np.int64).reshape(5, 5), urlpath=src_urlpath, mode="w")
+    _ = blosc2.Proxy(a, urlpath=proxy_urlpath, mode="w")
+
+    proxy = blosc2.open(proxy_urlpath, mode="r")
+    np.testing.assert_array_equal(proxy[:], a[:])
+
+
 # Test the ProxyNDSources interface
 @pytest.mark.parametrize(
     ("shape", "chunks", "blocks"),