Replaced max_blocksize -> items_per_block as it is more meaningful

Author: FrancescAlted

bench/batch_store.py

@@ -19,7 +19,7 @@
 NBATCHES = 10_000
 OBJECTS_PER_BATCH = 100
 TOTAL_OBJECTS = NBATCHES * OBJECTS_PER_BATCH
-BLOCKSIZE_MAX = 32
+ITEMS_PER_BLOCK = 32
 N_RANDOM_READS = 1_000
 
 
@@ -64,7 +64,7 @@ def build_store(
         storage = blosc2.Storage(mode="w")
         store = blosc2.BatchStore(
             storage=storage,
-            max_blocksize=BLOCKSIZE_MAX,
+            items_per_block=ITEMS_PER_BLOCK,
             serializer=serializer,
             cparams={
                 "codec": codec,
@@ -84,7 +84,7 @@ def build_store(
         "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC),
     }
     with blosc2.BatchStore(
-        storage=storage, max_blocksize=BLOCKSIZE_MAX, serializer=serializer, cparams=cparams
+        storage=storage, items_per_block=ITEMS_PER_BLOCK, serializer=serializer, cparams=cparams
     ) as store:
         for batch_index in range(NBATCHES):
             store.append(make_batch(batch_index))
@@ -132,7 +132,7 @@ def main() -> None:
         assert store is not None
         read_store = store
     else:
-        read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, max_blocksize=BLOCKSIZE_MAX)
+        read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, items_per_block=ITEMS_PER_BLOCK)
     samples, timings_ns = measure_random_reads(read_store)
     t0 = time.perf_counter()
     checksum = 0
@@ -147,7 +147,7 @@ def main() -> None:
     print(f"  build time: {build_time_s:.3f} s")
     print(f"  batches: {len(read_store)}")
     print(f"  items: {TOTAL_OBJECTS}")
-    print(f"  max_blocksize: {read_store.max_blocksize}")
+    print(f"  items_per_block: {read_store.items_per_block}")
     print()
     print(read_store.info)
     print(f"Random scalar reads: {N_RANDOM_READS}")

src/blosc2/batch_store.py

@@ -21,7 +21,7 @@
 from blosc2._msgpack_utils import msgpack_packb, msgpack_unpackb
 from blosc2.info import InfoReporter, format_nbytes_info
 
-_BATCHSTORE_META = {"version": 1, "serializer": "msgpack", "max_blocksize": None, "arrow_schema": None}
+_BATCHSTORE_META = {"version": 1, "serializer": "msgpack", "items_per_block": None, "arrow_schema": None}
 _SUPPORTED_SERIALIZERS = {"msgpack", "arrow"}
 _BATCHSTORE_VLMETA_KEY = "_batch_store_metadata"
 
@@ -82,9 +82,9 @@ def __getitem__(self, index: int | slice) -> Any | list[Any]:
             items = self._decode_items()
             index = self._normalize_index(index)
             return items[index]
-        max_blocksize = self._parent.max_blocksize
-        if max_blocksize is not None:
-            block_index, item_index = divmod(index, max_blocksize)
+        items_per_block = self._parent.items_per_block
+        if items_per_block is not None:
+            block_index, item_index = divmod(index, items_per_block)
             if block_index >= self._nblocks:
                 raise IndexError("Batch index out of range")
             block = self._get_block(block_index)
@@ -161,9 +161,10 @@ class BatchStore:
 
     Parameters
     ----------
-    max_blocksize : int, optional
+    items_per_block : int, optional
         Maximum number of items stored in each internal variable-length block.
-        If not provided, a value is inferred from the first batch.
+        The last block in a batch may contain fewer items than this cap. If not
+        provided, a value is inferred from the first batch.
     serializer : {"msgpack", "arrow"}, optional
         Serializer used for batch payloads. ``"msgpack"`` is the default and is
         the general-purpose choice for Python items, including nested Blosc2
@@ -229,7 +230,7 @@ def _attach_schunk(self, schunk: blosc2.SChunk) -> None:
         except KeyError:
             batchstore_meta = {}
         self._serializer = batchstore_meta.get("serializer", self._serializer)
-        self._max_blocksize = batchstore_meta.get("max_blocksize", self._max_blocksize)
+        self._items_per_block = batchstore_meta.get("items_per_block", self._items_per_block)
         self._arrow_schema = batchstore_meta.get("arrow_schema", self._arrow_schema)
         self._arrow_schema_obj = None
         self._batch_lengths = self._load_batch_lengths()
@@ -258,7 +259,7 @@ def _make_storage(self) -> blosc2.Storage:
 
     def __init__(
         self,
-        max_blocksize: int | None = None,
+        items_per_block: int | None = None,
         serializer: str = "msgpack",
         _from_schunk: blosc2.SChunk | None = None,
         **kwargs: Any,
@@ -269,11 +270,11 @@ def __init__(
         mode is ``"r"`` or ``"a"``, the container is reopened automatically.
         Otherwise a new empty store is created.
         """
-        if max_blocksize is not None and max_blocksize <= 0:
-            raise ValueError("max_blocksize must be a positive integer")
+        if items_per_block is not None and items_per_block <= 0:
+            raise ValueError("items_per_block must be a positive integer")
         if serializer not in _SUPPORTED_SERIALIZERS:
             raise ValueError(f"Unsupported BatchStore serializer: {serializer!r}")
-        self._max_blocksize: int | None = max_blocksize
+        self._items_per_block: int | None = items_per_block
         self._serializer = serializer
         self._arrow_schema: bytes | None = None
         self._arrow_schema_obj = None
@@ -306,7 +307,7 @@ def __init__(
         fixed_meta["batchstore"] = {
             **_BATCHSTORE_META,
             "serializer": self._serializer,
-            "max_blocksize": self._max_blocksize,
+            "items_per_block": self._items_per_block,
             "arrow_schema": self._arrow_schema,
         }
         storage.meta = fixed_meta
@@ -432,10 +433,10 @@ def _get_flat_item(self, index: int | slice) -> Any | list[Any]:
         return self[batch_index][item_index]
 
     def _block_sizes_from_batch_length(self, batch_length: int, nblocks: int) -> list[int]:
-        if self._max_blocksize is None or nblocks <= 0:
+        if self._items_per_block is None or nblocks <= 0:
             return []
-        full_blocks, remainder = divmod(batch_length, self._max_blocksize)
-        block_sizes = [self._max_blocksize] * full_blocks
+        full_blocks, remainder = divmod(batch_length, self._items_per_block)
+        block_sizes = [self._items_per_block] * full_blocks
         if remainder:
             block_sizes.append(remainder)
         if not block_sizes and batch_length > 0:
@@ -445,7 +446,7 @@ def _block_sizes_from_batch_length(self, batch_length: int, nblocks: int) -> lis
         return block_sizes
 
     def _get_block_sizes(self, batch_sizes: list[int]) -> list[int] | None:
-        if self._max_blocksize is None:
+        if self._items_per_block is None:
             return None
         block_sizes: list[int] = []
         for index, batch_length in enumerate(batch_sizes):
@@ -537,9 +538,9 @@ def _payload_sizes_for_batch(self, batch: Any) -> list[int]:
 
     def _ensure_layout_for_batch(self, batch: Any) -> None:
         layout_changed = False
-        if self._max_blocksize is None:
+        if self._items_per_block is None:
             payload_sizes = self._payload_sizes_for_batch(batch)
-            self._max_blocksize = self._guess_blocksize(payload_sizes)
+            self._items_per_block = self._guess_blocksize(payload_sizes)
             layout_changed = True
         if self._serializer == "arrow" and self._arrow_schema is not None:
             layout_changed = layout_changed or len(self) == 0
@@ -555,7 +556,7 @@ def _persist_layout_metadata(self) -> None:
         fixed_meta = dict(storage.meta or {})
         fixed_meta["batchstore"] = {
             **dict(fixed_meta.get("batchstore", {})),
-            "max_blocksize": self._max_blocksize,
+            "items_per_block": self._items_per_block,
             "serializer": self._serializer,
             "arrow_schema": self._arrow_schema,
         }
@@ -640,11 +641,11 @@ def _vl_dparams_kwargs(self) -> dict[str, Any]:
         return asdict(self.schunk.dparams)
 
     def _compress_batch(self, batch: Any) -> bytes:
-        if self._max_blocksize is None:
-            raise RuntimeError("BatchStore max_blocksize is not initialized")
+        if self._items_per_block is None:
+            raise RuntimeError("BatchStore items_per_block is not initialized")
         blocks = [
-            self._serialize_block(batch[i : i + self._max_blocksize])
-            for i in range(0, self._batch_len(batch), self._max_blocksize)
+            self._serialize_block(batch[i : i + self._items_per_block])
+            for i in range(0, self._batch_len(batch), self._items_per_block)
         ]
         return blosc2.blosc2_ext.vlcompress(blocks, **self._vl_cparams_kwargs())
 
@@ -823,8 +824,12 @@ def dparams(self):
         return self.schunk.dparams
 
     @property
-    def max_blocksize(self) -> int | None:
-        return self._max_blocksize
+    def items_per_block(self) -> int | None:
+        """Maximum number of items per internal block.
+
+        The last block in a batch may contain fewer items.
+        """
+        return self._items_per_block
 
     @property
     def items(self) -> BatchStoreItems:
@@ -903,7 +908,7 @@ def copy(self, **kwargs: Any) -> BatchStore:
             raise ValueError("meta should not be passed to copy")
         kwargs["cparams"] = kwargs.get("cparams", copy.deepcopy(self.cparams))
         kwargs["dparams"] = kwargs.get("dparams", copy.deepcopy(self.dparams))
-        kwargs["max_blocksize"] = kwargs.get("max_blocksize", self.max_blocksize)
+        kwargs["items_per_block"] = kwargs.get("items_per_block", self.items_per_block)
         kwargs["serializer"] = kwargs.get("serializer", self.serializer)
         user_vlmeta = self._user_vlmeta_items() if len(self.vlmeta) > 0 else {}
 

tests/test_batch_store.py

@@ -37,7 +37,7 @@ def _make_nested_blosc2_objects():
     nested_vlarray = blosc2.VLArray()
     nested_vlarray.extend(["alpha", {"beta": 2}])
 
-    nested_batchstore = blosc2.BatchStore(max_blocksize=2)
+    nested_batchstore = blosc2.BatchStore(items_per_block=2)
     nested_batchstore.extend([[1, 2], ["x", {"y": 3}]])
 
     estore = blosc2.EmbedStore()
@@ -65,8 +65,8 @@ def test_batchstore_roundtrip(contiguous, urlpath):
         assert barray.append(batch) == i
 
     assert len(barray) == len(BATCHES)
-    assert barray.max_blocksize is not None
-    assert 1 <= barray.max_blocksize <= len(BATCHES[0])
+    assert barray.items_per_block is not None
+    assert 1 <= barray.items_per_block <= len(BATCHES[0])
     assert [batch[:] for batch in barray] == BATCHES
     assert barray.append([1, 2]) == len(BATCHES) + 1
     assert [batch[:] for batch in barray][-1] == [1, 2]
@@ -102,7 +102,7 @@ def test_batchstore_roundtrip(contiguous, urlpath):
     if urlpath is not None:
         reopened = blosc2.open(urlpath, mode="r")
         assert isinstance(reopened, blosc2.BatchStore)
-        assert reopened.max_blocksize == barray.max_blocksize
+        assert reopened.items_per_block == barray.items_per_block
         assert [batch[:] for batch in reopened] == expected
         with pytest.raises(ValueError):
             reopened.append(["nope"])
@@ -203,7 +203,7 @@ def test_batchstore_from_cframe():
 def test_batchstore_msgpack_supports_blosc2_objects():
     ndarray, schunk, nested_vlarray, nested_batchstore, estore = _make_nested_blosc2_objects()
 
-    barray = blosc2.BatchStore(max_blocksize=2)
+    barray = blosc2.BatchStore(items_per_block=2)
     barray.append([ndarray, schunk, nested_vlarray, nested_batchstore, estore])
 
     restored = barray[0][:]
@@ -277,15 +277,15 @@ def fail_decode(*args, **kwargs):
 
 
 def test_batchstore_info_reports_exact_block_stats_from_lazy_chunks():
-    barray = blosc2.BatchStore(max_blocksize=2)
+    barray = blosc2.BatchStore(items_per_block=2)
     barray.extend([[1, 2, 3, 4, 5], [6, 7], [8]])
 
     items = dict(barray.info_items)
     assert items["nblocks"] == "5 (items per block: mean=1.60, max=2, min=1)"
 
 
 def test_batchstore_pop_keeps_batch_lengths_metadata_in_sync():
-    barray = blosc2.BatchStore(max_blocksize=2)
+    barray = blosc2.BatchStore(items_per_block=2)
     barray.extend([[1, 2, 3], [4, 5], [6]])
 
     removed = barray.pop(1)
@@ -335,9 +335,9 @@ def test_batchstore_zstd_does_not_use_dict_by_default():
     assert barray.cparams.use_dict is False
 
 
-def test_batchstore_explicit_max_blocksize():
-    barray = blosc2.BatchStore(max_blocksize=2)
-    assert barray.max_blocksize == 2
+def test_batchstore_explicit_items_per_block():
+    barray = blosc2.BatchStore(items_per_block=2)
+    assert barray.items_per_block == 2
     barray.append([1, 2, 3])
     barray.append([4])
     assert [batch[:] for batch in barray] == [[1, 2, 3], [4]]
@@ -348,10 +348,10 @@ def test_batchstore_get_vlblock_and_scalar_access():
     blosc2.remove_urlpath(urlpath)
 
     batch = [0, 1, 2, 3, 4]
-    barray = blosc2.BatchStore(storage=_storage(True, urlpath), max_blocksize=2)
+    barray = blosc2.BatchStore(storage=_storage(True, urlpath), items_per_block=2)
     barray.append(batch)
 
-    assert barray.max_blocksize == 2
+    assert barray.items_per_block == 2
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 0)) == batch[:2]
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 1)) == batch[2:4]
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 2)) == batch[4:]
@@ -362,7 +362,7 @@ def test_batchstore_get_vlblock_and_scalar_access():
 
     reopened = blosc2.open(urlpath, mode="r")
     assert isinstance(reopened, blosc2.BatchStore)
-    assert reopened.max_blocksize == 2
+    assert reopened.items_per_block == 2
     assert reopened[0][0] == 0
     assert reopened[0][2] == 2
     assert reopened[0][4] == 4
@@ -372,7 +372,7 @@ def test_batchstore_get_vlblock_and_scalar_access():
 
 
 def test_batchstore_scalar_reads_cache_vlblocks():
-    barray = blosc2.BatchStore(max_blocksize=2)
+    barray = blosc2.BatchStore(items_per_block=2)
     barray.append([0, 1, 2, 3, 4])
 
     batch = barray[0]
@@ -396,7 +396,7 @@ def wrapped_get_vlblock(nchunk, nblock):
 
 
 def test_batchstore_iter_items():
-    barray = blosc2.BatchStore(max_blocksize=2)
+    barray = blosc2.BatchStore(items_per_block=2)
     batches = [[1, 2, 3], [4], [5, 6]]
     barray.extend(batches)
 
@@ -424,21 +424,21 @@ def test_batchstore_respects_explicit_use_dict_and_non_zstd():
     assert barray.cparams.use_dict is False
 
 
-def test_batchstore_guess_max_blocksize_uses_l2_for_clevel_5(monkeypatch):
+def test_batchstore_guess_items_per_block_uses_l2_for_clevel_5(monkeypatch):
     monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 100)
     monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 1000)
     barray = blosc2.BatchStore(cparams={"clevel": 5})
     assert barray._guess_blocksize([30, 30, 30, 30]) == 4
 
 
-def test_batchstore_guess_max_blocksize_uses_l2_for_mid_clevel(monkeypatch):
+def test_batchstore_guess_items_per_block_uses_l2_for_mid_clevel(monkeypatch):
     monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 100)
     monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 150)
     barray = blosc2.BatchStore(cparams={"clevel": 6})
     assert barray._guess_blocksize([60, 60, 60, 60]) == 2
 
 
-def test_batchstore_guess_max_blocksize_uses_full_batch_for_clevel_9(monkeypatch):
+def test_batchstore_guess_items_per_block_uses_full_batch_for_clevel_9(monkeypatch):
     monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 1)
     monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 1)
     barray = blosc2.BatchStore(cparams={"clevel": 9})
@@ -596,7 +596,7 @@ def test_batchstore_items_accessor(contiguous, urlpath):
 
     batches = [["a", "b"], [10, 11, 12], [{"x": 1}], [None, True]]
     flat = [item for batch in batches for item in batch]
-    barray = blosc2.BatchStore(storage=_storage(contiguous, urlpath), max_blocksize=2)
+    barray = blosc2.BatchStore(storage=_storage(contiguous, urlpath), items_per_block=2)
     barray.extend(batches)
 
     assert len(barray.items) == len(flat)

tests/test_tree_store.py

@@ -675,7 +675,7 @@ def test_external_batchstore_support(tmp_path):
     store_path = tmp_path / "test_batchstore_external.b2d"
 
     with TreeStore(str(store_path), mode="w", threshold=0) as tstore:
-        bstore = blosc2.BatchStore(max_blocksize=2)
+        bstore = blosc2.BatchStore(items_per_block=2)
         bstore.extend([[{"id": 1}, {"id": 2}], [{"id": 3}]])
         tstore["/data/batchstore"] = bstore
 
@@ -693,7 +693,7 @@ def test_metadata_discovery_reopens_renamed_batchstore_leaf(storage_type, tmp_pa
     store_path = tmp_path / f"test_batchstore_renamed.{storage_type}"
 
     with TreeStore(str(store_path), mode="w", threshold=0) as tstore:
-        bstore = blosc2.BatchStore(max_blocksize=2)
+        bstore = blosc2.BatchStore(items_per_block=2)
         bstore.extend([[{"id": 1}, {"id": 2}], [{"id": 3}]])
         tstore["/data/batchstore"] = bstore
 

tests/test_vlarray.py

@@ -36,7 +36,7 @@ def _make_nested_blosc2_objects():
     nested_vlarray = blosc2.VLArray()
     nested_vlarray.extend(["alpha", {"beta": 2}])
 
-    nested_batchstore = blosc2.BatchStore(max_blocksize=2)
+    nested_batchstore = blosc2.BatchStore(items_per_block=2)
     nested_batchstore.extend([[1, 2], ["x", {"y": 3}]])
 
     estore = blosc2.EmbedStore()