Skip to content

Commit a9e5915

Browse files
committed
adapt to improved async iteration
1 parent 775d00c commit a9e5915

2 files changed

Lines changed: 4 additions & 12 deletions

File tree

src/crawlee/storage_clients/_file_system/_dataset_client.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -185,14 +185,10 @@ async def iterate_items(
185185
f'by the {self.__class__.__name__} client.'
186186
)
187187

188-
# The native client returns a list rather than an async iterator,
189-
# so we fetch all matching items and yield them one by one.
190-
items: list[Any] = await self._native_client.iterate_items(
188+
async for item in self._native_client.iterate_items(
191189
offset=offset,
192190
limit=limit,
193191
desc=desc,
194192
skip_empty=skip_empty,
195-
)
196-
197-
for item in items:
193+
):
198194
yield item

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,14 +139,10 @@ async def iterate_keys(
139139
exclusive_start_key: str | None = None,
140140
limit: int | None = None,
141141
) -> AsyncIterator[KeyValueStoreRecordMetadata]:
142-
# The native client returns a list, so we fetch all matching keys
143-
# and yield them one by one.
144-
items: list[dict[str, Any]] = await self._native_client.iterate_keys(
142+
async for item in self._native_client.iterate_keys(
145143
exclusive_start_key=exclusive_start_key,
146144
limit=limit,
147-
)
148-
149-
for item in items:
145+
):
150146
yield KeyValueStoreRecordMetadata(**item)
151147

152148
@override

0 commit comments

Comments
 (0)