Skip to content

Commit edd2f08

Browse files
author
Zach Banks
committed
MOD: Add parquet_schema to DBNStore.to_parquet()
1 parent 81774cd commit edd2f08

2 files changed

Lines changed: 12 additions & 2 deletions

File tree

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Changelog
22

3+
## TBD
4+
5+
#### Enhancements
6+
- Added `parquet_schema` option to `DBNStore.to_parquet()` for overriding the pyarrow schema.
7+
38
## 0.59.0 - 2025-07-15
49

510
#### Enhancements

databento/common/dbnstore.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -984,6 +984,7 @@ def to_parquet(
984984
map_symbols: bool = True,
985985
schema: Schema | str | None = None,
986986
mode: Literal["w", "x"] = "w",
987+
parquet_schema: pa.Schema | None = None,
987988
**kwargs: Any,
988989
) -> None:
989990
"""
@@ -1010,6 +1011,9 @@ def to_parquet(
10101011
This is only required when reading a DBN stream with mixed record types.
10111012
mode : str, default "w"
10121013
The file write mode to use, either "x" or "w".
1014+
parquet_schema : pyarrow.Schema, optional
1015+
The pyarrow parquet schema to use to write the parquet file.
1016+
This defaults to a detected schema based on the DataFrame representation.
10131017
**kwargs : Any
10141018
Keyword arguments to pass to the `pyarrow.parquet.ParquetWriter`.
10151019
These can be used to override the default behavior of the writer.
@@ -1046,10 +1050,11 @@ def to_parquet(
10461050
for frame in dataframe_iter:
10471051
if writer is None:
10481052
# Initialize the writer using the first DataFrame
1049-
parquet_schema = pa.Schema.from_pandas(frame)
1053+
if parquet_schema is None:
1054+
parquet_schema = pa.Schema.from_pandas(frame)
10501055
writer = pq.ParquetWriter(
10511056
where=kwargs.pop("where", file_path),
1052-
schema=kwargs.pop("schema", parquet_schema),
1057+
schema=parquet_schema,
10531058
**kwargs,
10541059
)
10551060
writer.write_table(

0 commit comments

Comments
 (0)