Skip to content

Commit f4ff0aa

Browse files
JanKaulJanKaul
authored
Merge pull request JanKaul#191 from JanKaul/datafusion-48
update datafusion
2 parents b981147 + 477e639 commit f4ff0aa

8 files changed

Lines changed: 132 additions & 140 deletions

File tree

‎Cargo.lock‎

Lines changed: 106 additions & 100 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@ chrono = { version = "0.4", default-features = false, features = [
2323
"serde",
2424
"clock",
2525
] }
26-
datafusion = "47"
27-
datafusion-common = "47"
28-
datafusion-execution = "47"
29-
datafusion-expr = "47"
30-
datafusion-functions = { version = "47", features = ["crypto_expressions"] }
31-
datafusion-functions-aggregate = "47"
32-
datafusion-sql = "47"
26+
datafusion = "48"
27+
datafusion-common = "48"
28+
datafusion-execution = "48"
29+
datafusion-expr = "48"
30+
datafusion-functions = { version = "48", features = ["crypto_expressions"] }
31+
datafusion-functions-aggregate = "48"
32+
datafusion-sql = "48"
3333
derive-getters = "0.5.0"
3434
derive_builder = "0.20"
3535
futures = "0.3.31"

‎datafusion_iceberg/src/materialized_view/delta_queries/aggregate_functions.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ pub fn incremental_aggregate_function(
4747
)))),
4848
vec![
4949
(
50-
Box::new(Expr::Literal(ScalarValue::Boolean(Some(true)))),
50+
Box::new(Expr::Literal(ScalarValue::Boolean(Some(true)), None)),
5151
Box::new(Expr::Column(Column::new(
5252
None::<String>,
5353
left[0].name_for_alias()?,
5454
))),
5555
),
5656
(
57-
Box::new(Expr::Literal(ScalarValue::Boolean(Some(false)))),
57+
Box::new(Expr::Literal(ScalarValue::Boolean(Some(false)), None)),
5858
Box::new(right[0].clone()),
5959
),
6060
],
@@ -71,14 +71,14 @@ pub fn incremental_aggregate_function(
7171
)))),
7272
vec![
7373
(
74-
Box::new(Expr::Literal(ScalarValue::Boolean(Some(true)))),
74+
Box::new(Expr::Literal(ScalarValue::Boolean(Some(true)), None)),
7575
Box::new(Expr::Column(Column::new(
7676
None::<String>,
7777
left[0].name_for_alias()?,
7878
))),
7979
),
8080
(
81-
Box::new(Expr::Literal(ScalarValue::Boolean(Some(false)))),
81+
Box::new(Expr::Literal(ScalarValue::Boolean(Some(false)), None)),
8282
Box::new(right[0].clone()),
8383
),
8484
],

‎datafusion_iceberg/src/materialized_view/delta_queries/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ mod tests {
99
use std::{sync::Arc, time::Duration};
1010

1111
use datafusion::{
12-
arrow::array::{Float64Array, Int64Array, StringArray},
12+
arrow::array::{Float64Array, Int64Array, StringArray, StringViewArray},
1313
common::tree_node::{TransformedResult, TreeNode},
1414
execution::SessionStateBuilder,
1515
prelude::SessionContext,
@@ -1490,7 +1490,7 @@ GROUP BY
14901490
batch
14911491
.column(0)
14921492
.as_any()
1493-
.downcast_ref::<StringArray>()
1493+
.downcast_ref::<StringViewArray>()
14941494
.unwrap(),
14951495
batch
14961496
.column(1)
@@ -1689,7 +1689,7 @@ GROUP BY
16891689
batch
16901690
.column(0)
16911691
.as_any()
1692-
.downcast_ref::<StringArray>()
1692+
.downcast_ref::<StringViewArray>()
16931693
.unwrap(),
16941694
batch
16951695
.column(1)

‎datafusion_iceberg/src/materialized_view/delta_queries/transform.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ pub(crate) fn delta_transform_down(
162162
.clone()
163163
.into_iter()
164164
.zip(storage_table_aggregate_exprs.into_iter())
165-
.map(|(x, y)| incremental_aggregate_function(&vec![x], &vec![y]))
165+
.map(|(x, y)| incremental_aggregate_function(&[x], &[y]))
166166
.collect::<Result<Vec<_>, _>>()?,
167167
);
168168

‎datafusion_iceberg/src/pruning_statistics.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -332,19 +332,19 @@ fn transform_literal(expr: Expr, transform: &Transform) -> Result<Expr, DataFusi
332332
match transform {
333333
Transform::Year => Ok(Expr::ScalarFunction(ScalarFunction::new_udf(
334334
Arc::new(ScalarUDF::new_from_impl(DateTransform::new())),
335-
vec![Expr::Literal(ScalarValue::new_utf8("year")), expr],
335+
vec![Expr::Literal(ScalarValue::new_utf8("year"), None), expr],
336336
))),
337337
Transform::Month => Ok(Expr::ScalarFunction(ScalarFunction::new_udf(
338338
Arc::new(ScalarUDF::new_from_impl(DateTransform::new())),
339-
vec![Expr::Literal(ScalarValue::new_utf8("month")), expr],
339+
vec![Expr::Literal(ScalarValue::new_utf8("month"), None), expr],
340340
))),
341341
Transform::Day => Ok(Expr::ScalarFunction(ScalarFunction::new_udf(
342342
Arc::new(ScalarUDF::new_from_impl(DateTransform::new())),
343-
vec![Expr::Literal(ScalarValue::new_utf8("day")), expr],
343+
vec![Expr::Literal(ScalarValue::new_utf8("day"), None), expr],
344344
))),
345345
Transform::Hour => Ok(Expr::ScalarFunction(ScalarFunction::new_udf(
346346
Arc::new(ScalarUDF::new_from_impl(DateTransform::new())),
347-
vec![Expr::Literal(ScalarValue::new_utf8("hour")), expr],
347+
vec![Expr::Literal(ScalarValue::new_utf8("hour"), None), expr],
348348
))),
349349
_ => Ok(expr),
350350
}

‎datafusion_iceberg/src/table.rs‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@ async fn table_scan(
574574
let file_source = Arc::new(
575575
if let Some(physical_predicate) = physical_predicate.clone() {
576576
ParquetSource::default()
577-
.with_predicate(Arc::clone(&file_schema), physical_predicate)
577+
.with_predicate(physical_predicate)
578578
.with_pushdown_filters(true)
579579
} else {
580580
ParquetSource::default()
@@ -692,10 +692,7 @@ async fn table_scan(
692692
let delete_file_source = Arc::new(
693693
if let Some(physical_predicate) = physical_predicate.clone() {
694694
ParquetSource::default()
695-
.with_predicate(
696-
Arc::clone(&delete_file_schema),
697-
physical_predicate,
698-
)
695+
.with_predicate(physical_predicate)
699696
.with_pushdown_filters(true)
700697
} else {
701698
ParquetSource::default()
@@ -714,11 +711,7 @@ async fn table_scan(
714711
.build();
715712

716713
let left = ParquetFormat::default()
717-
.create_physical_plan(
718-
session,
719-
delete_file_scan_config,
720-
physical_predicate.as_ref(),
721-
)
714+
.create_physical_plan(session, delete_file_scan_config)
722715
.await?;
723716

724717
let file_scan_config = FileScanConfigBuilder::new(
@@ -734,11 +727,7 @@ async fn table_scan(
734727
.build();
735728

736729
let data_files_scan = ParquetFormat::default()
737-
.create_physical_plan(
738-
session,
739-
file_scan_config,
740-
physical_predicate.as_ref(),
741-
)
730+
.create_physical_plan(session, file_scan_config)
742731
.await?;
743732

744733
let right = if let Some(acc) = acc {
@@ -818,11 +807,7 @@ async fn table_scan(
818807
.build();
819808

820809
let data_files_scan = ParquetFormat::default()
821-
.create_physical_plan(
822-
session,
823-
file_scan_config,
824-
physical_predicate.as_ref(),
825-
)
810+
.create_physical_plan(session, file_scan_config)
826811
.await?;
827812

828813
plan = Arc::new(UnionExec::new(vec![plan, data_files_scan]));
@@ -874,7 +859,7 @@ async fn table_scan(
874859
.build();
875860

876861
let other_plan = ParquetFormat::default()
877-
.create_physical_plan(session, file_scan_config, physical_predicate.as_ref())
862+
.create_physical_plan(session, file_scan_config)
878863
.await?;
879864

880865
if plans.is_empty() {

‎iceberg-rust-spec/src/arrow/schema.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ impl TryFrom<&DataType> for Type {
160160
DataType::Time64(_) => Ok(Type::Primitive(PrimitiveType::Time)),
161161
DataType::Timestamp(_, _) => Ok(Type::Primitive(PrimitiveType::Timestamp)),
162162
DataType::Utf8 => Ok(Type::Primitive(PrimitiveType::String)),
163+
DataType::Utf8View => Ok(Type::Primitive(PrimitiveType::String)),
163164
DataType::FixedSizeBinary(len) => {
164165
Ok(Type::Primitive(PrimitiveType::Fixed(*len as u64)))
165166
}

0 commit comments

Comments
 (0)