Skip to content

Commit 875fdb7

Browse files
authored
chore(deps): Bump DataFusion to 54.0.0 (#2648)
## Which issue does this PR close? DataFusion to 54.0.0 was recently released, we should try and track latests releases. <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes #. ## What changes are included in this PR? [ExecutionPlan](apache/datafusion#21263) [TableProvider, SchemaProvider, CatalogProvider, CatalogProviderList](apache/datafusion#21346) and [PhysicalExpr](apache/datafusion#21573) have all had their `as_any` removed from the trait. That's really the only change needed here. [Upgrade guide documents this](https://datafusion.apache.org/library-user-guide/upgrading/54.0.0.html#remove-as-any-from-physicalexpr-scalarudfimpl-aggregateudfimpl-windowudfimpl-executionplan-tableprovider-schemaprovider-catalogprovider-catalogproviderlist-tablesource-filesource-fileformat-fileformatfactory-datasource-and-datasink) <!-- Provide a summary of the modifications in this PR. List the main changes such as new features, bug fixes, refactoring, or any other updates. --> ## Are these changes tested? <!-- Specify what test covers (unit test, integration test, etc.). If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? -->
1 parent 894108b commit 875fdb7

15 files changed

Lines changed: 1026 additions & 1528 deletions

File tree

Cargo.lock

Lines changed: 1014 additions & 1445 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ cfg-if = "1"
7474
chrono = "0.4.41"
7575
clap = { version = "4.5.48", features = ["derive", "cargo"] }
7676
dashmap = "6"
77-
datafusion = "53.1.0"
78-
datafusion-cli = "53.0.0"
79-
datafusion-ffi = "53.0.0"
80-
datafusion-sqllogictest = "53.0.0"
77+
datafusion = "54.0.0"
78+
datafusion-cli = "54.0.0"
79+
datafusion-ffi = "54.0.0"
80+
datafusion-sqllogictest = "54.0.0"
8181
derive_builder = "0.20"
8282
dirs = "6"
8383
enum-ordinalize = "4.3.0"
@@ -93,7 +93,7 @@ fnv = "1.0.7"
9393
fs-err = "3.1.0"
9494
futures = "0.3"
9595
hive_metastore = "0.2.0"
96-
home = "=0.5.11"
96+
home = "0.5.12"
9797
http = "1.2"
9898
iceberg = { version = "0.10.0", path = "./crates/iceberg" }
9999
iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" }

crates/integrations/datafusion/public-api.txt

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&
88
impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider
99
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
1010
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider
11-
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::as_any(&self) -> &dyn core::any::Any
1211
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
1312
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef
1413
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType
@@ -25,7 +24,6 @@ pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt(&self, f: &mut c
2524
impl datafusion_physical_plan::display::DisplayAs for iceberg_datafusion::physical_plan::IcebergTableScan
2625
pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt_as(&self, _t: datafusion_physical_plan::display::DisplayFormatType, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
2726
impl datafusion_physical_plan::execution_plan::ExecutionPlan for iceberg_datafusion::physical_plan::IcebergTableScan
28-
pub fn iceberg_datafusion::physical_plan::IcebergTableScan::as_any(&self) -> &dyn core::any::Any
2927
pub fn iceberg_datafusion::physical_plan::IcebergTableScan::children(&self) -> alloc::vec::Vec<&alloc::sync::Arc<(dyn datafusion_physical_plan::execution_plan::ExecutionPlan + 'static)>>
3028
pub fn iceberg_datafusion::physical_plan::IcebergTableScan::execute(&self, _partition: usize, _context: alloc::sync::Arc<datafusion_execution::task::TaskContext>) -> datafusion_common::error::Result<datafusion_execution::stream::SendableRecordBatchStream>
3129
pub fn iceberg_datafusion::physical_plan::IcebergTableScan::name(&self) -> &str
@@ -43,7 +41,6 @@ pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&
4341
impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider
4442
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
4543
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider
46-
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::as_any(&self) -> &dyn core::any::Any
4744
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
4845
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef
4946
pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType
@@ -67,7 +64,6 @@ pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_d
6764
impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider
6865
pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
6966
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider
70-
pub fn iceberg_datafusion::IcebergStaticTableProvider::as_any(&self) -> &dyn core::any::Any
7167
pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait
7268
pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
7369
pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef
@@ -79,7 +75,6 @@ pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafus
7975
impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider
8076
pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
8177
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergTableProvider
82-
pub fn iceberg_datafusion::IcebergTableProvider::as_any(&self) -> &dyn core::any::Any
8378
pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait
8479
pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
8580
pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef
@@ -102,7 +97,6 @@ pub async fn iceberg_datafusion::IcebergCatalogProvider::try_new(client: alloc::
10297
impl core::fmt::Debug for iceberg_datafusion::IcebergCatalogProvider
10398
pub fn iceberg_datafusion::IcebergCatalogProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
10499
impl datafusion_catalog::catalog::CatalogProvider for iceberg_datafusion::IcebergCatalogProvider
105-
pub fn iceberg_datafusion::IcebergCatalogProvider::as_any(&self) -> &dyn core::any::Any
106100
pub fn iceberg_datafusion::IcebergCatalogProvider::schema(&self, name: &str) -> core::option::Option<alloc::sync::Arc<dyn datafusion_catalog::schema::SchemaProvider>>
107101
pub fn iceberg_datafusion::IcebergCatalogProvider::schema_names(&self) -> alloc::vec::Vec<alloc::string::String>
108102
pub struct iceberg_datafusion::IcebergStaticTableProvider
@@ -114,7 +108,6 @@ pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_d
114108
impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider
115109
pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
116110
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider
117-
pub fn iceberg_datafusion::IcebergStaticTableProvider::as_any(&self) -> &dyn core::any::Any
118111
pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait
119112
pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
120113
pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef
@@ -126,7 +119,6 @@ pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafus
126119
impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider
127120
pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
128121
impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergTableProvider
129-
pub fn iceberg_datafusion::IcebergTableProvider::as_any(&self) -> &dyn core::any::Any
130122
pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait
131123
pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec<usize>>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option<usize>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = datafusion_common::error::Result<alloc::sync::Arc<dyn datafusion_physical_plan::execution_plan::ExecutionPlan>>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait
132124
pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef

crates/integrations/datafusion/src/catalog.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::any::Any;
1918
use std::collections::HashMap;
2019
use std::sync::Arc;
2120

@@ -84,10 +83,6 @@ impl IcebergCatalogProvider {
8483
}
8584

8685
impl CatalogProvider for IcebergCatalogProvider {
87-
fn as_any(&self) -> &dyn Any {
88-
self
89-
}
90-
9186
fn schema_names(&self) -> Vec<String> {
9287
self.schemas.keys().cloned().collect()
9388
}

crates/integrations/datafusion/src/physical_plan/commit.rs

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::any::Any;
1918
use std::fmt::{Debug, Formatter};
2019
use std::sync::Arc;
2120

@@ -129,10 +128,6 @@ impl ExecutionPlan for IcebergCommitExec {
129128
"IcebergCommitExec"
130129
}
131130

132-
fn as_any(&self) -> &dyn Any {
133-
self
134-
}
135-
136131
fn properties(&self) -> &Arc<PlanProperties> {
137132
&self.plan_properties
138133
}
@@ -331,10 +326,6 @@ mod tests {
331326
"MockWriteExec"
332327
}
333328

334-
fn as_any(&self) -> &dyn Any {
335-
self
336-
}
337-
338329
fn schema(&self) -> Arc<ArrowSchema> {
339330
self.schema.clone()
340331
}

crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ fn to_iceberg_predicate(expr: &Expr) -> TransformedResult {
140140
}
141141
}
142142
Expr::Cast(c) => {
143-
if c.data_type == DataType::Date32 || c.data_type == DataType::Date64 {
143+
if *c.field.data_type() == DataType::Date32 || *c.field.data_type() == DataType::Date64
144+
{
144145
// Casts to date truncate the expression, we cannot simply extract it as it
145146
// can create erroneous predicates.
146147
return TransformedResult::NotTransformed;
@@ -262,7 +263,9 @@ fn resolve_nan_preserving_reference(expr: &Expr) -> Option<Reference> {
262263
Expr::Cast(cast) => {
263264
// Casts to date truncate the value and are not numeric, so they
264265
// cannot be treated as NaN-preserving.
265-
if cast.data_type == DataType::Date32 || cast.data_type == DataType::Date64 {
266+
if *cast.field.data_type() == DataType::Date32
267+
|| *cast.field.data_type() == DataType::Date64
268+
{
266269
return None;
267270
}
268271
resolve_nan_preserving_reference(&cast.expr)

crates/integrations/datafusion/src/physical_plan/metadata_scan.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,6 @@ impl ExecutionPlan for IcebergMetadataScan {
6262
"IcebergMetadataScan"
6363
}
6464

65-
fn as_any(&self) -> &dyn std::any::Any {
66-
self
67-
}
68-
6965
fn properties(&self) -> &Arc<PlanProperties> {
7066
&self.properties
7167
}

crates/integrations/datafusion/src/physical_plan/project.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,6 @@ impl PartialEq for PartitionExpr {
126126
impl Eq for PartitionExpr {}
127127

128128
impl PhysicalExpr for PartitionExpr {
129-
fn as_any(&self) -> &dyn std::any::Any {
130-
self
131-
}
132-
133129
fn data_type(&self, _input_schema: &ArrowSchema) -> DFResult<DataType> {
134130
Ok(self.calculator.partition_arrow_type().clone())
135131
}

crates/integrations/datafusion/src/physical_plan/repartition.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -503,8 +503,7 @@ mod tests {
503503
let column_names: Vec<String> = exprs
504504
.iter()
505505
.filter_map(|expr| {
506-
expr.as_any()
507-
.downcast_ref::<Column>()
506+
expr.downcast_ref::<Column>()
508507
.map(|col| col.name().to_string())
509508
})
510509
.collect();
@@ -731,8 +730,7 @@ mod tests {
731730
let column_names: Vec<String> = exprs
732731
.iter()
733732
.filter_map(|expr| {
734-
expr.as_any()
735-
.downcast_ref::<Column>()
733+
expr.downcast_ref::<Column>()
736734
.map(|col| col.name().to_string())
737735
})
738736
.collect();

crates/integrations/datafusion/src/physical_plan/scan.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::any::Any;
1918
use std::pin::Pin;
2019
use std::sync::Arc;
2120
use std::vec;
@@ -122,10 +121,6 @@ impl ExecutionPlan for IcebergTableScan {
122121
"IcebergTableScan"
123122
}
124123

125-
fn as_any(&self) -> &dyn Any {
126-
self
127-
}
128-
129124
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan + 'static>> {
130125
vec![]
131126
}

0 commit comments

Comments
 (0)