From 911a0b3bb96d5753e30af5dcfa207a221676da12 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 Dec 2024 15:20:39 -0700 Subject: [PATCH] clippy --- .../expressions/bloom_filter_might_contain.rs | 19 +++---- .../datafusion/expressions/checkoverflow.rs | 24 ++++---- .../datafusion/expressions/negative.rs | 13 +++-- .../datafusion/expressions/strings.rs | 25 ++++----- .../datafusion/expressions/subquery.rs | 2 +- .../datafusion/expressions/unbound.rs | 6 +- .../datafusion/util/spark_bit_array.rs | 2 +- .../datafusion/util/spark_bloom_filter.rs | 2 +- native/core/src/execution/jni_api.rs | 9 ++- native/spark-expr/src/bitwise_not.rs | 13 ++++- native/spark-expr/src/cast.rs | 31 ++++++---- native/spark-expr/src/if_expr.rs | 13 ++++- native/spark-expr/src/list.rs | 28 +++++----- native/spark-expr/src/normalize_nan.rs | 29 ++++++---- native/spark-expr/src/structs.rs | 12 ++-- native/spark-expr/src/temporal.rs | 56 +++++++++---------- native/spark-expr/src/to_json.rs | 12 ++-- 17 files changed, 156 insertions(+), 140 deletions(-) diff --git a/native/core/src/execution/datafusion/expressions/bloom_filter_might_contain.rs b/native/core/src/execution/datafusion/expressions/bloom_filter_might_contain.rs index 3e52e4f18..b98f0ca0f 100644 --- a/native/core/src/execution/datafusion/expressions/bloom_filter_might_contain.rs +++ b/native/core/src/execution/datafusion/expressions/bloom_filter_might_contain.rs @@ -15,42 +15,37 @@ // specific language governing permissions and limitations // under the License. -use crate::execution::datafusion::expressions::strings::SubstringExpr; use crate::{ execution::datafusion::util::spark_bloom_filter::SparkBloomFilter, parquet::data_type::AsBytes, }; use arrow::record_batch::RecordBatch; use arrow_array::cast::as_primitive_array; use arrow_schema::{DataType, Schema}; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion::physical_plan::ColumnarValue; use datafusion_comet_spark_expr::utils::down_cast_any_ref; use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_physical_expr::PhysicalExpr; -use std::{ - any::Any, - fmt::Display, - hash::{Hash, Hasher}, - sync::Arc, -}; +use std::hash::Hasher; +use std::{any::Any, fmt::Display, sync::Arc}; /// A physical expression that checks if a value might be in a bloom filter. It corresponds to the /// Spark's `BloomFilterMightContain` expression. -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct BloomFilterMightContain { pub bloom_filter_expr: Arc, pub value_expr: Arc, bloom_filter: Option, } -impl DynEq for BloomFilterMightContain { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for BloomFilterMightContain { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for BloomFilterMightContain { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/core/src/execution/datafusion/expressions/checkoverflow.rs b/native/core/src/execution/datafusion/expressions/checkoverflow.rs index e67283dc9..ef4facdad 100644 --- a/native/core/src/execution/datafusion/expressions/checkoverflow.rs +++ b/native/core/src/execution/datafusion/expressions/checkoverflow.rs @@ -15,14 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::{ - any::Any, - fmt::{Display, Formatter}, - hash::{Hash, Hasher}, - sync::Arc, -}; - -use crate::execution::datafusion::expressions::strings::SubstringExpr; use arrow::{ array::{as_primitive_array, Array, ArrayRef, Decimal128Array}, datatypes::{Decimal128Type, DecimalType}, @@ -30,30 +22,36 @@ use arrow::{ }; use arrow_schema::{DataType, Schema}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_comet_spark_expr::utils::down_cast_any_ref; use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; +use std::{ + any::Any, + fmt::{Display, Formatter}, + sync::Arc, +}; /// This is from Spark `CheckOverflow` expression. Spark `CheckOverflow` expression rounds decimals /// to given scale and check if the decimals can fit in given precision. As `cast` kernel rounds /// decimals already, Comet `CheckOverflow` expression only checks if the decimals can fit in the /// precision. -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct CheckOverflow { pub child: Arc, pub data_type: DataType, pub fail_on_error: bool, } -impl DynEq for CheckOverflow { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for CheckOverflow { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for CheckOverflow { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/core/src/execution/datafusion/expressions/negative.rs b/native/core/src/execution/datafusion/expressions/negative.rs index c58f4b6a8..7cd66c152 100644 --- a/native/core/src/execution/datafusion/expressions/negative.rs +++ b/native/core/src/execution/datafusion/expressions/negative.rs @@ -21,7 +21,7 @@ use arrow::{compute::kernels::numeric::neg_wrapping, datatypes::IntervalDayTimeT use arrow_array::RecordBatch; use arrow_buffer::IntervalDayTime; use arrow_schema::{DataType, Schema}; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion::{ logical_expr::{interval_arithmetic::Interval, ColumnarValue}, physical_expr::PhysicalExpr, @@ -30,7 +30,8 @@ use datafusion_comet_spark_expr::utils::down_cast_any_ref; use datafusion_comet_spark_expr::SparkError; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::sort_properties::ExprProperties; -use std::{any::Any, hash::Hash, sync::Arc}; +use std::hash::Hasher; +use std::{any::Any, sync::Arc}; pub fn create_negate_expr( expr: Arc, @@ -40,21 +41,21 @@ pub fn create_negate_expr( } /// Negative expression -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct NegativeExpr { /// Input expression arg: Arc, fail_on_error: bool, } -impl DynEq for NegativeExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for NegativeExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for NegativeExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/core/src/execution/datafusion/expressions/strings.rs b/native/core/src/execution/datafusion/expressions/strings.rs index 6d972c86a..3aef4cd69 100644 --- a/native/core/src/execution/datafusion/expressions/strings.rs +++ b/native/core/src/execution/datafusion/expressions/strings.rs @@ -17,7 +17,6 @@ #![allow(deprecated)] -use crate::execution::datafusion::expressions::checkoverflow::CheckOverflow; use crate::execution::kernels::strings::{string_space, substring}; use arrow::{ compute::{ @@ -28,18 +27,18 @@ use arrow::{ }; use arrow_schema::{DataType, Schema}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; -use datafusion::physical_expr_common::physical_expr::DynHash; +use datafusion::physical_expr_common::physical_expr::{DynEq, DynHash}; use datafusion_comet_spark_expr::utils::down_cast_any_ref; -use datafusion_comet_spark_expr::ToJson; use datafusion_common::{DataFusionError, ScalarValue::Utf8}; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; use std::{ any::Any, fmt::{Display, Formatter}, - hash::{Hash, Hasher}, + hash::Hash, sync::Arc, }; + macro_rules! make_predicate_function { ($name: ident, $kernel: ident, $str_scalar_kernel: ident) => { #[derive(Debug, Hash)] @@ -155,26 +154,26 @@ make_predicate_function!(EndsWith, ends_with_dyn, ends_with_utf8_scalar_dyn); make_predicate_function!(Contains, contains_dyn, contains_utf8_scalar_dyn); -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct SubstringExpr { pub child: Arc, pub start: i64, pub len: u64, } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct StringSpaceExpr { pub child: Arc, } -impl DynEq for StringSpaceExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for StringSpaceExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for StringSpaceExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -216,14 +215,14 @@ impl PartialEq for SubstringExpr { } } -impl DynEq for SubstringExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for SubstringExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for SubstringExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/core/src/execution/datafusion/expressions/subquery.rs b/native/core/src/execution/datafusion/expressions/subquery.rs index 3be7df9d2..a3eaa8c5c 100644 --- a/native/core/src/execution/datafusion/expressions/subquery.rs +++ b/native/core/src/execution/datafusion/expressions/subquery.rs @@ -32,7 +32,7 @@ use jni::{ use std::{ any::Any, fmt::{Display, Formatter}, - hash::{Hash, Hasher}, + hash::Hash, sync::Arc, }; diff --git a/native/core/src/execution/datafusion/expressions/unbound.rs b/native/core/src/execution/datafusion/expressions/unbound.rs index f0395103c..b0f239d89 100644 --- a/native/core/src/execution/datafusion/expressions/unbound.rs +++ b/native/core/src/execution/datafusion/expressions/unbound.rs @@ -21,11 +21,7 @@ use datafusion::physical_plan::ColumnarValue; use datafusion_comet_spark_expr::utils::down_cast_any_ref; use datafusion_common::{internal_err, Result}; use datafusion_physical_expr::PhysicalExpr; -use std::{ - any::Any, - hash::{Hash, Hasher}, - sync::Arc, -}; +use std::{any::Any, hash::Hash, sync::Arc}; /// This is similar to `UnKnownColumn` in DataFusion, but it has data type. /// This is only used when the column is not bound to a schema, for example, the diff --git a/native/core/src/execution/datafusion/util/spark_bit_array.rs b/native/core/src/execution/datafusion/util/spark_bit_array.rs index 68b97d660..923df014f 100644 --- a/native/core/src/execution/datafusion/util/spark_bit_array.rs +++ b/native/core/src/execution/datafusion/util/spark_bit_array.rs @@ -22,7 +22,7 @@ use std::iter::zip; /// A simple bit array implementation that simulates the behavior of Spark's BitArray which is /// used in the BloomFilter implementation. Some methods are not implemented as they are not /// required for the current use case. -#[derive(Debug, Hash)] +#[derive(Debug, Hash, PartialEq, Eq)] pub struct SparkBitArray { data: Vec, bit_count: usize, diff --git a/native/core/src/execution/datafusion/util/spark_bloom_filter.rs b/native/core/src/execution/datafusion/util/spark_bloom_filter.rs index 35fa23b46..f42a148c6 100644 --- a/native/core/src/execution/datafusion/util/spark_bloom_filter.rs +++ b/native/core/src/execution/datafusion/util/spark_bloom_filter.rs @@ -27,7 +27,7 @@ const SPARK_BLOOM_FILTER_VERSION_1: i32 = 1; /// A Bloom filter implementation that simulates the behavior of Spark's BloomFilter. /// It's not a complete implementation of Spark's BloomFilter, but just add the minimum /// methods to support mightContainsLong in the native side. -#[derive(Debug, Hash)] +#[derive(Debug, Hash, PartialEq, Eq)] pub struct SparkBloomFilter { bits: SparkBitArray, num_hash_functions: u32, diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 8afe134cd..8dab0088f 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -20,10 +20,7 @@ use arrow::datatypes::DataType as ArrowDataType; use arrow_array::RecordBatch; use datafusion::{ - execution::{ - disk_manager::DiskManagerConfig, - runtime_env::{RuntimeConfig, RuntimeEnv}, - }, + execution::{disk_manager::DiskManagerConfig, runtime_env::RuntimeEnv}, physical_plan::{display::DisplayableExecutionPlan, SendableRecordBatchStream}, prelude::{SessionConfig, SessionContext}, }; @@ -52,6 +49,7 @@ use crate::{ }; use datafusion_comet_proto::spark_operator::Operator; use datafusion_common::ScalarValue; +use datafusion_execution::runtime_env::RuntimeEnvBuilder; use futures::stream::StreamExt; use jni::{ objects::GlobalRef, @@ -176,7 +174,7 @@ fn prepare_datafusion_session_context( batch_size: usize, comet_task_memory_manager: Arc, ) -> CometResult { - let mut rt_config = RuntimeConfig::new().with_disk_manager(DiskManagerConfig::NewOs); + let mut rt_config = RuntimeEnvBuilder::new().with_disk_manager(DiskManagerConfig::NewOs); // Set Comet memory pool for native let memory_pool = CometMemoryPool::new(comet_task_memory_manager); @@ -198,6 +196,7 @@ fn prepare_datafusion_session_context( &ScalarValue::Float64(Some(1.1)), ); + #[allow(deprecated)] let runtime = RuntimeEnv::try_new(rt_config)?; let mut session_ctx = SessionContext::new_with_config_rt(session_config, Arc::new(runtime)); diff --git a/native/spark-expr/src/bitwise_not.rs b/native/spark-expr/src/bitwise_not.rs index 677bdb596..818064554 100644 --- a/native/spark-expr/src/bitwise_not.rs +++ b/native/spark-expr/src/bitwise_not.rs @@ -15,17 +15,18 @@ // specific language governing permissions and limitations // under the License. -use std::{any::Any, hash::Hash, sync::Arc}; - use crate::utils::down_cast_any_ref; use arrow::{ array::*, datatypes::{DataType, Schema}, record_batch::RecordBatch, }; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion::{error::DataFusionError, logical_expr::ColumnarValue}; use datafusion_common::Result; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; +use std::{any::Any, sync::Arc}; macro_rules! compute_op { ($OPERAND:expr, $DT:ident) => {{ @@ -39,12 +40,18 @@ macro_rules! compute_op { } /// BitwiseNot expression -#[derive(Debug, Hash, Eq)] +#[derive(Debug, Eq)] pub struct BitwiseNotExpr { /// Input expression arg: Arc, } +impl DynHash for BitwiseNotExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { + todo!() + } +} + impl PartialEq for BitwiseNotExpr { fn eq(&self, other: &Self) -> bool { self.arg.eq(&other.arg) diff --git a/native/spark-expr/src/cast.rs b/native/spark-expr/src/cast.rs index 471242f9e..8cfeada0e 100644 --- a/native/spark-expr/src/cast.rs +++ b/native/spark-expr/src/cast.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::utils::down_cast_any_ref; use arrow::{ array::{ cast::AsArray, @@ -35,28 +36,28 @@ use arrow::{ use arrow_array::builder::StringBuilder; use arrow_array::{DictionaryArray, StringArray, StructArray}; use arrow_schema::{DataType, Field, Schema}; +use chrono::{NaiveDate, NaiveDateTime, TimeZone, Timelike}; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::{ cast::as_generic_string_array, internal_err, Result as DataFusionResult, ScalarValue, }; use datafusion_expr::ColumnarValue; use datafusion_physical_expr::PhysicalExpr; +use num::{ + cast::AsPrimitive, integer::div_floor, traits::CheckedNeg, CheckedSub, Integer, Num, + ToPrimitive, +}; +use regex::Regex; +use std::hash::Hasher; use std::str::FromStr; use std::{ any::Any, fmt::{Debug, Display, Formatter}, - hash::{Hash, Hasher}, + hash::Hash, num::Wrapping, sync::Arc, }; -use crate::utils::down_cast_any_ref; -use chrono::{NaiveDate, NaiveDateTime, TimeZone, Timelike}; -use num::{ - cast::AsPrimitive, integer::div_floor, traits::CheckedNeg, CheckedSub, Integer, Num, - ToPrimitive, -}; -use regex::Regex; - use crate::timezone; use crate::utils::array_with_timezone; @@ -134,7 +135,7 @@ impl TimeStampInfo { } } -#[derive(Debug, Hash, Eq)] +#[derive(Debug, Eq)] pub struct Cast { pub child: Arc, pub data_type: DataType, @@ -1450,6 +1451,16 @@ impl Display for Cast { } } +impl DynHash for Cast { + fn dyn_hash(&self, state: &mut dyn Hasher) { + let mut s = state; + self.type_id().hash(&mut s); + self.child.hash(&mut s); + self.data_type.hash(&mut s); + self.cast_options.hash(&mut s); + } +} + impl PartialEq for Cast { fn eq(&self, other: &dyn Any) -> bool { down_cast_any_ref(other) diff --git a/native/spark-expr/src/if_expr.rs b/native/spark-expr/src/if_expr.rs index a0b01bb61..d6eeffb4b 100644 --- a/native/spark-expr/src/if_expr.rs +++ b/native/spark-expr/src/if_expr.rs @@ -15,20 +15,21 @@ // specific language governing permissions and limitations // under the License. -use std::{any::Any, hash::Hash, sync::Arc}; - use crate::utils::down_cast_any_ref; use arrow::{ datatypes::{DataType, Schema}, record_batch::RecordBatch, }; use datafusion::logical_expr::ColumnarValue; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::Result; use datafusion_physical_expr::{expressions::CaseExpr, PhysicalExpr}; +use std::hash::Hasher; +use std::{any::Any, sync::Arc}; /// IfExpr is a wrapper around CaseExpr, because `IF(a, b, c)` is semantically equivalent to /// `CASE WHEN a THEN b ELSE c END`. -#[derive(Debug, Hash, Eq)] +#[derive(Debug, Eq)] pub struct IfExpr { if_expr: Arc, true_expr: Arc, @@ -37,6 +38,12 @@ pub struct IfExpr { case_expr: Arc, } +impl DynHash for IfExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { + todo!() + } +} + impl PartialEq for IfExpr { fn eq(&self, other: &Self) -> bool { self.if_expr.eq(&other.if_expr) diff --git a/native/spark-expr/src/list.rs b/native/spark-expr/src/list.rs index a313f88da..7445784d4 100644 --- a/native/spark-expr/src/list.rs +++ b/native/spark-expr/src/list.rs @@ -27,16 +27,16 @@ use arrow_array::{ }; use arrow_schema::{DataType, Field, FieldRef, Schema}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::{ cast::{as_int32_array, as_large_list_array, as_list_array}, internal_err, DataFusionError, Result as DataFusionResult, ScalarValue, }; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; use std::{ any::Any, fmt::{Debug, Display, Formatter}, - hash::Hash, sync::Arc, }; @@ -45,7 +45,7 @@ use std::{ // https://github.com/apache/spark/blob/master/common/utils/src/main/java/org/apache/spark/unsafe/array/ByteArrayUtils.java const MAX_ROUNDED_ARRAY_LENGTH: usize = 2147483632; -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct ListExtract { child: Arc, ordinal: Arc, @@ -54,14 +54,14 @@ pub struct ListExtract { fail_on_error: bool, } -impl DynEq for ListExtract { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for ListExtract { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for ListExtract { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -291,20 +291,20 @@ impl PartialEq for ListExtract { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct GetArrayStructFields { child: Arc, ordinal: usize, } -impl DynEq for GetArrayStructFields { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for GetArrayStructFields { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for GetArrayStructFields { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -434,7 +434,7 @@ impl PartialEq for GetArrayStructFields { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct ArrayInsert { src_array_expr: Arc, pos_expr: Arc, @@ -442,14 +442,14 @@ pub struct ArrayInsert { legacy_negative_index: bool, } -impl DynEq for ArrayInsert { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for ArrayInsert { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for ArrayInsert { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/spark-expr/src/normalize_nan.rs b/native/spark-expr/src/normalize_nan.rs index 0b9a5df8a..5ead1489b 100644 --- a/native/spark-expr/src/normalize_nan.rs +++ b/native/spark-expr/src/normalize_nan.rs @@ -15,13 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::{ - any::Any, - fmt::{Display, Formatter}, - hash::Hash, - sync::Arc, -}; - use crate::utils::down_cast_any_ref; use arrow::{ array::{as_primitive_array, ArrayAccessor, ArrayIter, Float32Array, Float64Array}, @@ -30,23 +23,35 @@ use arrow::{ }; use arrow_schema::{DataType, Schema}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; +use std::{ + any::Any, + fmt::{Display, Formatter}, + sync::Arc, +}; -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct NormalizeNaNAndZero { pub data_type: DataType, pub child: Arc, } -impl DynEq for NormalizeNaNAndZero { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for NormalizeNaNAndZero { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } +// impl DynEq for NormalizeNaNAndZero { +// fn dyn_eq(&self, _other: &dyn Any) -> bool { +// todo!() +// } +// } + impl PartialEq for NormalizeNaNAndZero { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/spark-expr/src/structs.rs b/native/spark-expr/src/structs.rs index 3c450ab85..48ef43785 100644 --- a/native/spark-expr/src/structs.rs +++ b/native/spark-expr/src/structs.rs @@ -16,14 +16,14 @@ // under the License. use crate::utils::down_cast_any_ref; -use crate::ArrayInsert; use arrow::record_batch::RecordBatch; use arrow_array::{Array, StructArray}; use arrow_schema::{DataType, Field, Schema}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::{DataFusionError, Result as DataFusionResult, ScalarValue}; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; use std::{ any::Any, fmt::{Display, Formatter}, @@ -126,20 +126,20 @@ impl PartialEq for CreateNamedStruct { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct GetStructField { child: Arc, ordinal: usize, } -impl DynEq for GetStructField { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for GetStructField { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for GetStructField { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/spark-expr/src/temporal.rs b/native/spark-expr/src/temporal.rs index 9cddae350..4c5afa6b4 100644 --- a/native/spark-expr/src/temporal.rs +++ b/native/spark-expr/src/temporal.rs @@ -15,13 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::{ - any::Any, - fmt::{Debug, Display, Formatter}, - hash::Hash, - sync::Arc, -}; - use crate::utils::down_cast_any_ref; use arrow::{ compute::{date_part, DatePart}, @@ -29,32 +22,37 @@ use arrow::{ }; use arrow_schema::{DataType, Schema, TimeUnit::Microsecond}; use datafusion::logical_expr::ColumnarValue; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::{DataFusionError, ScalarValue::Utf8}; use datafusion_physical_expr::PhysicalExpr; +use std::hash::Hasher; +use std::{ + any::Any, + fmt::{Debug, Display, Formatter}, + sync::Arc, +}; use crate::utils::array_with_timezone; use crate::kernels::temporal::{ date_trunc_array_fmt_dyn, date_trunc_dyn, timestamp_trunc_array_fmt_dyn, timestamp_trunc_dyn, }; -use crate::NormalizeNaNAndZero; -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct HourExpr { /// An array with DataType::Timestamp(TimeUnit::Microsecond, None) child: Arc, timezone: String, } -impl DynEq for HourExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for HourExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for HourExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -139,21 +137,21 @@ impl PhysicalExpr for HourExpr { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct MinuteExpr { /// An array with DataType::Timestamp(TimeUnit::Microsecond, None) child: Arc, timezone: String, } -impl DynEq for MinuteExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for MinuteExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for MinuteExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -238,21 +236,21 @@ impl PhysicalExpr for MinuteExpr { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct SecondExpr { /// An array with DataType::Timestamp(TimeUnit::Microsecond, None) child: Arc, timezone: String, } -impl DynEq for SecondExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for SecondExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for SecondExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -337,7 +335,7 @@ impl PhysicalExpr for SecondExpr { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct DateTruncExpr { /// An array with DataType::Date32 child: Arc, @@ -345,14 +343,14 @@ pub struct DateTruncExpr { format: Arc, } -impl DynEq for DateTruncExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for DateTruncExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for DateTruncExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } @@ -429,7 +427,7 @@ impl PhysicalExpr for DateTruncExpr { } } -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct TimestampTruncExpr { /// An array with DataType::Timestamp(TimeUnit::Microsecond, None) child: Arc, @@ -444,14 +442,14 @@ pub struct TimestampTruncExpr { timezone: String, } -impl DynEq for TimestampTruncExpr { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for TimestampTruncExpr { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for TimestampTruncExpr { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } } diff --git a/native/spark-expr/src/to_json.rs b/native/spark-expr/src/to_json.rs index 1dca149c8..9c636763a 100644 --- a/native/spark-expr/src/to_json.rs +++ b/native/spark-expr/src/to_json.rs @@ -24,17 +24,17 @@ use crate::{spark_cast, EvalMode}; use arrow_array::builder::StringBuilder; use arrow_array::{Array, ArrayRef, RecordBatch, StringArray, StructArray}; use arrow_schema::{DataType, Schema}; -use datafusion::physical_expr_common::physical_expr::DynEq; +use datafusion::physical_expr_common::physical_expr::DynHash; use datafusion_common::Result; use datafusion_expr::ColumnarValue; use datafusion_physical_expr::PhysicalExpr; use std::any::Any; use std::fmt::{Debug, Display, Formatter}; -use std::hash::{Hash, Hasher}; +use std::hash::Hasher; use std::sync::Arc; /// to_json function -#[derive(Debug, Hash)] +#[derive(Debug, Eq)] pub struct ToJson { /// The input to convert to JSON expr: Arc, @@ -42,14 +42,14 @@ pub struct ToJson { timezone: String, } -impl DynEq for ToJson { - fn dyn_eq(&self, other: &dyn Any) -> bool { +impl DynHash for ToJson { + fn dyn_hash(&self, _state: &mut dyn Hasher) { todo!() } } impl PartialEq for ToJson { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, _other: &Self) -> bool { todo!() } }