From 14d2398a63477f86eba77215738875d9a8bb95fa Mon Sep 17 00:00:00 2001 From: my-vegetable-has-exploded Date: Tue, 12 Dec 2023 18:47:18 +0800 Subject: [PATCH 1/5] init --- datafusion/sql/src/expr/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 27351e10eb34..6895d4b670fe 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -513,6 +513,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.parse_struct(values, fields, schema, planner_context) } + SQLExpr::Tuple(values) => { + // self. + todo!() + } + _ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"), } } @@ -583,6 +588,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ))) } + fn parse_tuple( + &self, + values: Vec, + input_schema: &DFSchema, + planner_context: &mut PlannerContext, + ) -> Result { + todo!() + } + fn sql_in_list_to_expr( &self, expr: SQLExpr, From 48207793e327623fa8f79c401091e6f5f808ef61 Mon Sep 17 00:00:00 2001 From: my-vegetable-has-exploded Date: Tue, 12 Dec 2023 21:28:18 +0800 Subject: [PATCH 2/5] Implement tuple to struct. --- datafusion/common/src/scalar.rs | 1 + .../optimizer/src/analyzer/type_coercion.rs | 1 + datafusion/sql/src/expr/mod.rs | 18 +++++++++++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index d730fbf89b72..03b4a7ed74c4 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -2490,6 +2490,7 @@ impl ScalarValue { eq_array_primitive!(array, index, DurationNanosecondArray, val)? } ScalarValue::Struct(_, _) => { + //Todo@wy return _not_impl_err!("Struct is not supported yet") } ScalarValue::Dictionary(key_type, v) => { diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index c5e1180b9f97..382fbaa23652 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -292,6 +292,7 @@ impl TreeNodeRewriter for TypeCoercionRewriter { .map(|list_expr| list_expr.get_type(&self.schema)) .collect::>>()?; let result_type = + //Todo@wy to coerce get_coerce_type_for_list(&expr_data_type, &list_data_types); match result_type { None => plan_err!( diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 6895d4b670fe..e191311171f3 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -27,7 +27,10 @@ mod substring; mod unary_op; mod value; +use std::vec; + use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; + use arrow_schema::DataType; use arrow_schema::TimeUnit; use datafusion_common::{ @@ -514,8 +517,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } SQLExpr::Tuple(values) => { - // self. - todo!() + self.parse_tuple(values, schema, planner_context) } _ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"), @@ -594,7 +596,17 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { input_schema: &DFSchema, planner_context: &mut PlannerContext, ) -> Result { - todo!() + if values.is_empty() { + return not_impl_err!("Empty tuple not supported yet"); + } + match values.get(0).unwrap() { + SQLExpr::Identifier(_) | SQLExpr::Value(_)=> { + self.parse_struct(values, vec![], input_schema, planner_context) + } + _ => { + not_impl_err!("Tuple not supported yet") + } + } } fn sql_in_list_to_expr( From 6486ef99751903afa7ddd9450323452e53359142 Mon Sep 17 00:00:00 2001 From: my-vegetable-has-exploded Date: Sun, 17 Dec 2023 21:45:31 +0800 Subject: [PATCH 3/5] add todos --- datafusion/optimizer/src/analyzer/type_coercion.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 382fbaa23652..cda310a70d19 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -300,6 +300,7 @@ impl TreeNodeRewriter for TypeCoercionRewriter { ), Some(coerced_type) => { // find the coerced type + // Todo@wy try to cast the expr to the coerced type for struct let cast_expr = expr.cast_to(&coerced_type, &self.schema)?; let cast_list_expr = list .into_iter() From 50f7aa7f2ebd560cac4fa825e8a22a19c4c3798f Mon Sep 17 00:00:00 2001 From: my-vegetable-has-exploded Date: Tue, 19 Dec 2023 23:07:07 +0800 Subject: [PATCH 4/5] add type_coercion for struct. --- datafusion/common/src/scalar.rs | 1 - datafusion/expr/src/type_coercion/binary.rs | 26 ++++++++++++++++++- .../optimizer/src/analyzer/type_coercion.rs | 2 -- datafusion/sql/src/expr/mod.rs | 20 +++++++------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index 03b4a7ed74c4..d730fbf89b72 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -2490,7 +2490,6 @@ impl ScalarValue { eq_array_primitive!(array, index, DurationNanosecondArray, val)? } ScalarValue::Struct(_, _) => { - //Todo@wy return _not_impl_err!("Struct is not supported yet") } ScalarValue::Dictionary(key_type, v) => { diff --git a/datafusion/expr/src/type_coercion/binary.rs b/datafusion/expr/src/type_coercion/binary.rs index dd9449198796..5141d569558b 100644 --- a/datafusion/expr/src/type_coercion/binary.rs +++ b/datafusion/expr/src/type_coercion/binary.rs @@ -24,7 +24,7 @@ use crate::Operator; use arrow::array::{new_empty_array, Array}; use arrow::compute::can_cast_types; use arrow::datatypes::{ - DataType, Field, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, + DataType, Field, FieldRef, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, }; @@ -298,6 +298,7 @@ pub fn comparison_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option Option } } +fn struct_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option { + use arrow::datatypes::DataType::*; + match (lhs_type, rhs_type) { + (Struct(lhs_fields), Struct(rhs_fields)) => { + let types = lhs_fields + .iter() + .map(|f| f.data_type()) + .zip(rhs_fields.iter().map(|f| f.data_type())) + .map(|(lhs, rhs)| comparison_coercion(lhs, rhs)) + .collect::>>()?; + let fields = types + .into_iter() + .enumerate() + .map(|(i, datatype)| { + Arc::new(Field::new(format!("c{}", i), datatype, true)) + }) + .collect::>(); + Some(Struct(fields.into())) + } + _ => None, + } +} + /// coercion rules for like operations. /// This is a union of string coercion rules and dictionary coercion rules pub fn like_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option { diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index cda310a70d19..c5e1180b9f97 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -292,7 +292,6 @@ impl TreeNodeRewriter for TypeCoercionRewriter { .map(|list_expr| list_expr.get_type(&self.schema)) .collect::>>()?; let result_type = - //Todo@wy to coerce get_coerce_type_for_list(&expr_data_type, &list_data_types); match result_type { None => plan_err!( @@ -300,7 +299,6 @@ impl TreeNodeRewriter for TypeCoercionRewriter { ), Some(coerced_type) => { // find the coerced type - // Todo@wy try to cast the expr to the coerced type for struct let cast_expr = expr.cast_to(&coerced_type, &self.schema)?; let cast_list_expr = list .into_iter() diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index e191311171f3..e9c7425d763d 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -516,9 +516,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.parse_struct(values, fields, schema, planner_context) } - SQLExpr::Tuple(values) => { - self.parse_tuple(values, schema, planner_context) - } + SQLExpr::Tuple(values) => self.parse_tuple(values, schema, planner_context), _ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"), } @@ -599,14 +597,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { if values.is_empty() { return not_impl_err!("Empty tuple not supported yet"); } - match values.get(0).unwrap() { - SQLExpr::Identifier(_) | SQLExpr::Value(_)=> { - self.parse_struct(values, vec![], input_schema, planner_context) - } - _ => { - not_impl_err!("Tuple not supported yet") - } - } + match values.get(0).unwrap() { + SQLExpr::Identifier(_) | SQLExpr::Value(_) => { + self.parse_struct(values, vec![], input_schema, planner_context) + } + _ => { + not_impl_err!("Tuple not supported yet") + } + } } fn sql_in_list_to_expr( From c5ac82f81bc795106a1ca8f3d67714f7c1eab6c1 Mon Sep 17 00:00:00 2001 From: my-vegetable-has-exploded Date: Wed, 20 Dec 2023 13:28:13 +0800 Subject: [PATCH 5/5] fix error. --- datafusion/sql/src/expr/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index e9c7425d763d..d83a696f73be 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -602,7 +602,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.parse_struct(values, vec![], input_schema, planner_context) } _ => { - not_impl_err!("Tuple not supported yet") + not_impl_err!("Only identifiers and literals are supported in tuples") } } }