uncomment tests (#8881)

apache · Jan 17, 2024 · 9004eb4 · 9004eb4
1 parent 31094b0
commit 9004eb4
Show file tree

Hide file tree

Showing 3 changed files with 108 additions and 94 deletions.
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
@@ -2957,6 +2957,20 @@ impl TryFrom<&DataType> for ScalarValue {
                 .to_owned()
                 .into(),
             ),
+            // 'ScalarValue::LargeList' contains single element `LargeListArray
+            DataType::LargeList(field) => ScalarValue::LargeList(
+                new_null_array(
+                    &DataType::LargeList(Arc::new(Field::new(
+                        "item",
+                        field.data_type().clone(),
+                        true,
+                    ))),
+                    1,
+                )
+                .as_list::<i64>()
+                .to_owned()
+                .into(),
+            ),
             DataType::Struct(fields) => ScalarValue::Struct(None, fields.clone()),
             DataType::Null => ScalarValue::Null,
             _ => {

diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs
@@ -1074,7 +1074,9 @@ pub fn array_prepend(args: &[ArrayRef]) -> Result<ArrayRef> {
     }
 }
 
-fn align_array_dimensions(args: Vec<ArrayRef>) -> Result<Vec<ArrayRef>> {
+fn align_array_dimensions<O: OffsetSizeTrait>(
+    args: Vec<ArrayRef>,
+) -> Result<Vec<ArrayRef>> {
     let args_ndim = args
         .iter()
         .map(|arg| datafusion_common::utils::list_ndims(arg.data_type()))
@@ -1091,9 +1093,9 @@ fn align_array_dimensions(args: Vec<ArrayRef>) -> Result<Vec<ArrayRef>> {
                 for _ in 0..(max_ndim - ndim) {
                     let data_type = aligned_array.data_type().to_owned();
                     let array_lengths = vec![1; aligned_array.len()];
-                    let offsets = OffsetBuffer::<i32>::from_lengths(array_lengths);
+                    let offsets = OffsetBuffer::<O>::from_lengths(array_lengths);
 
-                    aligned_array = Arc::new(ListArray::try_new(
+                    aligned_array = Arc::new(GenericListArray::<O>::try_new(
                         Arc::new(Field::new("item", data_type, true)),
                         offsets,
                         aligned_array,
@@ -1112,13 +1114,12 @@ fn align_array_dimensions(args: Vec<ArrayRef>) -> Result<Vec<ArrayRef>> {
 
 // Concatenate arrays on the same row.
 fn concat_internal<O: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
-    let args = align_array_dimensions(args.to_vec())?;
+    let args = align_array_dimensions::<O>(args.to_vec())?;
 
     let list_arrays = args
         .iter()
         .map(|arg| as_generic_list_array::<O>(arg))
         .collect::<Result<Vec<_>>>()?;
-
     // Assume number of rows is the same for all arrays
     let row_count = list_arrays[0].len();
 
@@ -2733,9 +2734,11 @@ mod tests {
         let array2d_1 = Arc::new(array_into_list_array(array1d_1.clone())) as ArrayRef;
         let array2d_2 = Arc::new(array_into_list_array(array1d_2.clone())) as ArrayRef;
 
-        let res =
-            align_array_dimensions(vec![array1d_1.to_owned(), array2d_2.to_owned()])
-                .unwrap();
+        let res = align_array_dimensions::<i32>(vec![
+            array1d_1.to_owned(),
+            array2d_2.to_owned(),
+        ])
+        .unwrap();
 
         let expected = as_list_array(&array2d_1).unwrap();
         let expected_dim = datafusion_common::utils::list_ndims(array2d_1.data_type());
@@ -2748,7 +2751,8 @@ mod tests {
         let array3d_1 = Arc::new(array_into_list_array(array2d_1)) as ArrayRef;
         let array3d_2 = array_into_list_array(array2d_2.to_owned());
         let res =
-            align_array_dimensions(vec![array1d_1, Arc::new(array3d_2.clone())]).unwrap();
+            align_array_dimensions::<i32>(vec![array1d_1, Arc::new(array3d_2.clone())])
+                .unwrap();
 
         let expected = as_list_array(&array3d_1).unwrap();
         let expected_dim = datafusion_common::utils::list_ndims(array3d_1.data_type());

diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt
@@ -107,18 +107,17 @@ AS VALUES
   (make_array(make_array(4, 5, 6), make_array(10, 11, 12), make_array(4, 9, 8), make_array(7, 8, 9), make_array(10, 11, 12), make_array(1, 8, 7)), make_array(10, 11, 12), 3, make_array([[11, 12, 13], [14, 15, 16]], [[17, 18, 19], [20, 21, 22]]), make_array(121, 131, 141))
 ;
 
-# TODO: add this when #8305 is fixed
-# statement ok
-# CREATE TABLE large_nested_arrays
-# AS
-#   SELECT
-#     arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1,
-#     arrow_cast(column2, 'LargeList(Int64)') AS column2,
-#     column3,
-#     arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4,
-#     arrow_cast(column5, 'LargeList(Int64)') AS column5
-#   FROM nested_arrays
-# ;
+statement ok
+CREATE TABLE large_nested_arrays
+AS
+  SELECT
+    arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1,
+    arrow_cast(column2, 'LargeList(Int64)') AS column2,
+    column3,
+    arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4,
+    arrow_cast(column5, 'LargeList(Int64)') AS column5
+  FROM nested_arrays
+;
 
 statement ok
 CREATE TABLE arrays_values
@@ -155,16 +154,15 @@ AS VALUES
   (NULL, NULL, NULL, NULL)
 ;
 
-# TODO: add this when #8305 is fixed
-# statement ok
-# CREATE TABLE large_arrays_values_v2
-# AS SELECT
-#   arrow_cast(column1, 'LargeList(Int64)') AS column1,
-#   arrow_cast(column2, 'LargeList(Int64)') AS column2,
-#   column3,
-#   arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4
-# FROM arrays_values_v2
-# ;
+statement ok
+CREATE TABLE large_arrays_values_v2
+AS SELECT
+  arrow_cast(column1, 'LargeList(Int64)') AS column1,
+  arrow_cast(column2, 'LargeList(Int64)') AS column2,
+  column3,
+  arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4
+FROM arrays_values_v2
+;
 
 statement ok
 CREATE TABLE flatten_table
@@ -1576,16 +1574,15 @@ select
 ----
 [4] [] [1, , 3, 4] [, , 1]
 
-# TODO: add this when #8305 is fixed
-# query ????
-# select
-#   array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4),
-#   array_append(make_array(), null),
-#   array_append(make_array(1, null, 3), 4),
-#   array_append(make_array(null, null), 1)
-# ;
-# ----
-# [4] [] [1, , 3, 4] [, , 1]
+query ????
+select
+  array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4),
+  array_append(make_array(), null),
+  array_append(make_array(1, null, 3), 4),
+  array_append(make_array(null, null), 1)
+;
+----
+[4] [] [1, , 3, 4] [, , 1]
 
 # test invalid (non-null)
 query error
@@ -1604,13 +1601,12 @@ select
 ----
 [[1, , 3], []] [[1, , 3], ]
 
-# TODO: add this when #8305 is fixed
-# query ??
-# select
-#   array_append(arrow_cast(make_array(make_array(1, null, 3), 'LargeList(LargeList(Int64))')), arrow_cast(make_array(null), 'LargeList(Int64)')),
-#   array_append(arrow_cast(make_array(make_array(1, null, 3), 'LargeList(LargeList(Int64))')), null);
-# ----
-# [[1, , 3], []] [[1, , 3], ]
+query ??
+select
+  array_append(arrow_cast(make_array(make_array(1, null, 3)), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(null), 'LargeList(Int64)')),
+  array_append(arrow_cast(make_array(make_array(1, null, 3)), 'LargeList(LargeList(Int64))'), null);
+----
+[[1, , 3], []] [[1, , 3], ]
 
 # array_append scalar function #3
 query ???
@@ -1629,11 +1625,10 @@ select array_append(make_array([1], [2], [3]), make_array(4)), array_append(make
 ----
 [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
 
-# TODO: add this when #8305 is fixed
-# query ???
-# select array_append(arrow_cast(make_array([1], [2], [3]), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')), array_append(arrow_cast(make_array([1.0], [2.0], [3.0]), 'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0), 'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'], ['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'), 'LargeList(Utf8)'));
-# ----
-# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
+query ???
+select array_append(arrow_cast(make_array([1], [2], [3]), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')), array_append(arrow_cast(make_array([1.0], [2.0], [3.0]), 'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0), 'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'], ['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'), 'LargeList(Utf8)'));
+----
+[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
 
 # list_append scalar function #5 (function alias `array_append`)
 query ???
@@ -1700,12 +1695,11 @@ select array_append(column1, column2) from nested_arrays;
 [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
 [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]]
 
-# TODO: add this when #8305 is fixed
-# query ?
-# select array_append(column1, column2) from large_nested_arrays;
-# ----
-# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
-# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]]
+query ?
+select array_append(column1, column2) from large_nested_arrays;
+----
+[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
+[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]]
 
 # array_append with columns and scalars #1
 query ??
@@ -1737,12 +1731,11 @@ select array_append(column1, make_array(1, 11, 111)), array_append(make_array(ma
 [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
 [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
 
-# TODO: add this when #8305 is fixed
-# query ??
-# select array_append(column1, arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from large_nested_arrays;
-# ----
-# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
-# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
+query ??
+select array_append(column1, arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from large_nested_arrays;
+----
+[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
+[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
 
 ## array_prepend (aliases: `list_prepend`, `array_push_front`, `list_push_front`)
 
@@ -1816,11 +1809,12 @@ select array_prepend(make_array(1), make_array(make_array(2), make_array(3), mak
 ----
 [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
 
-# TODO: add this when #8305 is fixed
-# query ???
-# select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'), arrow_cast(make_array(make_array(2), make_array(3), make_array(4)), 'LargeList(LargeList(Int64))')), array_prepend(arrow_cast(make_array(1.0), 'LargeList(Float64)'), arrow_cast(make_array([2.0], [3.0], [4.0]), 'LargeList(LargeList(Float64))')), array_prepend(arrow_cast(make_array('h'), 'LargeList(Utf8)'), arrow_cast(make_array(['e'], ['l'], ['l'], ['o']), 'LargeList(LargeList(Utf8))''));
-# ----
-# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
+query ???
+select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'), arrow_cast(make_array(make_array(2), make_array(3), make_array(4)), 'LargeList(LargeList(Int64))')),
+       array_prepend(arrow_cast(make_array(1.0), 'LargeList(Float64)'), arrow_cast(make_array([2.0], [3.0], [4.0]), 'LargeList(LargeList(Float64))')),
+       array_prepend(arrow_cast(make_array('h'), 'LargeList(Utf8)'), arrow_cast(make_array(['e'], ['l'], ['l'], ['o']), 'LargeList(LargeList(Utf8))'));
+----
+[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
 
 # list_prepend scalar function #5 (function alias `array_prepend`)
 query ???
@@ -1887,12 +1881,11 @@ select array_prepend(column2, column1) from nested_arrays;
 [[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
 [[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]]
 
-# TODO: add this when #8305 is fixed
-# query ?
-# select array_prepend(column2, column1) from large_nested_arrays;
-# ----
-# [[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
-# [[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]]
+query ?
+select array_prepend(column2, column1) from large_nested_arrays;
+----
+[[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
+[[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]]
 
 # array_prepend with columns and scalars #1
 query ??
@@ -1924,12 +1917,11 @@ select array_prepend(make_array(1, 11, 111), column1), array_prepend(column2, ma
 [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
 [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
 
-# TODO: add this when #8305 is fixed
-# query ??
-# select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'), column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from large_nested_arrays;
-# ----
-# [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
-# [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
+query ??
+select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'), column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from large_nested_arrays;
+----
+[[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
+[[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
 
 ## array_repeat (aliases: `list_repeat`)
 
@@ -2372,12 +2364,11 @@ select array_position(column1, column2), array_position(column1, column2, column
 3 3
 2 5
 
-#TODO: add this test when #8305 is fixed
-#query II
-#select array_position(column1, column2), array_position(column1, column2, column3) from nested_arrays;
-#----
-#3 3
-#2 5
+query II
+select array_position(column1, column2), array_position(column1, column2, column3) from nested_arrays;
+----
+3 3
+2 5
 
 # array_position with columns and scalars #1
 query III
@@ -2403,12 +2394,11 @@ select array_position(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), column2),
 NULL 6 4
 NULL 1 NULL
 
-#TODO: add this test when #8305 is fixed
-#query III
-#select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), 'LargeList(List(Int64))'), column2), array_position(column1, make_array(4, 5, 6)), array_position(column1, make_array(1, 2, 3), 2) from large_nested_arrays;
-#----
-#NULL 6 4
-#NULL 1 NULL
+query III
+select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), 'LargeList(LargeList(Int64))'), column2), array_position(column1, arrow_cast(make_array(4, 5, 6), 'LargeList(Int64)')), array_position(column1, arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 2) from large_nested_arrays;
+----
+NULL 6 4
+NULL 1 NULL
 
 ## array_positions (aliases: `list_positions`)
 
@@ -4989,6 +4979,9 @@ drop table values_without_nulls;
 statement ok
 drop table nested_arrays;
 
+statement ok
+drop table large_nested_arrays;
+
 statement ok
 drop table arrays;
 
@@ -5007,6 +5000,9 @@ drop table arrays_values;
 statement ok
 drop table arrays_values_v2;
 
+statement ok
+drop table large_arrays_values_v2;
+
 statement ok
 drop table array_has_table_1D;