From 9004eb4e4851a92e9bba60f0ca027514918ba1ae Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Thu, 18 Jan 2024 04:26:12 +0800 Subject: [PATCH] uncomment tests (#8881) --- datafusion/common/src/scalar.rs | 14 ++ .../physical-expr/src/array_expressions.rs | 22 ++- datafusion/sqllogictest/test_files/array.slt | 166 +++++++++--------- 3 files changed, 108 insertions(+), 94 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index 9cbd9e292ff3..20d03c70960a 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -2957,6 +2957,20 @@ impl TryFrom<&DataType> for ScalarValue { .to_owned() .into(), ), + // 'ScalarValue::LargeList' contains single element `LargeListArray + DataType::LargeList(field) => ScalarValue::LargeList( + new_null_array( + &DataType::LargeList(Arc::new(Field::new( + "item", + field.data_type().clone(), + true, + ))), + 1, + ) + .as_list::() + .to_owned() + .into(), + ), DataType::Struct(fields) => ScalarValue::Struct(None, fields.clone()), DataType::Null => ScalarValue::Null, _ => { diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 5b35c4b9d8fb..af6587631df5 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -1074,7 +1074,9 @@ pub fn array_prepend(args: &[ArrayRef]) -> Result { } } -fn align_array_dimensions(args: Vec) -> Result> { +fn align_array_dimensions( + args: Vec, +) -> Result> { let args_ndim = args .iter() .map(|arg| datafusion_common::utils::list_ndims(arg.data_type())) @@ -1091,9 +1093,9 @@ fn align_array_dimensions(args: Vec) -> Result> { for _ in 0..(max_ndim - ndim) { let data_type = aligned_array.data_type().to_owned(); let array_lengths = vec![1; aligned_array.len()]; - let offsets = OffsetBuffer::::from_lengths(array_lengths); + let offsets = OffsetBuffer::::from_lengths(array_lengths); - aligned_array = Arc::new(ListArray::try_new( + aligned_array = Arc::new(GenericListArray::::try_new( Arc::new(Field::new("item", data_type, true)), offsets, aligned_array, @@ -1112,13 +1114,12 @@ fn align_array_dimensions(args: Vec) -> Result> { // Concatenate arrays on the same row. fn concat_internal(args: &[ArrayRef]) -> Result { - let args = align_array_dimensions(args.to_vec())?; + let args = align_array_dimensions::(args.to_vec())?; let list_arrays = args .iter() .map(|arg| as_generic_list_array::(arg)) .collect::>>()?; - // Assume number of rows is the same for all arrays let row_count = list_arrays[0].len(); @@ -2733,9 +2734,11 @@ mod tests { let array2d_1 = Arc::new(array_into_list_array(array1d_1.clone())) as ArrayRef; let array2d_2 = Arc::new(array_into_list_array(array1d_2.clone())) as ArrayRef; - let res = - align_array_dimensions(vec![array1d_1.to_owned(), array2d_2.to_owned()]) - .unwrap(); + let res = align_array_dimensions::(vec![ + array1d_1.to_owned(), + array2d_2.to_owned(), + ]) + .unwrap(); let expected = as_list_array(&array2d_1).unwrap(); let expected_dim = datafusion_common::utils::list_ndims(array2d_1.data_type()); @@ -2748,7 +2751,8 @@ mod tests { let array3d_1 = Arc::new(array_into_list_array(array2d_1)) as ArrayRef; let array3d_2 = array_into_list_array(array2d_2.to_owned()); let res = - align_array_dimensions(vec![array1d_1, Arc::new(array3d_2.clone())]).unwrap(); + align_array_dimensions::(vec![array1d_1, Arc::new(array3d_2.clone())]) + .unwrap(); let expected = as_list_array(&array3d_1).unwrap(); let expected_dim = datafusion_common::utils::list_ndims(array3d_1.data_type()); diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 6b45f204fefc..342fcb5bec3f 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -107,18 +107,17 @@ AS VALUES (make_array(make_array(4, 5, 6), make_array(10, 11, 12), make_array(4, 9, 8), make_array(7, 8, 9), make_array(10, 11, 12), make_array(1, 8, 7)), make_array(10, 11, 12), 3, make_array([[11, 12, 13], [14, 15, 16]], [[17, 18, 19], [20, 21, 22]]), make_array(121, 131, 141)) ; -# TODO: add this when #8305 is fixed -# statement ok -# CREATE TABLE large_nested_arrays -# AS -# SELECT -# arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1, -# arrow_cast(column2, 'LargeList(Int64)') AS column2, -# column3, -# arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4, -# arrow_cast(column5, 'LargeList(Int64)') AS column5 -# FROM nested_arrays -# ; +statement ok +CREATE TABLE large_nested_arrays +AS + SELECT + arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1, + arrow_cast(column2, 'LargeList(Int64)') AS column2, + column3, + arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4, + arrow_cast(column5, 'LargeList(Int64)') AS column5 + FROM nested_arrays +; statement ok CREATE TABLE arrays_values @@ -155,16 +154,15 @@ AS VALUES (NULL, NULL, NULL, NULL) ; -# TODO: add this when #8305 is fixed -# statement ok -# CREATE TABLE large_arrays_values_v2 -# AS SELECT -# arrow_cast(column1, 'LargeList(Int64)') AS column1, -# arrow_cast(column2, 'LargeList(Int64)') AS column2, -# column3, -# arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4 -# FROM arrays_values_v2 -# ; +statement ok +CREATE TABLE large_arrays_values_v2 +AS SELECT + arrow_cast(column1, 'LargeList(Int64)') AS column1, + arrow_cast(column2, 'LargeList(Int64)') AS column2, + column3, + arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4 +FROM arrays_values_v2 +; statement ok CREATE TABLE flatten_table @@ -1576,16 +1574,15 @@ select ---- [4] [] [1, , 3, 4] [, , 1] -# TODO: add this when #8305 is fixed -# query ???? -# select -# array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4), -# array_append(make_array(), null), -# array_append(make_array(1, null, 3), 4), -# array_append(make_array(null, null), 1) -# ; -# ---- -# [4] [] [1, , 3, 4] [, , 1] +query ???? +select + array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4), + array_append(make_array(), null), + array_append(make_array(1, null, 3), 4), + array_append(make_array(null, null), 1) +; +---- +[4] [] [1, , 3, 4] [, , 1] # test invalid (non-null) query error @@ -1604,13 +1601,12 @@ select ---- [[1, , 3], []] [[1, , 3], ] -# TODO: add this when #8305 is fixed -# query ?? -# select -# array_append(arrow_cast(make_array(make_array(1, null, 3), 'LargeList(LargeList(Int64))')), arrow_cast(make_array(null), 'LargeList(Int64)')), -# array_append(arrow_cast(make_array(make_array(1, null, 3), 'LargeList(LargeList(Int64))')), null); -# ---- -# [[1, , 3], []] [[1, , 3], ] +query ?? +select + array_append(arrow_cast(make_array(make_array(1, null, 3)), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(null), 'LargeList(Int64)')), + array_append(arrow_cast(make_array(make_array(1, null, 3)), 'LargeList(LargeList(Int64))'), null); +---- +[[1, , 3], []] [[1, , 3], ] # array_append scalar function #3 query ??? @@ -1629,11 +1625,10 @@ select array_append(make_array([1], [2], [3]), make_array(4)), array_append(make ---- [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] -# TODO: add this when #8305 is fixed -# query ??? -# select array_append(arrow_cast(make_array([1], [2], [3]), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')), array_append(arrow_cast(make_array([1.0], [2.0], [3.0]), 'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0), 'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'], ['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'), 'LargeList(Utf8)')); -# ---- -# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] +query ??? +select array_append(arrow_cast(make_array([1], [2], [3]), 'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')), array_append(arrow_cast(make_array([1.0], [2.0], [3.0]), 'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0), 'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'], ['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'), 'LargeList(Utf8)')); +---- +[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] # list_append scalar function #5 (function alias `array_append`) query ??? @@ -1700,12 +1695,11 @@ select array_append(column1, column2) from nested_arrays; [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]] [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]] -# TODO: add this when #8305 is fixed -# query ? -# select array_append(column1, column2) from large_nested_arrays; -# ---- -# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]] -# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]] +query ? +select array_append(column1, column2) from large_nested_arrays; +---- +[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]] +[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10, 11, 12]] # array_append with columns and scalars #1 query ?? @@ -1737,12 +1731,11 @@ select array_append(column1, make_array(1, 11, 111)), array_append(make_array(ma [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]] [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]] -# TODO: add this when #8305 is fixed -# query ?? -# select array_append(column1, arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from large_nested_arrays; -# ---- -# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]] -# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]] +query ?? +select array_append(column1, arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from large_nested_arrays; +---- +[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]] +[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]] ## array_prepend (aliases: `list_prepend`, `array_push_front`, `list_push_front`) @@ -1816,11 +1809,12 @@ select array_prepend(make_array(1), make_array(make_array(2), make_array(3), mak ---- [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] -# TODO: add this when #8305 is fixed -# query ??? -# select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'), arrow_cast(make_array(make_array(2), make_array(3), make_array(4)), 'LargeList(LargeList(Int64))')), array_prepend(arrow_cast(make_array(1.0), 'LargeList(Float64)'), arrow_cast(make_array([2.0], [3.0], [4.0]), 'LargeList(LargeList(Float64))')), array_prepend(arrow_cast(make_array('h'), 'LargeList(Utf8)'), arrow_cast(make_array(['e'], ['l'], ['l'], ['o']), 'LargeList(LargeList(Utf8))'')); -# ---- -# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] +query ??? +select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'), arrow_cast(make_array(make_array(2), make_array(3), make_array(4)), 'LargeList(LargeList(Int64))')), + array_prepend(arrow_cast(make_array(1.0), 'LargeList(Float64)'), arrow_cast(make_array([2.0], [3.0], [4.0]), 'LargeList(LargeList(Float64))')), + array_prepend(arrow_cast(make_array('h'), 'LargeList(Utf8)'), arrow_cast(make_array(['e'], ['l'], ['l'], ['o']), 'LargeList(LargeList(Utf8))')); +---- +[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]] # list_prepend scalar function #5 (function alias `array_prepend`) query ??? @@ -1887,12 +1881,11 @@ select array_prepend(column2, column1) from nested_arrays; [[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] -# TODO: add this when #8305 is fixed -# query ? -# select array_prepend(column2, column1) from large_nested_arrays; -# ---- -# [[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] -# [[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] +query ? +select array_prepend(column2, column1) from large_nested_arrays; +---- +[[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] +[[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] # array_prepend with columns and scalars #1 query ?? @@ -1924,12 +1917,11 @@ select array_prepend(make_array(1, 11, 111), column1), array_prepend(column2, ma [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]] [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]] -# TODO: add this when #8305 is fixed -# query ?? -# select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'), column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from large_nested_arrays; -# ---- -# [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]] -# [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]] +query ?? +select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'), column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3), make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from large_nested_arrays; +---- +[[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]] +[[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]] ## array_repeat (aliases: `list_repeat`) @@ -2372,12 +2364,11 @@ select array_position(column1, column2), array_position(column1, column2, column 3 3 2 5 -#TODO: add this test when #8305 is fixed -#query II -#select array_position(column1, column2), array_position(column1, column2, column3) from nested_arrays; -#---- -#3 3 -#2 5 +query II +select array_position(column1, column2), array_position(column1, column2, column3) from nested_arrays; +---- +3 3 +2 5 # array_position with columns and scalars #1 query III @@ -2403,12 +2394,11 @@ select array_position(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), column2), NULL 6 4 NULL 1 NULL -#TODO: add this test when #8305 is fixed -#query III -#select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), 'LargeList(List(Int64))'), column2), array_position(column1, make_array(4, 5, 6)), array_position(column1, make_array(1, 2, 3), 2) from large_nested_arrays; -#---- -#NULL 6 4 -#NULL 1 NULL +query III +select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12, 13]), 'LargeList(LargeList(Int64))'), column2), array_position(column1, arrow_cast(make_array(4, 5, 6), 'LargeList(Int64)')), array_position(column1, arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 2) from large_nested_arrays; +---- +NULL 6 4 +NULL 1 NULL ## array_positions (aliases: `list_positions`) @@ -4989,6 +4979,9 @@ drop table values_without_nulls; statement ok drop table nested_arrays; +statement ok +drop table large_nested_arrays; + statement ok drop table arrays; @@ -5007,6 +5000,9 @@ drop table arrays_values; statement ok drop table arrays_values_v2; +statement ok +drop table large_arrays_values_v2; + statement ok drop table array_has_table_1D;