Bump Calcite to preserve LATERALs in LATERAL UNNEST calls (#542)

* extract union on lateral outer join unit test * bump calcite * fix indent * bump calcite to latest * remove print * remove unused
linkedin · Nov 14, 2024 · 15fc504 · 15fc504
1 parent c51d3b5
commit 15fc504
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java b/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java
@@ -261,6 +261,17 @@ public void testLateralViewOuter() {
     assertEquals(convertToSparkSql, targetSql);
   }
 
+  @Test
+  public void testLateralViewOuterWithExtractUnion() {
+    RelNode relNode = TestUtils.toRelNode(String.join("\n", "", "SELECT extract_union(ut), t.ccol", "FROM complex",
+        "LATERAL VIEW OUTER explode(complex.c) t as ccol"));
+    String convertToSparkSql = createCoralSpark(relNode).getSparkSql();
+
+    String targetSql = "SELECT coalesce_struct(complex.ut, 'uniontype<int>'), t0.ccol\n"
+        + "FROM default.complex complex LATERAL VIEW OUTER EXPLODE(complex.c) t0 AS ccol";
+    assertEquals(convertToSparkSql, targetSql);
+  }
+
   @Test
   public void testMultipleLateralView() {
     RelNode relNode = TestUtils.toRelNode(String.join("\n", "", "SELECT a, t.ccol, t2.ccol2", "FROM complex ",

diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java b/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java
@@ -60,7 +60,7 @@ public static void initializeViews(HiveConf conf) throws HiveException, MetaExce
     run(driver, "CREATE TABLE IF NOT EXISTS baz(`timestamp` int, `select` double)");
     run(driver, "CREATE VIEW IF NOT EXISTS baz_view as select `select`, `timestamp` from baz");
     run(driver,
-        "CREATE TABLE IF NOT EXISTS complex(a int, b string, c array<double>, s struct<name:string, age:int>, m map<string, int>, sarr array<struct<name:string, age:int>>)");
+        "CREATE TABLE IF NOT EXISTS complex(a int, b string, c array<double>, s struct<name:string, age:int>, m map<string, int>, sarr array<struct<name:string, age:int>>, ut uniontype<int>)");
 
     String baseComplexSchema = loadSchema("base-complex.avsc");
     executeCreateTableQuery(driver, "default", "basecomplex", baseComplexSchema);

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
@@ -9,7 +9,7 @@ def versions = [
   'jetbrains': '16.0.2',
   'jline': '0.9.94',
   'kryo': '2.22',
-  'linkedin-calcite-core': '1.21.0.262',
+  'linkedin-calcite-core': '1.21.0.265',
   'pig': '0.15.0',
   'spark': '2.4.0',
   'spark3': '3.1.1',