coral-schema: supporting projection of inner field of Record type by …

…implementing visitFieldAccess (#83) Co-authored-by: Wenye Zhang <[email protected]>
linkedin · May 7, 2021 · f3437d7 · f3437d7
1 parent 04f9bdb
commit f3437d7
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 2 deletions.
diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java
@@ -52,6 +52,7 @@
 import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
 import org.apache.calcite.util.Pair;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -449,8 +450,31 @@ public RexNode visitRangeRef(RexRangeRef rexRangeRef) {
 
     @Override
     public RexNode visitFieldAccess(RexFieldAccess rexFieldAccess) {
-      // TODO: implement this method
-      return super.visitFieldAccess(rexFieldAccess);
+      if (rexFieldAccess.getReferenceExpr() instanceof RexInputRef) {
+        RexInputRef relInputRef = (RexInputRef) rexFieldAccess.getReferenceExpr();
+
+        String oldFieldName = rexFieldAccess.getField().getName();
+        String suggestNewFieldName = suggestedFieldNames.poll();
+        String newFieldName = SchemaUtilities.getFieldName(oldFieldName, suggestNewFieldName);
+
+        Schema topSchema = inputSchema.getFields().get(relInputRef.getIndex()).schema();
+        if (AvroSerdeUtils.isNullableType(topSchema)) {
+          topSchema = AvroSerdeUtils.getOtherTypeFromNullableType(topSchema);
+        }
+
+        Schema.Field accessedField = null;
+        for (Schema.Field field : topSchema.getFields()) {
+          if (field.name().equalsIgnoreCase(oldFieldName)) {
+            accessedField = field;
+            break;
+          }
+        }
+        SchemaUtilities.appendField(newFieldName, accessedField, fieldAssembler);
+
+        return rexFieldAccess;
+      } else {
+        return super.visitFieldAccess(rexFieldAccess);
+      }
     }
 
     @Override

diff --git a/...l-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/...l-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java
@@ -612,6 +612,20 @@ public void testSelectStarFromNestComplex() {
         TestUtils.loadSchema("testSelectStarFromNestComplex-expected.avsc"));
   }
 
+  @Test
+  public void testProjectStructInnerField() {
+    String viewSql = "CREATE VIEW v AS "
+        + "SELECT bc.Id AS Id_View_Col, Struct_Col.Bool_Field AS Struct_Inner_Bool_Col, Struct_Col.Int_Field AS Struct_Inner_Int_Col, Struct_Col.Bigint_Field AS Struct_Inner_Bigint_Col "
+        + "FROM basecomplex bc " + "WHERE bc.Id > 0 AND bc.Struct_Col IS NOT NULL";
+
+    TestUtils.executeCreateViewQuery("default", "v", viewSql);
+
+    ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient);
+    Schema actualSchema = viewToAvroSchemaConverter.toAvroSchema("default", "v");
+
+    Assert.assertEquals(actualSchema.toString(true), TestUtils.loadSchema("testProjectStructInnerField-expected.avsc"));
+  }
+
   @Test
   public void testSubQueryWhere() {
     // TODO: implement this test

diff --git a/coral-schema/src/test/resources/testProjectStructInnerField-expected.avsc b/coral-schema/src/test/resources/testProjectStructInnerField-expected.avsc
@@ -0,0 +1,19 @@
+{
+  "type" : "record",
+  "name" : "v",
+  "namespace" : "default.v",
+  "fields" : [ {
+    "name" : "Id_View_Col",
+    "type" : "int"
+  }, {
+    "name" : "Struct_Inner_Bool_Col",
+    "type" : "boolean"
+  }, {
+    "name" : "Struct_Inner_Int_Col",
+    "type" : [ "null", "int" ],
+    "default" : null
+  }, {
+    "name" : "Struct_Inner_Bigint_Col",
+    "type" : "long"
+  } ]
+}