Change NULL array, struct to Empty array, struct

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkArrayLoaderFactory.java

@@ -18,7 +18,7 @@
 import jp.co.yahoo.yosegi.inmemory.ILoader;
 import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkArrayLoader;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyArrayLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkRunLengthEncodingArrayLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionArrayLoader;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -37,7 +37,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
       throws IOException {
     if (columnBinary == null) {
       // FIXME:
-      return new SparkNullLoader(vector, loadSize);
+      return new SparkEmptyArrayLoader(vector, loadSize);
     }
     switch (getLoadType(columnBinary, loadSize)) {
       case ARRAY:
@@ -47,8 +47,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
       case UNION:
         return new SparkUnionArrayLoader(vector, loadSize);
       default:
-        // FIXME:
-        return new SparkNullLoader(vector, loadSize);
+        return new SparkEmptyArrayLoader(vector, loadSize);
     }
   }
 }

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkMapLoaderFactory.java

@@ -17,8 +17,8 @@
 import jp.co.yahoo.yosegi.binary.ColumnBinary;
 import jp.co.yahoo.yosegi.inmemory.ILoader;
 import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyMapLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkMapLoader;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionMapLoader;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
@@ -36,7 +36,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
       throws IOException {
     if (columnBinary == null) {
       // FIXME:
-      return new SparkNullLoader(vector, loadSize);
+      return new SparkEmptyMapLoader(vector, loadSize);
     }
     switch (getLoadType(columnBinary, loadSize)) {
       case SPREAD:
@@ -45,7 +45,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
         return new SparkUnionMapLoader(vector, loadSize);
       default:
         // FIXME:
-        return new SparkNullLoader(vector, loadSize);
+        return new SparkEmptyMapLoader(vector, loadSize);
     }
   }
 }

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkStructLoaderFactory.java

@@ -17,7 +17,7 @@
 import jp.co.yahoo.yosegi.binary.ColumnBinary;
 import jp.co.yahoo.yosegi.inmemory.ILoader;
 import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyStructLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkStructLoader;
 import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionStructLoader;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -36,7 +36,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
       throws IOException {
     if (columnBinary == null) {
       // FIXME:
-      return new SparkNullLoader(vector, loadSize);
+      return new SparkEmptyStructLoader(vector, loadSize);
     }
     switch (getLoadType(columnBinary, loadSize)) {
       case SPREAD:
@@ -45,7 +45,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
         return new SparkUnionStructLoader(vector, loadSize);
       default:
         // FIXME:
-        return new SparkNullLoader(vector, loadSize);
+        return new SparkEmptyStructLoader(vector, loadSize);
     }
   }
 }

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkArrayLoader.java

@@ -38,7 +38,7 @@ public int getLoadSize() {
 
   @Override
   public void setNull(final int index) throws IOException {
-    vector.putNull(index);
+    vector.putArray(index, 0, 0);
   }
 
   @Override

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyArrayLoader.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+
+import java.io.IOException;
+
+public class SparkEmptyArrayLoader implements ILoader<WritableColumnVector> {
+
+  private final WritableColumnVector vector;
+  private final int loadSize;
+
+  public SparkEmptyArrayLoader(final WritableColumnVector vector, final int loadSize) {
+    this.vector = vector;
+    this.loadSize = loadSize;
+    this.vector.getChild(0).reset();
+    this.vector.getChild(0).reserve(0);
+    if (this.vector.getChild(0).hasDictionary()) {
+      this.vector.getChild(0).reserveDictionaryIds(0);
+      this.vector.getChild(0).setDictionary(null);
+    }
+  }
+
+  @Override
+  public LoadType getLoaderType() {
+    return LoadType.NULL;
+  }
+
+  @Override
+  public int getLoadSize() {
+    return loadSize;
+  }
+
+  @Override
+  public void setNull(final int index) throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public void finish() throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public WritableColumnVector build() throws IOException {
+    for (int i = 0; i < loadSize; i++) {
+      vector.putArray(i, 0, 0);
+    }
+    return vector;
+  }
+
+  @Override
+  public boolean isLoadingSkipped() {
+    return true;
+  }
+}

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyLoader.java

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.MapType;
+import org.apache.spark.sql.types.StructType;
+
+import java.io.IOException;
+
+public class SparkEmptyLoader {
+    public static void load(final WritableColumnVector vector, final int loadSize) throws IOException {
+        final Class klass = vector.dataType().getClass();
+        if (klass == ArrayType.class) {
+            new SparkEmptyArrayLoader(vector, loadSize).build();
+        } else if (klass == StructType.class) {
+            new SparkEmptyStructLoader(vector, loadSize).build();
+        } else if (klass == MapType.class) {
+            new SparkEmptyMapLoader(vector, loadSize).build();
+        } else {
+            new SparkNullLoader(vector, loadSize).build();
+        }
+    }
+}

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyMapLoader.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+
+import java.io.IOException;
+
+public class SparkEmptyMapLoader implements ILoader<WritableColumnVector> {
+
+  private final WritableColumnVector vector;
+  private final int loadSize;
+
+  public SparkEmptyMapLoader(final WritableColumnVector vector, final int loadSize) {
+    this.vector = vector;
+    this.loadSize = loadSize;
+  }
+
+  @Override
+  public LoadType getLoaderType() {
+    return LoadType.NULL;
+  }
+
+  @Override
+  public int getLoadSize() {
+    return loadSize;
+  }
+
+  @Override
+  public void setNull(final int index) throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public void finish() throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public WritableColumnVector build() throws IOException {
+    vector.getChild(0).reset();
+    vector.getChild(0).reserve(0);
+    vector.getChild(1).reset();
+    vector.getChild(1).reserve(0);
+    return vector;
+  }
+
+  @Override
+  public boolean isLoadingSkipped() {
+    return true;
+  }
+}

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyStructLoader.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.StructType;
+
+import java.io.IOException;
+
+public class SparkEmptyStructLoader implements ILoader<WritableColumnVector> {
+
+  private final WritableColumnVector vector;
+  private final int loadSize;
+  private final String[] names;
+
+  public SparkEmptyStructLoader(final WritableColumnVector vector, final int loadSize) {
+    this.vector = vector;
+    this.loadSize = loadSize;
+    final StructType structType = (StructType) vector.dataType();
+    this.names = structType.fieldNames();
+    for (int i = 0; i < names.length; i++) {
+      vector.getChild(i).reset();
+      vector.getChild(i).reserve(loadSize);
+      if (vector.getChild(i).hasDictionary()) {
+        vector.getChild(i).reserveDictionaryIds(0);
+        vector.getChild(i).setDictionary(null);
+      }
+    }
+  }
+
+  @Override
+  public LoadType getLoaderType() {
+    return LoadType.NULL;
+  }
+
+  @Override
+  public int getLoadSize() {
+    return loadSize;
+  }
+
+  @Override
+  public void setNull(final int index) throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public void finish() throws IOException {
+    // FIXME:
+  }
+
+  @Override
+  public WritableColumnVector build() throws IOException {
+    for (int i = 0; i < names.length; i++) {
+      SparkEmptyLoader.load(vector.getChild(i), loadSize);
+    }
+    return vector;
+  }
+
+  @Override
+  public boolean isLoadingSkipped() {
+    return true;
+  }
+}

src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkNullLoader.java

@@ -14,13 +14,13 @@
  */
 package jp.co.yahoo.yosegi.spark.inmemory.loader;
 
-import jp.co.yahoo.yosegi.inmemory.ISequentialLoader;
+import jp.co.yahoo.yosegi.inmemory.ILoader;
 import jp.co.yahoo.yosegi.inmemory.LoadType;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
 import java.io.IOException;
 
-public class SparkNullLoader implements ISequentialLoader<WritableColumnVector> {
+public class SparkNullLoader implements ILoader<WritableColumnVector> {
 
   private final WritableColumnVector vector;
   private final int loadSize;
@@ -42,17 +42,22 @@ public int getLoadSize() {
 
   @Override
   public void setNull(final int index) throws IOException {
-    // TODO:
+    // FIXME:
   }
 
   @Override
   public void finish() throws IOException {
     // FIXME:
-    vector.putNulls(0, loadSize);
   }
 
   @Override
   public WritableColumnVector build() throws IOException {
+    vector.putNulls(0, loadSize);
     return vector;
   }
+
+  @Override
+  public boolean isLoadingSkipped() {
+    return true;
+  }
 }