feat(java/python): new xlang type system spec implementation

BUILD

@@ -62,6 +62,7 @@ pyx_library(
     ),
     deps = [
         "//cpp/fury/util:fury_util",
+        "//cpp/fury/type:fury_type",
         "@com_google_absl//absl/container:flat_hash_map",
     ],
 )

README.md

@@ -278,7 +278,7 @@ class SomeClass:
     f3: Dict[str, str]
 
 fury = pyfury.Fury(ref_tracking=True)
-fury.register_class(SomeClass, type_tag="example.SomeClass")
+fury.register_type(SomeClass, typename="example.SomeClass")
 obj = SomeClass()
 obj.f2 = {"k1": "v1", "k2": "v2"}
 obj.f1, obj.f3 = obj, obj.f2

cpp/fury/type/BUILD

@@ -0,0 +1,15 @@
+load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+
+cc_library(
+    name = "fury_type",
+    srcs = glob(["*.cc"], exclude=["*test.cc"]),
+    hdrs = glob(["*.h"]),
+    copts = ["-mavx2"],  # Enable AVX2 support
+    linkopts = ["-mavx2"],  # Ensure linker also knows about AVX2
+    strip_include_prefix = "/cpp",
+    alwayslink=True,
+    linkstatic=True,
+    deps = [
+    ],
+    visibility = ["//visibility:public"],
+)

cpp/fury/type/type.h

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstdint> // For fixed-width integer types
+
+namespace fury {
+enum class TypeId : int32_t {
+  // Fury added type for cross-language serialization.
+  BOOL = 1,
+  INT8 = 2,
+  INT16 = 3,
+  INT32 = 4,
+  VAR_INT32 = 5,
+  INT64 = 6,
+  VAR_INT64 = 7,
+  SLI_INT64 = 8,
+  FLOAT16 = 9,
+  FLOAT32 = 10,
+  FLOAT64 = 11,
+  STRING = 12,
+  ENUM = 13,
+  NAMED_ENUM = 14,
+  STRUCT = 15,
+  POLYMORPHIC_STRUCT = 16,
+  COMPATIBLE_STRUCT = 17,
+  POLYMORPHIC_COMPATIBLE_STRUCT = 18,
+  NAMED_STRUCT = 19,
+  NAMED_POLYMORPHIC_STRUCT = 20,
+  NAMED_COMPATIBLE_STRUCT = 21,
+  NAMED_POLYMORPHIC_COMPATIBLE_STRUCT = 22,
+  EXT = 23,
+  POLYMORPHIC_EXT = 24,
+  NAMED_EXT = 25,
+  NAMED_POLYMORPHIC_EXT = 26,
+  LIST = 27,
+  SET = 28,
+  MAP = 29,
+  DURATION = 30,
+  TIMESTAMP = 31,
+  LOCAL_DATE = 32,
+  DECIMAL = 33,
+  BINARY = 34,
+  ARRAY = 35,
+  BOOL_ARRAY = 36,
+  INT8_ARRAY = 37,
+  INT16_ARRAY = 38,
+  INT32_ARRAY = 39,
+  INT64_ARRAY = 40,
+  FLOAT16_ARRAY = 41,
+  FLOAT32_ARRAY = 42,
+  FLOAT64_ARRAY = 43,
+  ARROW_RECORD_BATCH = 44,
+  ARROW_TABLE = 45
+};
+
+inline bool IsNamespacedType(int32_t type_id) {
+  switch (static_cast<TypeId>(type_id)) {
+  case TypeId::NAMED_ENUM:
+  case TypeId::NAMED_STRUCT:
+  case TypeId::NAMED_POLYMORPHIC_STRUCT:
+  case TypeId::NAMED_COMPATIBLE_STRUCT:
+  case TypeId::NAMED_POLYMORPHIC_COMPATIBLE_STRUCT:
+  case TypeId::NAMED_EXT:
+  case TypeId::NAMED_POLYMORPHIC_EXT:
+    return true;
+  default:
+    return false;
+  }
+}
+
+} // namespace fury

cpp/fury/util/buffer.h

@@ -27,6 +27,7 @@
 
 #include "fury/util/bit_util.h"
 #include "fury/util/logging.h"
+#include "fury/util/status.h"
 
 namespace fury {
 
@@ -133,6 +134,29 @@ class Buffer {
 
   inline double GetDouble(uint32_t offset) { return Get<double>(offset); }
 
+  inline Status GetBytesAsInt64(uint32_t offset, uint32_t length,
+                                int64_t *target) {
+    if (length == 0) {
+      *target = 0;
+      return Status::OK();
+    }
+    if (size_ - (offset + 8) > 0) {
+      uint64_t mask = 0xffffffffffffffff;
+      uint64_t x = (mask >> (8 - length) * 8);
+      *target = GetInt64(offset) & x;
+    } else {
+      if (size_ - (offset + length) < 0) {
+        return Status::OutOfBound("buffer out of bound");
+      }
+      int64_t result = 0;
+      for (size_t i = 0; i < length; i++) {
+        result = result | ((int64_t)(data_[offset + i])) << (i * 8);
+      }
+      *target = result;
+    }
+    return Status::OK();
+  }
+
   inline uint32_t PutVarUint32(uint32_t offset, int32_t value) {
     if (value >> 7 == 0) {
       data_[offset] = (int8_t)value;

cpp/fury/util/buffer_test.cc

@@ -65,6 +65,16 @@ TEST(Buffer, TestVarUint) {
   }
 }
 
+TEST(Buffer, TestGetBytesAsInt64) {
+  std::shared_ptr<Buffer> buffer;
+  AllocateBuffer(64, &buffer);
+  buffer->UnsafePut<int32_t>(0, 100);
+  int64_t result = -1;
+  EXPECT_TRUE(buffer->GetBytesAsInt64(0, 0, &result).ok());
+  EXPECT_EQ(result, 0);
+  EXPECT_TRUE(buffer->GetBytesAsInt64(0, 1, &result).ok());
+  EXPECT_EQ(result, 100);
+}
 } // namespace fury
 
 int main(int argc, char **argv) {

cpp/fury/util/logging.cc

@@ -111,4 +111,4 @@ bool FuryLog::IsLevelEnabled(FuryLogLevel log_level) {
   return log_level >= fury_severity_threshold;
 }
 
-} // namespace fury
+} // namespace fury

cpp/fury/util/status.h

@@ -83,11 +83,12 @@ namespace fury {
 enum class StatusCode : char {
   OK = 0,
   OutOfMemory = 1,
-  KeyError = 2,
-  TypeError = 3,
-  Invalid = 4,
-  IOError = 5,
-  UnknownError = 6,
+  OutOfBound = 2,
+  KeyError = 3,
+  TypeError = 4,
+  Invalid = 5,
+  IOError = 6,
+  UnknownError = 7,
 };
 
 class Status {
@@ -123,6 +124,10 @@ class Status {
     return Status(StatusCode::OutOfMemory, msg);
   }
 
+  static Status OutOfBound(const std::string &msg) {
+    return Status(StatusCode::OutOfMemory, msg);
+  }
+
   static Status KeyError(const std::string &msg) {
     return Status(StatusCode::KeyError, msg);
   }

docs/guide/xlang_serialization_guide.md

@@ -225,8 +225,8 @@ class SomeClass2:
 
 if __name__ == "__main__":
     f = pyfury.Fury()
-    f.register_class(SomeClass1, type_tag="example.SomeClass1")
-    f.register_class(SomeClass2, type_tag="example.SomeClass2")
+    f.register_type(SomeClass1, typename="example.SomeClass1")
+    f.register_type(SomeClass2, typename="example.SomeClass2")
     obj1 = SomeClass1(f1=True, f2={-1: 2})
     obj = SomeClass2(
         f1=obj1,
@@ -444,7 +444,7 @@ class SomeClass:
     f3: Dict[str, str]
 
 fury = pyfury.Fury(ref_tracking=True)
-fury.register_class(SomeClass, type_tag="example.SomeClass")
+fury.register_type(SomeClass, typename="example.SomeClass")
 obj = SomeClass()
 obj.f2 = {"k1": "v1", "k2": "v2"}
 obj.f1, obj.f3 = obj, obj.f2

docs/specification/xlang_serialization_spec.md

@@ -68,6 +68,11 @@ Note:
 
 - Unsigned int/long are not added here, since not every language support those types.
 
+### Polymorphisms
+
+For polymorphism, if one non-final class is registered, and only one subclass is registered, then we can take all
+elements in List/Map have same type, thus reduce runtime check cost.
+
 ### Type disambiguation
 
 Due to differences between type systems of languages, those types can't be mapped one-to-one between languages. When

go/fury/fury_xlang_test.go

@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build skiptest
+// +build skiptest
+
 package fury_test
 
 import (

java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java

@@ -38,15 +38,25 @@ public void register(Class<?> cls, boolean createSerializer) {
   }
 
   @Override
-  public void register(Class<?> cls, Short id) {
+  public void register(Class<?> cls, int id) {
     registerCallback(fury -> fury.register(cls, id));
   }
 
   @Override
-  public void register(Class<?> cls, Short id, boolean createSerializer) {
+  public void register(Class<?> cls, int id, boolean createSerializer) {
     registerCallback(fury -> fury.register(cls, id, createSerializer));
   }
 
+  @Override
+  public void register(Class<?> cls, String typeName) {
+    registerCallback(fury -> fury.register(cls, typeName));
+  }
+
+  @Override
+  public void register(Class<?> cls, String namespace, String typeName) {
+    registerCallback(fury -> fury.register(cls, namespace, typeName));
+  }
+
   @Override
   public <T> void registerSerializer(Class<T> type, Class<? extends Serializer> serializerClass) {
     registerCallback(fury -> fury.registerSerializer(type, serializerClass));

java/fury-core/src/main/java/org/apache/fury/BaseFury.java

@@ -33,24 +33,28 @@
 public interface BaseFury {
 
   /**
-   * register class.
+   * Register class and allocate an auto-grown ID for this class. Note that the registration order
+   * is important. If registration order is inconsistent, the allocated ID will be different, and
+   * the deserialization will failed.
    *
    * @param cls class to register.
    */
   void register(Class<?> cls);
 
+  /** register class with given id. */
+  void register(Class<?> cls, int id);
+
   /**
-   * Register class.
+   * Register class and allocate an auto-grown ID for this class. Note that the registration order
+   * is important. If registration order is inconsistent, the allocated ID will be different, and
+   * the deserialization will failed.
    *
    * @param cls class to register.
    * @param createSerializer whether to create serializer, if true and codegen enabled, this will
    *     generate the serializer code too.
    */
   void register(Class<?> cls, boolean createSerializer);
 
-  /** register class with given id. */
-  void register(Class<?> cls, Short id);
-
   /**
    * Register class with specified id.
    *
@@ -59,21 +63,40 @@ public interface BaseFury {
    * @param createSerializer whether to create serializer, if true and codegen enabled, this will
    *     generate the serializer code too.
    */
-  void register(Class<?> cls, Short id, boolean createSerializer);
+  void register(Class<?> cls, int id, boolean createSerializer);
+
+  /** register class with given type name which will be used for cross-language serialization. */
+  void register(Class<?> cls, String typeName);
 
   /**
-   * Register a Serializer.
+   * register class with given type namespace and name which will be used for cross-language
+   * serialization.
+   */
+  void register(Class<?> cls, String namespace, String typeName);
+
+  /**
+   * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not
+   * registered yet. Note that the registration order is important. If registration order is
+   * inconsistent, the allocated ID will be different, and the deserialization will failed.
    *
    * @param type class needed to be serialized/deserialized.
    * @param serializerClass serializer class can be created with {@link Serializers#newSerializer}.
    * @param <T> type of class.
    */
   <T> void registerSerializer(Class<T> type, Class<? extends Serializer> serializerClass);
 
+  /**
+   * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not
+   * registered yet. Note that the registration order is important. If registration order is
+   * inconsistent, the allocated ID will be different, and the deserialization will failed.
+   */
   void registerSerializer(Class<?> type, Serializer<?> serializer);
 
   /**
-   * Register a Serializer created by serializerCreator when fury created.
+   * Register a Serializer created by serializerCreator when fury created. And allocate an
+   * auto-grown ID for this class if it's not registered yet. Note that the registration order is
+   * important. If registration order is inconsistent, the allocated ID will be different, and the
+   * deserialization will failed.
    *
    * @param type class needed to be serialized/deserialized.
    * @param serializerCreator serializer creator with param {@link Fury}
@@ -107,6 +130,8 @@ public interface BaseFury {
   /** Deserialize <code>obj</code> from a byte array. */
   Object deserialize(byte[] bytes);
 
+  <T> T deserialize(byte[] bytes, Class<T> type);
+
   Object deserialize(byte[] bytes, Iterable<MemoryBuffer> outOfBandBuffers);
 
   /**