Rebase with latest API branch

xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java

@@ -25,15 +25,27 @@
 import lombok.NonNull;
 import lombok.Value;
 
-/** Defines the configuration for an external catalog. */
+/**
+ * Defines the configuration for an external catalog, user needs to populate at-least one of
+ * catalogType or catalogImpl
+ */
 @Value
 @Builder
-public class ExternalCatalogConfig implements CatalogConfig {
+public class ExternalCatalogConfig {
   /** The name of the catalog, it also acts as a unique identifier for each catalog */
-  @NonNull String catalogName;
+  @NonNull String catalogId;
+
+  /**
+   * The type of the catalog. If the catalogType implementation exists in XTable, the implementation
+   * class will be inferred.
+   */
+  String catalogType;
 
-  /** The implementation class path for the catalog */
-  @NonNull String catalogImpl;
+  /**
+   * (Optional) A fully qualified class name that implements the interfaces for CatalogSyncClient,
+   * it can be used if the implementation for catalogType doesn't exist in XTable.
+   */
+  String catalogImpl;
 
   /**
    * The properties for each catalog, used for providing any custom behaviour during catalog sync

xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java

@@ -32,7 +32,7 @@
 @Builder
 public class TargetCatalogConfig {
   /**
-   * The tableIdentifiers(databaseName, tableName) that will be used when syncing {@link
+   * The tableIdentifiers(catalogName.databaseName.tableName) that will be used when syncing {@link
    * TargetTable} to the catalog.
    */
   @NonNull CatalogTableIdentifier catalogTableIdentifier;

xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java

@@ -26,12 +26,12 @@
 public class ExternalCatalogConfigFactory {
 
   public static ExternalCatalogConfig fromCatalogType(
-      String catalogType, String catalogName, Map<String, String> properties) {
+      String catalogType, String catalogId, Map<String, String> properties) {
     // TODO: Choose existing implementation based on catalogType.
     String catalogImpl = "";
     return ExternalCatalogConfig.builder()
         .catalogImpl(catalogImpl)
-        .catalogName(catalogName)
+        .catalogId(catalogId)
         .catalogOptions(properties)
         .build();
   }

xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java

@@ -30,7 +30,7 @@
 /**
  * Iceberg requires a catalog to perform any operation, if no catalog is provided the default
  * catalog (HadoopCatalog or storage based catalog) is used. For syncing iceberg to multiple
- * catalogs, you can use {@link org.apache.xtable.catalog.ExternalCatalogConfig} instead which
+ * catalogs, you can use {@link org.apache.xtable.conversion.ExternalCatalogConfig} instead which
  * allows syncing the latest version of iceberg metadata to multiple catalogs.
  */
 @Value

xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java

@@ -38,7 +38,7 @@ class TestCatalogConversionFactory {
   void createSourceForConfig() {
     ExternalCatalogConfig sourceCatalog =
         ExternalCatalogConfig.builder()
-            .catalogName("catalogName")
+            .catalogId("catalogId")
             .catalogImpl(TestCatalogImpl.class.getName())
             .catalogOptions(Collections.emptyMap())
             .build();
@@ -53,7 +53,7 @@ void createForCatalog() {
         TargetCatalogConfig.builder()
             .catalogConfig(
                 ExternalCatalogConfig.builder()
-                    .catalogName("catalogName")
+                    .catalogId("catalogId")
                     .catalogImpl(TestCatalogImpl.class.getName())
                     .catalogOptions(Collections.emptyMap())
                     .build())

xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java

@@ -589,7 +589,7 @@ private TargetCatalogConfig getTargetCatalog(String suffix) {
     return TargetCatalogConfig.builder()
         .catalogConfig(
             ExternalCatalogConfig.builder()
-                .catalogName("catalogName-" + suffix)
+                .catalogId("catalogId-" + suffix)
                 .catalogImpl("catalogImpl-" + suffix)
                 .catalogOptions(Collections.emptyMap())
                 .build())

xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java

@@ -65,7 +65,7 @@ public SourceTable getSourceTable(CatalogTableIdentifier tableIdentifier) {
     }
 
     @Override
-    public String getCatalogName() {
+    public String getCatalogId() {
       return null;
     }
 

xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java

@@ -134,7 +134,7 @@ public static void main(String[] args) throws Exception {
 
     Map<String, DatasetConfig.Catalog> catalogsByName =
         datasetConfig.getTargetCatalogs().stream()
-            .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogName, Function.identity()));
+            .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogId, Function.identity()));
     ExternalCatalogConfig sourceCatalogConfig = getCatalogConfig(datasetConfig.getSourceCatalog());
     CatalogConversionSource catalogConversionSource =
         CatalogConversionFactory.createCatalogConversionSource(sourceCatalogConfig, hadoopConf);
@@ -183,7 +183,7 @@ public static void main(String[] args) throws Exception {
                         targetCatalogTableIdentifier.getCatalogTableIdentifier())
                     .catalogConfig(
                         getCatalogConfig(
-                            catalogsByName.get(targetCatalogTableIdentifier.getCatalogName())))
+                            catalogsByName.get(targetCatalogTableIdentifier.getCatalogId())))
                     .build());
       }
       ConversionConfig conversionConfig =
@@ -211,10 +211,10 @@ public static void main(String[] args) throws Exception {
   static ExternalCatalogConfig getCatalogConfig(DatasetConfig.Catalog catalog) {
     if (!StringUtils.isEmpty(catalog.getCatalogType())) {
       return ExternalCatalogConfigFactory.fromCatalogType(
-          catalog.getCatalogType(), catalog.getCatalogName(), catalog.getCatalogProperties());
+          catalog.getCatalogType(), catalog.getCatalogId(), catalog.getCatalogProperties());
     } else {
       return ExternalCatalogConfig.builder()
-          .catalogName(catalog.getCatalogName())
+          .catalogId(catalog.getCatalogId())
           .catalogImpl(catalog.getCatalogImpl())
           .catalogOptions(catalog.getCatalogProperties())
           .build();
@@ -264,8 +264,8 @@ public static class DatasetConfig {
     /** Configuration for catalog. */
     @Data
     public static class Catalog {
-      /** A unique name for the catalog. */
-      private String catalogName;
+      /** A user defined unique identifier for the catalog. */
+      private String catalogId;
       /**
        * The type of the source catalog. This might be a specific type understood by XTable, such as
        * Hive, Glue etc.
@@ -305,8 +305,11 @@ public static class SourceTableIdentifier {
 
     @Data
     public static class TargetTableIdentifier {
-      /** name of the target catalog where the table will be created or updated */
-      String catalogName;
+      /**
+       * The user defined unique identifier of the target {@link Catalog} where the table will be
+       * created or updated
+       */
+      String catalogId;
       /**
        * The target table format (e.g., DELTA, HUDI, ICEBERG), specifying how the data will be
        * stored at the target.

xtable-utilities/src/test/resources/catalogConfig.yaml

@@ -15,26 +15,26 @@
 # limitations under the License.
 #
 sourceCatalog:
-  catalogName: "source-1"
+  catalogId: "source-1"
   catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl"
   catalogProperties:
     key01: "value1"
     key02: "value2"
     key03: "value3"
 targetCatalogs:
-  - catalogName: "target-1"
+  - catalogId: "target-1"
     catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl"
     catalogProperties:
       key11: "value1"
       key12: "value2"
       key13: "value3"
-  - catalogName: "target-2"
+  - catalogId: "target-2"
     catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl"
     catalogProperties:
       key21: "value1"
       key22: "value2"
       key23: "value3"
-  - catalogName: "target-3"
+  - catalogId: "target-3"
     catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl"
     catalogProperties:
       key31: "value1"
@@ -46,12 +46,12 @@ datasets:
         databaseName: "source-database-1"
         tableName: "source-1"
     targetCatalogTableIdentifiers:
-      - catalogName: "target-1"
+      - catalogId: "target-1"
         tableFormat: "DELTA"
         catalogTableIdentifier:
           databaseName: "target-database-1"
           tableName: "target-tableName-1"
-      - catalogName: "target-2"
+      - catalogId: "target-2"
         tableFormat: "HUDI"
         catalogTableIdentifier:
           databaseName: "target-database-2"
@@ -63,13 +63,14 @@ datasets:
         partitionSpec: cs_sold_date_sk:VALUE
         tableFormat: "HUDI"
     targetCatalogTableIdentifiers:
-      - catalogName: "target-2"
+      - catalogId: "target-2"
         tableFormat: "ICEBERG"
         catalogTableIdentifier:
           databaseName: "target-database-2"
           tableName: "target-tableName-2"
-      - catalogName: "target-3"
+      - catalogId: "target-3"
         tableFormat: "HUDI"
         catalogTableIdentifier:
+          catalogName: "default-catalog-2"
           databaseName: "target-database-3"
           tableName: "target-tableName-3"