This is an automated email from the ASF dual-hosted git repository. vinish pushed a commit to branch 590-CatalogSync in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit 893b1a6727af8b59e7d4cec79b4dc0311d6e6134 Author: Vinish Reddy <[email protected]> AuthorDate: Mon Dec 23 17:33:08 2024 -0800 Rebase with HierarchicalTableIdentifier changes --- .../catalog/TestCatalogConversionFactory.java | 7 ++---- .../conversion/TestConversionController.java | 8 +++---- .../org/apache/xtable/testutil/ITTestUtils.java | 4 ++-- .../apache/xtable/utilities/RunCatalogSync.java | 28 ++++++++++++++++++---- .../src/test/resources/catalogConfig.yaml | 26 ++++++++------------ 5 files changed, 41 insertions(+), 32 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java index 53e68923..3d75221a 100644 --- a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java +++ b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java @@ -27,7 +27,7 @@ import org.junit.jupiter.api.Test; import org.apache.xtable.conversion.ExternalCatalogConfig; import org.apache.xtable.conversion.TargetCatalogConfig; -import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.spi.extractor.CatalogConversionSource; import org.apache.xtable.spi.sync.CatalogSyncClient; import org.apache.xtable.testutil.ITTestUtils.TestCatalogConversionSourceImpl; @@ -61,10 +61,7 @@ class TestCatalogConversionFactory { .catalogProperties(Collections.emptyMap()) .build()) .catalogTableIdentifier( - CatalogTableIdentifier.builder() - .databaseName("target-database") - .tableName("target-tableName") - .build()) + new HierarchicalTableIdentifier("target-database", "target-tableName")) .build(); CatalogSyncClient catalogSyncClient = CatalogConversionFactory.getInstance() diff --git a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java index 0600c5ee..2a6461de 100644 --- a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java @@ -57,7 +57,7 @@ import org.apache.xtable.model.InstantsForIncrementalSync; import org.apache.xtable.model.InternalSnapshot; import org.apache.xtable.model.InternalTable; import org.apache.xtable.model.TableChange; -import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.model.metadata.TableSyncMetadata; import org.apache.xtable.model.storage.TableFormat; import org.apache.xtable.model.sync.SyncMode; @@ -594,10 +594,8 @@ public class TestConversionController { .catalogProperties(Collections.emptyMap()) .build()) .catalogTableIdentifier( - CatalogTableIdentifier.builder() - .databaseName("target-database" + suffix) - .tableName("target-tableName" + suffix) - .build()) + new HierarchicalTableIdentifier( + "target-database" + suffix, "target-tableName" + suffix)) .build(); } diff --git a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java index 04be989d..d194558a 100644 --- a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java +++ b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java @@ -67,12 +67,12 @@ public class ITTestUtils { } @Override - public boolean hasDatabase(String databaseName) { + public boolean hasDatabase(CatalogTableIdentifier tableIdentifier) { return false; } @Override - public void createDatabase(String databaseName) {} + public void createDatabase(CatalogTableIdentifier tableIdentifier) {} @Override public Object getTable(CatalogTableIdentifier tableIdentifier) { diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java index a423fbc2..4d12d088 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java @@ -57,10 +57,12 @@ import org.apache.xtable.conversion.SourceTable; import org.apache.xtable.conversion.TargetCatalogConfig; import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.model.sync.SyncMode; import org.apache.xtable.reflection.ReflectionUtils; import org.apache.xtable.spi.extractor.CatalogConversionSource; import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.StorageIdentifier; +import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TableIdentifier; import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TargetTableIdentifier; /** @@ -156,7 +158,8 @@ public class RunCatalogSync { } else { sourceTable = catalogConversionSource.getSourceTable( - dataset.getSourceCatalogTableIdentifier().getCatalogTableIdentifier()); + getCatalogTableIdentifier( + dataset.getSourceCatalogTableIdentifier().getTableIdentifier())); } List<TargetTable> targetTables = new ArrayList<>(); Map<TargetTable, List<TargetCatalogConfig>> targetCatalogs = new HashMap<>(); @@ -178,7 +181,8 @@ public class RunCatalogSync { .add( TargetCatalogConfig.builder() .catalogTableIdentifier( - targetCatalogTableIdentifier.getCatalogTableIdentifier()) + getCatalogTableIdentifier( + targetCatalogTableIdentifier.getTableIdentifier())) .catalogConfig(catalogsById.get(targetCatalogTableIdentifier.getCatalogId())) .build()); } @@ -229,6 +233,13 @@ public class RunCatalogSync { return CONVERSION_SOURCE_PROVIDERS; } + static CatalogTableIdentifier getCatalogTableIdentifier(TableIdentifier tableIdentifier) { + if (tableIdentifier.getHierarchicalId() != null) { + return new HierarchicalTableIdentifier(tableIdentifier.getHierarchicalId()); + } + throw new IllegalArgumentException("Invalid tableIdentifier configuration provided"); + } + @Data public static class DatasetConfig { /** @@ -258,7 +269,7 @@ public class RunCatalogSync { @Data public static class SourceTableIdentifier { /** Specifies the table identifier in the source catalog. */ - CatalogTableIdentifier catalogTableIdentifier; + TableIdentifier tableIdentifier; /** * (Optional) Provides direct storage details such as a table’s base path (like an S3 * location) and the partition specification. This allows reading from a source even if it is @@ -280,7 +291,16 @@ public class RunCatalogSync { */ String tableFormat; /** Specifies the table identifier in the target catalog. */ - CatalogTableIdentifier catalogTableIdentifier; + TableIdentifier tableIdentifier; + } + + @Data + public static class TableIdentifier { + /** + * Specifics the three level hierarchical table identifier for {@link + * HierarchicalTableIdentifier} + */ + String hierarchicalId; } /** diff --git a/xtable-utilities/src/test/resources/catalogConfig.yaml b/xtable-utilities/src/test/resources/catalogConfig.yaml index 84773235..05b2df4b 100644 --- a/xtable-utilities/src/test/resources/catalogConfig.yaml +++ b/xtable-utilities/src/test/resources/catalogConfig.yaml @@ -43,20 +43,17 @@ targetCatalogs: key33: "value3" datasets: - sourceCatalogTableIdentifier: - catalogTableIdentifier: - databaseName: "source-database-1" - tableName: "source-1" + tableIdentifier: + hierarchicalId: "source-database-1.source-1" targetCatalogTableIdentifiers: - catalogId: "target-1" tableFormat: "DELTA" - catalogTableIdentifier: - databaseName: "target-database-1" - tableName: "target-tableName-1" + tableIdentifier: + hierarchicalId: "target-database-1.target-tableName-1" - catalogId: "target-2" tableFormat: "HUDI" - catalogTableIdentifier: - databaseName: "target-database-2" - tableName: "target-tableName-2-delta" + tableIdentifier: + hierarchicalId: "target-database-2.target-tableName-2-delta" - sourceCatalogTableIdentifier: storageIdentifier: tableBasePath: s3://tpc-ds-datasets/1GB/hudi/catalog_sales @@ -66,12 +63,9 @@ datasets: targetCatalogTableIdentifiers: - catalogId: "target-2" tableFormat: "ICEBERG" - catalogTableIdentifier: - databaseName: "target-database-2" - tableName: "target-tableName-2" + tableIdentifier: + hierarchicalId: "target-database-2.target-tableName-2" - catalogId: "target-3" tableFormat: "HUDI" - catalogTableIdentifier: - catalogName: "default-catalog-2" - databaseName: "target-database-3" - tableName: "target-tableName-3" + tableIdentifier: + hierarchicalId: "default-catalog-2.target-database-3.target-tableName-3"
