This is an automated email from the ASF dual-hosted git repository.
vinish pushed a commit to branch 590-CatalogSync
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/590-CatalogSync by this push:
new 4e339992 Refactor catalogId to existing catalogName
4e339992 is described below
commit 4e33999296528cdb67e7231631fcb71f30fb14fb
Author: Vinish Reddy <[email protected]>
AuthorDate: Fri Dec 13 15:04:20 2024 -0800
Refactor catalogId to existing catalogName
---
.../java/org/apache/xtable/conversion/CatalogConfig.java | 2 +-
.../apache/xtable/conversion/ExternalCatalogConfig.java | 2 +-
.../java/org/apache/xtable/model/sync/SyncResult.java | 2 +-
.../java/org/apache/xtable/spi/sync/CatalogSync.java | 8 ++++----
.../org/apache/xtable/spi/sync/CatalogSyncClient.java | 2 +-
.../java/org/apache/xtable/spi/sync/TestCatalogSync.java | 2 +-
.../xtable/catalog/ExternalCatalogConfigFactory.java | 4 ++--
.../org/apache/xtable/iceberg/IcebergCatalogConfig.java | 2 +-
.../org/apache/xtable/iceberg/IcebergTableManager.java | 2 +-
.../xtable/catalog/TestCatalogConversionFactory.java | 6 +++---
.../xtable/conversion/TestConversionController.java | 2 +-
.../apache/xtable/iceberg/TestIcebergTableManager.java | 8 ++++----
.../java/org/apache/xtable/utilities/RunCatalogSync.java | 14 +++++++-------
.../org/apache/xtable/utilities/TestRunCatalogSync.java | 2 +-
.../java/org/apache/xtable/utilities/TestRunSync.java | 2 +-
xtable-utilities/src/test/resources/catalogConfig.yaml | 16 ++++++++--------
16 files changed, 38 insertions(+), 38 deletions(-)
diff --git
a/xtable-api/src/main/java/org/apache/xtable/conversion/CatalogConfig.java
b/xtable-api/src/main/java/org/apache/xtable/conversion/CatalogConfig.java
index f265197a..a432b749 100644
--- a/xtable-api/src/main/java/org/apache/xtable/conversion/CatalogConfig.java
+++ b/xtable-api/src/main/java/org/apache/xtable/conversion/CatalogConfig.java
@@ -29,7 +29,7 @@ import java.util.Map;
public interface CatalogConfig {
String getCatalogImpl();
- String getCatalogId();
+ String getCatalogName();
Map<String, String> getCatalogOptions();
}
diff --git
a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java
b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java
index 53508ce5..16785ec6 100644
---
a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java
+++
b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java
@@ -30,7 +30,7 @@ import lombok.Value;
@Builder
public class ExternalCatalogConfig implements CatalogConfig {
/** The name of the catalog, it also acts as a unique identifier for each
catalog */
- @NonNull String catalogId;
+ @NonNull String catalogName;
/** The implementation class path for the catalog */
@NonNull String catalogImpl;
diff --git
a/xtable-api/src/main/java/org/apache/xtable/model/sync/SyncResult.java
b/xtable-api/src/main/java/org/apache/xtable/model/sync/SyncResult.java
index f5707b81..390ab2e1 100644
--- a/xtable-api/src/main/java/org/apache/xtable/model/sync/SyncResult.java
+++ b/xtable-api/src/main/java/org/apache/xtable/model/sync/SyncResult.java
@@ -69,7 +69,7 @@ public class SyncResult {
@Builder
public static class CatalogSyncStatus {
// Catalog Identifier.
- String catalogId;
+ String catalogName;
// Status code
SyncStatusCode statusCode;
// errorDetails if any
diff --git
a/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSync.java
b/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSync.java
index f115d5b7..a8eb147d 100644
--- a/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSync.java
+++ b/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSync.java
@@ -66,7 +66,7 @@ public class CatalogSync {
catalogSyncClient.getCatalogImpl());
results.add(
getCatalogSyncFailureStatus(
- catalogSyncClient.getCatalogId(),
catalogSyncClient.getCatalogImpl(), e));
+ catalogSyncClient.getCatalogName(),
catalogSyncClient.getCatalogImpl(), e));
}
}));
return SyncResult.builder()
@@ -107,15 +107,15 @@ public class CatalogSync {
catalogSyncClient.refreshTable(table, catalogTable, tableIdentifier);
}
return CatalogSyncStatus.builder()
- .catalogId(catalogSyncClient.getCatalogId())
+ .catalogName(catalogSyncClient.getCatalogName())
.statusCode(SyncResult.SyncStatusCode.SUCCESS)
.build();
}
private CatalogSyncStatus getCatalogSyncFailureStatus(
- String catalogId, String catalogImpl, Exception e) {
+ String catalogName, String catalogImpl, Exception e) {
return CatalogSyncStatus.builder()
- .catalogId(catalogId)
+ .catalogName(catalogName)
.statusCode(SyncResult.SyncStatusCode.ERROR)
.errorDetails(
SyncResult.ErrorDetails.builder()
diff --git
a/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSyncClient.java
b/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSyncClient.java
index 5f32628d..94b1ceab 100644
--- a/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSyncClient.java
+++ b/xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSyncClient.java
@@ -32,7 +32,7 @@ public interface CatalogSyncClient<TABLE> extends
AutoCloseable {
* Returns a unique identifier for the catalog, allows user to sync table to
multiple catalogs of
* the same type eg: HMS catalogs with url1, HMS catalog with url2.
*/
- String getCatalogId();
+ String getCatalogName();
/**
* Returns the full class name which implements the interface for
CatalogSyncClient.
diff --git
a/xtable-api/src/test/java/org/apache/xtable/spi/sync/TestCatalogSync.java
b/xtable-api/src/test/java/org/apache/xtable/spi/sync/TestCatalogSync.java
index 2f644cb9..bdc7f8db 100644
--- a/xtable-api/src/test/java/org/apache/xtable/spi/sync/TestCatalogSync.java
+++ b/xtable-api/src/test/java/org/apache/xtable/spi/sync/TestCatalogSync.java
@@ -95,7 +95,7 @@ public class TestCatalogSync<TABLE> {
when(mockClient2.getStorageDescriptorLocation(any())).thenReturn("/tmp/test");
when(mockClient3.getStorageDescriptorLocation(any())).thenReturn("/tmp/test");
- when(mockClient4.getCatalogId()).thenReturn("catalogId4");
+ when(mockClient4.getCatalogName()).thenReturn("catalogName4");
when(mockClient4.getCatalogImpl()).thenReturn("catalogImpl4");
Map<CatalogTableIdentifier, CatalogSyncClient> catalogSyncClients =
diff --git
a/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java
b/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java
index f053df32..09bf1566 100644
---
a/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java
+++
b/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java
@@ -26,12 +26,12 @@ import org.apache.xtable.conversion.ExternalCatalogConfig;
public class ExternalCatalogConfigFactory {
public static ExternalCatalogConfig fromCatalogType(
- String catalogType, String catalogId, Map<String, String> properties) {
+ String catalogType, String catalogName, Map<String, String> properties) {
// TODO: Choose existing implementation based on catalogType.
String catalogImpl = "";
return ExternalCatalogConfig.builder()
.catalogImpl(catalogImpl)
- .catalogId(catalogId)
+ .catalogName(catalogName)
.catalogOptions(properties)
.build();
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java
index e3123959..b678bf00 100644
---
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java
+++
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java
@@ -36,7 +36,7 @@ import org.apache.xtable.conversion.CatalogConfig;
@Value
@Builder
public class IcebergCatalogConfig implements CatalogConfig {
- @NonNull String catalogId;
+ @NonNull String catalogName;
@NonNull String catalogImpl;
@NonNull @Builder.Default Map<String, String> catalogOptions =
Collections.emptyMap();
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergTableManager.java
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergTableManager.java
index 9a3010a5..431184ce 100644
---
a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergTableManager.java
+++
b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergTableManager.java
@@ -115,7 +115,7 @@ class IcebergTableManager {
config ->
CatalogUtil.loadCatalog(
config.getCatalogImpl(),
- config.getCatalogId(),
+ config.getCatalogName(),
config.getCatalogOptions(),
hadoopConfiguration)));
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
index a648c5b1..90847a85 100644
---
a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
+++
b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
@@ -39,7 +39,7 @@ class TestCatalogConversionFactory {
void createSourceForConfig() {
ExternalCatalogConfig sourceCatalog =
ExternalCatalogConfig.builder()
- .catalogId("catalogId")
+ .catalogName("catalogName")
.catalogImpl(TestCatalogImpl.class.getName())
.catalogOptions(Collections.emptyMap())
.build();
@@ -54,7 +54,7 @@ class TestCatalogConversionFactory {
TargetCatalogConfig.builder()
.catalogConfig(
ExternalCatalogConfig.builder()
- .catalogId("catalogId")
+ .catalogName("catalogName")
.catalogImpl(TestCatalogImpl.class.getName())
.catalogOptions(Collections.emptyMap())
.build())
@@ -81,7 +81,7 @@ class TestCatalogConversionFactory {
}
@Override
- public String getCatalogId() {
+ public String getCatalogName() {
return null;
}
diff --git
a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java
b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java
index bfafd73d..faf44888 100644
---
a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java
+++
b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java
@@ -588,7 +588,7 @@ public class TestConversionController {
return TargetCatalogConfig.builder()
.catalogConfig(
ExternalCatalogConfig.builder()
- .catalogId("catalogName-" + suffix)
+ .catalogName("catalogName-" + suffix)
.catalogImpl("catalogImpl-" + suffix)
.catalogOptions(Collections.emptyMap())
.build())
diff --git
a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergTableManager.java
b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergTableManager.java
index be12784b..f81f1336 100644
---
a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergTableManager.java
+++
b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergTableManager.java
@@ -55,7 +55,7 @@ public class TestIcebergTableManager {
IcebergCatalogConfig catalogConfig =
IcebergCatalogConfig.builder()
.catalogImpl(StubCatalog.class.getName())
- .catalogId(catalogName)
+ .catalogName(catalogName)
.catalogOptions(OPTIONS)
.build();
Table mockTable = mock(Table.class);
@@ -74,7 +74,7 @@ public class TestIcebergTableManager {
IcebergCatalogConfig catalogConfig =
IcebergCatalogConfig.builder()
.catalogImpl(StubCatalog.class.getName())
- .catalogId(catalogName)
+ .catalogName(catalogName)
.catalogOptions(OPTIONS)
.build();
Table mockTable = mock(Table.class);
@@ -98,7 +98,7 @@ public class TestIcebergTableManager {
IcebergCatalogConfig catalogConfig =
IcebergCatalogConfig.builder()
.catalogImpl(StubCatalog.class.getName())
- .catalogId(catalogName)
+ .catalogName(catalogName)
.catalogOptions(OPTIONS)
.build();
Table mockTable = mock(Table.class);
@@ -131,7 +131,7 @@ public class TestIcebergTableManager {
IcebergCatalogConfig catalogConfig =
IcebergCatalogConfig.builder()
.catalogImpl(StubCatalog.class.getName())
- .catalogId(catalogName)
+ .catalogName(catalogName)
.catalogOptions(OPTIONS)
.build();
Table mockTable = mock(Table.class);
diff --git
a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
index f2adc707..6c67d9ef 100644
---
a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
+++
b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
@@ -132,9 +132,9 @@ public class RunCatalogSync {
RunSync.TableFormatConverters tableFormatConverters =
loadTableFormatConversionConfigs(customConfig);
- Map<String, DatasetConfig.Catalog> catalogsById =
+ Map<String, DatasetConfig.Catalog> catalogsByName =
datasetConfig.getTargetCatalogs().stream()
- .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogId,
Function.identity()));
+ .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogName,
Function.identity()));
ExternalCatalogConfig sourceCatalogConfig =
getCatalogConfig(datasetConfig.getSourceCatalog());
CatalogConversionSource catalogConversionSource =
CatalogConversionFactory.createCatalogConversionSource(sourceCatalogConfig,
hadoopConf);
@@ -183,7 +183,7 @@ public class RunCatalogSync {
targetCatalogTableIdentifier.getCatalogTableIdentifier())
.catalogConfig(
getCatalogConfig(
-
catalogsById.get(targetCatalogTableIdentifier.getCatalogId())))
+
catalogsByName.get(targetCatalogTableIdentifier.getCatalogName())))
.build());
}
ConversionConfig conversionConfig =
@@ -211,10 +211,10 @@ public class RunCatalogSync {
static ExternalCatalogConfig getCatalogConfig(DatasetConfig.Catalog catalog)
{
if (!StringUtils.isEmpty(catalog.getCatalogType())) {
return ExternalCatalogConfigFactory.fromCatalogType(
- catalog.getCatalogType(), catalog.getCatalogId(),
catalog.getCatalogProperties());
+ catalog.getCatalogType(), catalog.getCatalogName(),
catalog.getCatalogProperties());
} else {
return ExternalCatalogConfig.builder()
- .catalogId(catalog.getCatalogId())
+ .catalogName(catalog.getCatalogName())
.catalogImpl(catalog.getCatalogImpl())
.catalogOptions(catalog.getCatalogProperties())
.build();
@@ -254,7 +254,7 @@ public class RunCatalogSync {
@Data
public static class Catalog {
- private String catalogId;
+ private String catalogName;
private String catalogType;
private String catalogImpl;
private Map<String, String> catalogProperties;
@@ -278,7 +278,7 @@ public class RunCatalogSync {
@Data
public static class TargetTableIdentifier {
- String catalogId;
+ String catalogName;
String tableFormat;
CatalogTableIdentifier catalogTableIdentifier;
}
diff --git
a/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunCatalogSync.java
b/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunCatalogSync.java
index 7e40975a..a30a7720 100644
---
a/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunCatalogSync.java
+++
b/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunCatalogSync.java
@@ -58,7 +58,7 @@ class TestRunCatalogSync {
}
@Override
- public String getCatalogId() {
+ public String getCatalogName() {
return null;
}
diff --git
a/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunSync.java
b/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunSync.java
index f6ff1820..a61d948e 100644
---
a/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunSync.java
+++
b/xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunSync.java
@@ -135,7 +135,7 @@ class TestRunSync {
+ " option2: value2";
IcebergCatalogConfig catalogConfig =
RunSync.loadIcebergCatalogConfig(icebergConfig.getBytes());
Assertions.assertEquals("org.apache.xtable.CatalogImpl",
catalogConfig.getCatalogImpl());
- Assertions.assertEquals("test", catalogConfig.getCatalogId());
+ Assertions.assertEquals("test", catalogConfig.getCatalogName());
Assertions.assertEquals(2, catalogConfig.getCatalogOptions().size());
Assertions.assertEquals("value1",
catalogConfig.getCatalogOptions().get("option1"));
Assertions.assertEquals("value2",
catalogConfig.getCatalogOptions().get("option2"));
diff --git a/xtable-utilities/src/test/resources/catalogConfig.yaml
b/xtable-utilities/src/test/resources/catalogConfig.yaml
index b59af884..6a4df0f0 100644
--- a/xtable-utilities/src/test/resources/catalogConfig.yaml
+++ b/xtable-utilities/src/test/resources/catalogConfig.yaml
@@ -15,26 +15,26 @@
# limitations under the License.
#
sourceCatalog:
- catalogId: "source-1"
+ catalogName: "source-1"
catalogImpl: "org.apache.xtable.utilities.TestRunCatalogSync$TestCatalogImpl"
catalogProperties:
key01: "value1"
key02: "value2"
key03: "value3"
targetCatalogs:
- - catalogId: "target-1"
+ - catalogName: "target-1"
catalogImpl:
"org.apache.xtable.utilities.TestRunCatalogSync$TestCatalogImpl"
catalogProperties:
key11: "value1"
key12: "value2"
key13: "value3"
- - catalogId: "target-2"
+ - catalogName: "target-2"
catalogImpl:
"org.apache.xtable.utilities.TestRunCatalogSync$TestCatalogImpl"
catalogProperties:
key21: "value1"
key22: "value2"
key23: "value3"
- - catalogId: "target-3"
+ - catalogName: "target-3"
catalogImpl:
"org.apache.xtable.utilities.TestRunCatalogSync$TestCatalogImpl"
catalogProperties:
key31: "value1"
@@ -46,12 +46,12 @@ datasets:
databaseName: "source-database-1"
tableName: "source-1"
targetCatalogTableIdentifiers:
- - catalogId: "target-1"
+ - catalogName: "target-1"
tableFormat: "DELTA"
catalogTableIdentifier:
databaseName: "target-database-1"
tableName: "target-tableName-1"
- - catalogId: "target-2"
+ - catalogName: "target-2"
tableFormat: "HUDI"
catalogTableIdentifier:
databaseName: "target-database-2"
@@ -63,12 +63,12 @@ datasets:
partitionSpec: cs_sold_date_sk:VALUE
tableFormat: "HUDI"
targetCatalogTableIdentifiers:
- - catalogId: "target-2"
+ - catalogName: "target-2"
tableFormat: "ICEBERG"
catalogTableIdentifier:
databaseName: "target-database-2"
tableName: "target-tableName-2"
- - catalogId: "target-3"
+ - catalogName: "target-3"
tableFormat: "HUDI"
catalogTableIdentifier:
databaseName: "target-database-3"
