This is an automated email from the ASF dual-hosted git repository.
vinish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new aa31ae84 Add additional properties when doing conversion for hudi
(#714)
aa31ae84 is described below
commit aa31ae8492f3651470afec78eb713cd410f2bf0b
Author: Rahil C <32500120+rahi...@users.noreply.github.com>
AuthorDate: Fri May 30 11:31:00 2025 -0700
Add additional properties when doing conversion for hudi (#714)
* Add additional properties when doing conversion for hudi
* fix other issue with hudi partitiong when iceberg is source, and fix UT
* spotless apply
---
.../org/apache/xtable/service/ConversionService.java | 20 ++++++++++++++++++--
.../xtable/service/models/ConvertTableRequest.java | 5 +++++
.../apache/xtable/service/TestConversionService.java | 10 +++++++---
3 files changed, 30 insertions(+), 5 deletions(-)
diff --git
a/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java
b/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java
index 1d4ad32e..7d19512a 100644
---
a/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java
+++
b/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java
@@ -19,6 +19,7 @@
package org.apache.xtable.service;
import static
org.apache.xtable.conversion.ConversionUtils.convertToSourceTable;
+import static
org.apache.xtable.hudi.HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG;
import static org.apache.xtable.model.storage.TableFormat.DELTA;
import static org.apache.xtable.model.storage.TableFormat.HUDI;
import static org.apache.xtable.model.storage.TableFormat.ICEBERG;
@@ -27,6 +28,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import lombok.extern.log4j.Log4j2;
@@ -186,11 +188,23 @@ public class ConversionService {
* @return a ConvertTableResponse containing details of the converted target
tables
*/
public ConvertTableResponse convertTable(ConvertTableRequest
convertTableRequest) {
+
+ Properties sourceProperties = new Properties();
+ if (convertTableRequest.getConfigurations() != null) {
+ String partitionSpec =
+
convertTableRequest.getConfigurations().getOrDefault("partition-spec", null);
+ if (partitionSpec != null) {
+ sourceProperties.put(PARTITION_FIELD_SPEC_CONFIG, partitionSpec);
+ }
+ }
+
SourceTable sourceTable =
SourceTable.builder()
.name(convertTableRequest.getSourceTableName())
.basePath(convertTableRequest.getSourceTablePath())
+ .dataPath(convertTableRequest.getSourceDataPath())
.formatName(convertTableRequest.getSourceFormat())
+ .additionalProperties(sourceProperties)
.build();
List<TargetTable> targetTables = new ArrayList<>();
@@ -198,8 +212,10 @@ public class ConversionService {
TargetTable targetTable =
TargetTable.builder()
.name(convertTableRequest.getSourceTableName())
- .basePath(convertTableRequest.getSourceTablePath())
+ // set the metadata path to the data path as the default
(required by Hudi)
+ .basePath(convertTableRequest.getSourceDataPath())
.formatName(targetFormat)
+ .additionalProperties(sourceProperties)
.build();
targetTables.add(targetTable);
}
@@ -220,7 +236,7 @@ public class ConversionService {
String schemaString = extractSchemaString(targetTable, internalTable);
convertedTables.add(
ConvertedTable.builder()
- .targetFormat(internalTable.getName())
+ .targetFormat(internalTable.getTableFormat())
.targetSchema(schemaString)
.targetMetadataPath(internalTable.getLatestMetdataPath())
.build());
diff --git
a/xtable-service/src/main/java/org/apache/xtable/service/models/ConvertTableRequest.java
b/xtable-service/src/main/java/org/apache/xtable/service/models/ConvertTableRequest.java
index 465c3c0c..1e151300 100644
---
a/xtable-service/src/main/java/org/apache/xtable/service/models/ConvertTableRequest.java
+++
b/xtable-service/src/main/java/org/apache/xtable/service/models/ConvertTableRequest.java
@@ -39,6 +39,9 @@ public class ConvertTableRequest {
@JsonProperty("source-table-path")
private String sourceTablePath;
+ @JsonProperty("source-data-path")
+ private String sourceDataPath;
+
@JsonProperty("target-formats")
private List<String> targetFormats;
@@ -52,12 +55,14 @@ public class ConvertTableRequest {
@JsonProperty("source-format") String sourceFormat,
@JsonProperty("source-table-name") String sourceTableName,
@JsonProperty("source-table-path") String sourceTablePath,
+ @JsonProperty("source-data-path") String sourceDataPath,
@JsonProperty("target-format") List<String> targetFormat,
@JsonProperty("configurations") Map<String, String> configurations) {
this.sourceFormat = sourceFormat;
this.sourceTableName = sourceTableName;
this.sourceTablePath = sourceTablePath;
+ this.sourceDataPath = sourceDataPath;
this.targetFormats = targetFormat;
this.configurations = configurations;
}
diff --git
a/xtable-service/src/test/java/org/apache/xtable/service/TestConversionService.java
b/xtable-service/src/test/java/org/apache/xtable/service/TestConversionService.java
index d22b561c..465b1c4c 100644
---
a/xtable-service/src/test/java/org/apache/xtable/service/TestConversionService.java
+++
b/xtable-service/src/test/java/org/apache/xtable/service/TestConversionService.java
@@ -59,6 +59,7 @@ import org.apache.xtable.spi.extractor.ConversionSource;
class TestConversionService {
private static final String SOURCE_NAME = "users";
private static final String SOURCE_PATH = "s3://bucket/tables/users";
+ private static final String SOURCE_DATA_PATH =
"s3://bucket/tables/users/data";
private static final String HUDI_META_PATH =
"s3://bucket/tables/users/.hoodie";
private static final String ICEBERG_META_PATH =
"s3://bucket/tables/users/metadata/v1.metadata.json";
@@ -111,6 +112,7 @@ class TestConversionService {
.sourceFormat(TableFormat.DELTA)
.sourceTableName(SOURCE_NAME)
.sourceTablePath(SOURCE_PATH)
+ .sourceDataPath(SOURCE_DATA_PATH)
.targetFormats(Collections.singletonList(TableFormat.HUDI))
.build();
@@ -120,7 +122,7 @@ class TestConversionService {
when(provider.getConversionSourceInstance(any())).thenReturn(conversionSrc);
when(conversionSrc.getCurrentTable()).thenReturn(internalTbl);
- when(internalTbl.getName()).thenReturn(TableFormat.HUDI);
+ when(internalTbl.getTableFormat()).thenReturn(TableFormat.HUDI);
when(internalTbl.getLatestMetdataPath()).thenReturn(HUDI_META_PATH);
when(internalTbl.getReadSchema()).thenReturn(internalSchema);
@@ -146,6 +148,7 @@ class TestConversionService {
.sourceFormat(TableFormat.DELTA)
.sourceTableName(SOURCE_NAME)
.sourceTablePath(SOURCE_PATH)
+ .sourceDataPath(SOURCE_DATA_PATH)
.targetFormats(Collections.singletonList(TableFormat.ICEBERG))
.build();
@@ -157,7 +160,7 @@ class TestConversionService {
when(provider.getConversionSourceInstance(any())).thenReturn(conversionSrc);
when(conversionSrc.getCurrentTable()).thenReturn(internalTbl);
- when(internalTbl.getName()).thenReturn(TableFormat.ICEBERG);
+ when(internalTbl.getTableFormat()).thenReturn(TableFormat.ICEBERG);
when(internalTbl.getLatestMetdataPath()).thenReturn(ICEBERG_META_PATH);
when(internalTbl.getReadSchema()).thenReturn(internalSchema);
@@ -185,6 +188,7 @@ class TestConversionService {
.sourceFormat(TableFormat.ICEBERG)
.sourceTableName(SOURCE_NAME)
.sourceTablePath(SOURCE_PATH)
+ .sourceDataPath(SOURCE_DATA_PATH)
.targetFormats(Collections.singletonList(TableFormat.DELTA))
.build();
@@ -194,7 +198,7 @@ class TestConversionService {
when(provider.getConversionSourceInstance(any())).thenReturn(conversionSrc);
when(conversionSrc.getCurrentTable()).thenReturn(internalTbl);
- when(internalTbl.getName()).thenReturn(TableFormat.DELTA);
+ when(internalTbl.getTableFormat()).thenReturn(TableFormat.DELTA);
when(internalTbl.getLatestMetdataPath()).thenReturn(DELTA_META_PATH);
when(internalTbl.getReadSchema()).thenReturn(internalSchema);
