This is an automated email from the ASF dual-hosted git repository.
timbrown pushed a commit to branch 297-properties-based-config-2
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/297-properties-based-config-2
by this push:
new 437cc155 lint
437cc155 is described below
commit 437cc155a466934e13730ae7e7d1cd729404222a
Author: Timothy Brown <[email protected]>
AuthorDate: Fri Aug 2 14:41:10 2024 -0400
lint
---
.../apache/xtable/conversion/ConversionConfig.java | 4 +--
.../apache/xtable/conversion/ExternalTable.java | 32 ++++++----------------
.../org/apache/xtable/conversion/SourceTable.java | 8 ++----
.../xtable/conversion/TestExternalTable.java | 3 +-
.../delta/DeltaConversionSourceProvider.java | 2 --
.../xtable/hudi/HudiConversionSourceProvider.java | 6 ++--
.../apache/xtable/hudi/HudiConversionTarget.java | 6 ++--
.../org/apache/xtable/hudi/HudiSourceConfig.java | 1 -
.../org/apache/xtable/ITConversionController.java | 2 +-
.../delta/ITDeltaConversionTargetSource.java | 3 +-
.../java/org/apache/xtable/loadtest/LoadTest.java | 2 +-
.../apache/xtable/hudi/sync/XTableSyncTool.java | 7 ++++-
.../java/org/apache/xtable/utilities/RunSync.java | 3 +-
13 files changed, 31 insertions(+), 48 deletions(-)
diff --git
a/xtable-api/src/main/java/org/apache/xtable/conversion/ConversionConfig.java
b/xtable-api/src/main/java/org/apache/xtable/conversion/ConversionConfig.java
index d21e256a..73e2628d 100644
---
a/xtable-api/src/main/java/org/apache/xtable/conversion/ConversionConfig.java
+++
b/xtable-api/src/main/java/org/apache/xtable/conversion/ConversionConfig.java
@@ -39,9 +39,7 @@ public class ConversionConfig {
@Builder
ConversionConfig(
- @NonNull SourceTable sourceTable,
- List<TargetTable> targetTables,
- SyncMode syncMode) {
+ @NonNull SourceTable sourceTable, List<TargetTable> targetTables,
SyncMode syncMode) {
this.sourceTable = sourceTable;
this.targetTables = targetTables;
Preconditions.checkArgument(
diff --git
a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java
b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java
index f6580c4b..939c59c0 100644
--- a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java
+++ b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java
@@ -18,6 +18,8 @@
package org.apache.xtable.conversion;
+import java.util.Properties;
+
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
@@ -26,38 +28,22 @@ import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
-import java.util.Properties;
-
-/**
- * Defines a reference to a table in a particular format.
- */
+/** Defines a reference to a table in a particular format. */
@Getter
@EqualsAndHashCode
class ExternalTable {
- /**
- * The name of the table.
- */
+ /** The name of the table. */
protected final @NonNull String name;
- /**
- * The format of the table (e.g. DELTA, ICEBERG, HUDI)
- */
+ /** The format of the table (e.g. DELTA, ICEBERG, HUDI) */
protected final @NonNull String formatName;
- /**
- * The path to the root of the table or the metadata directory depending on
the format
- */
+ /** The path to the root of the table or the metadata directory depending on
the format */
protected final @NonNull String basePath;
- /**
- * Optional namespace for the table
- */
+ /** Optional namespace for the table */
protected final String[] namespace;
- /**
- * The configuration for interacting with the catalog that manages this table
- */
+ /** The configuration for interacting with the catalog that manages this
table */
protected final CatalogConfig catalogConfig;
- /**
- * Optional, additional properties that can be used to define interactions
with the table
- */
+ /** Optional, additional properties that can be used to define interactions
with the table */
protected final Properties additionalProperties;
ExternalTable(
diff --git
a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java
b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java
index 37224476..b37e1c1e 100644
--- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java
+++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java
@@ -18,19 +18,17 @@
package org.apache.xtable.conversion;
+import java.util.Properties;
+
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
-import java.util.Properties;
-
@EqualsAndHashCode(callSuper = true)
@Getter
public class SourceTable extends ExternalTable {
- /**
- * The path to the data files, defaults to the metadataPath
- */
+ /** The path to the data files, defaults to the metadataPath */
@NonNull private final String dataPath;
@Builder(toBuilder = true)
diff --git
a/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java
b/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java
index 6bc42da4..5422b0a7 100644
---
a/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java
+++
b/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java
@@ -42,7 +42,8 @@ public class TestExternalTable {
@Test
void errorIfRequiredArgsNotSet() {
assertThrows(
- NullPointerException.class, () -> new ExternalTable("name", "hudi",
null, null, null, null));
+ NullPointerException.class,
+ () -> new ExternalTable("name", "hudi", null, null, null, null));
assertThrows(
NullPointerException.class,
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSourceProvider.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSourceProvider.java
index 2ba79a88..045e2b72 100644
---
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSourceProvider.java
+++
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSourceProvider.java
@@ -18,8 +18,6 @@
package org.apache.xtable.delta;
-import java.util.Map;
-
import org.apache.spark.sql.SparkSession;
import io.delta.tables.DeltaTable;
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSourceProvider.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSourceProvider.java
index 51c0afb9..0ddbbcb7 100644
---
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSourceProvider.java
+++
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSourceProvider.java
@@ -32,8 +32,7 @@ import org.apache.xtable.conversion.SourceTable;
public class HudiConversionSourceProvider extends
ConversionSourceProvider<HoodieInstant> {
@Override
- public HudiConversionSource getConversionSourceInstance(
- SourceTable sourceTable) {
+ public HudiConversionSource getConversionSourceInstance(SourceTable
sourceTable) {
HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder()
.setConf(hadoopConf)
@@ -45,7 +44,8 @@ public class HudiConversionSourceProvider extends
ConversionSourceProvider<Hoodi
}
final HudiSourcePartitionSpecExtractor sourcePartitionSpecExtractor =
-
HudiSourceConfig.fromProperties(sourceTable.getAdditionalProperties()).loadSourcePartitionSpecExtractor();
+ HudiSourceConfig.fromProperties(sourceTable.getAdditionalProperties())
+ .loadSourcePartitionSpecExtractor();
return new HudiConversionSource(metaClient, sourcePartitionSpecExtractor);
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionTarget.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionTarget.java
index c1852ea8..c3ef6f92 100644
--- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionTarget.java
+++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionTarget.java
@@ -116,8 +116,7 @@ public class HudiConversionTarget implements
ConversionTarget {
(int) targetTable.getMetadataRetention().toHours(),
maxNumDeltaCommitsBeforeCompaction,
BaseFileUpdatesExtractor.of(
- new HoodieJavaEngineContext(configuration),
- new CachingPath(targetTable.getBasePath())),
+ new HoodieJavaEngineContext(configuration), new
CachingPath(targetTable.getBasePath())),
AvroSchemaConverter.getInstance(),
HudiTableManager.of(configuration),
CommitState::new);
@@ -169,8 +168,7 @@ public class HudiConversionTarget implements
ConversionTarget {
(int) targetTable.getMetadataRetention().toHours(),
HoodieMetadataConfig.COMPACT_NUM_DELTA_COMMITS.defaultValue(),
BaseFileUpdatesExtractor.of(
- new HoodieJavaEngineContext(configuration),
- new CachingPath(targetTable.getBasePath())),
+ new HoodieJavaEngineContext(configuration), new
CachingPath(targetTable.getBasePath())),
AvroSchemaConverter.getInstance(),
HudiTableManager.of(configuration),
CommitState::new);
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java
index 88717d1e..606b9281 100644
--- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java
+++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.Properties;
import lombok.Value;
diff --git
a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java
b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java
index e9e35686..f6aa215a 100644
--- a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java
+++ b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java
@@ -58,7 +58,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
-import org.apache.xtable.conversion.ConversionConfig;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
@@ -85,6 +84,7 @@ import org.apache.spark.sql.delta.DeltaLog;
import com.google.common.collect.ImmutableList;
+import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.ConversionController;
import org.apache.xtable.conversion.ConversionSourceProvider;
import org.apache.xtable.conversion.SourceTable;
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionTargetSource.java
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionTargetSource.java
index 21152787..8fcf0753 100644
---
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionTargetSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionTargetSource.java
@@ -415,8 +415,7 @@ public class ITDeltaConversionTargetSource {
InstantsForIncrementalSync.builder()
.lastSyncInstant(Instant.ofEpochMilli(timestamp1))
.build();
- conversionSource =
- conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ conversionSource =
conversionSourceProvider.getConversionSourceInstance(tableConfig);
CommitsBacklog<Long> instantCurrentCommitState =
conversionSource.getCommitsBacklog(instantsForIncrementalSync);
boolean areFilesRemoved = false;
diff --git a/xtable-core/src/test/java/org/apache/xtable/loadtest/LoadTest.java
b/xtable-core/src/test/java/org/apache/xtable/loadtest/LoadTest.java
index be6c2a37..341b2cb0 100644
--- a/xtable-core/src/test/java/org/apache/xtable/loadtest/LoadTest.java
+++ b/xtable-core/src/test/java/org/apache/xtable/loadtest/LoadTest.java
@@ -39,10 +39,10 @@ import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.xtable.GenericTable;
import org.apache.xtable.TestJavaHudiTable;
+import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.ConversionController;
import org.apache.xtable.conversion.ConversionSourceProvider;
import org.apache.xtable.conversion.SourceTable;
-import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.TargetTable;
import org.apache.xtable.hudi.HudiConversionSourceProvider;
import org.apache.xtable.model.storage.TableFormat;
diff --git
a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java
b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java
index 969b05b5..a9653eb5 100644
---
a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java
+++
b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java
@@ -72,7 +72,12 @@ public class XTableSyncTool extends HoodieSyncTool {
Properties sourceProperties = new Properties();
sourceProperties.put(PARTITION_FIELD_SPEC_CONFIG,
getPartitionSpecConfig());
SourceTable sourceTable =
-
SourceTable.builder().name(tableName).formatName(HUDI).basePath(basePath).additionalProperties(sourceProperties).build();
+ SourceTable.builder()
+ .name(tableName)
+ .formatName(HUDI)
+ .basePath(basePath)
+ .additionalProperties(sourceProperties)
+ .build();
Duration metadataRetention =
config.contains(XTableSyncConfig.XTABLE_TARGET_METADATA_RETENTION_HOURS)
? Duration.ofHours(
diff --git
a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
index e0aeb98e..7a27ad87 100644
--- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
+++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
@@ -152,7 +152,8 @@ public class RunSync {
tableFormatList);
Properties sourceProperties = new Properties();
if (table.getPartitionSpec() != null) {
- sourceProperties.put(HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG,
table.getPartitionSpec());
+ sourceProperties.put(
+ HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG,
table.getPartitionSpec());
}
SourceTable sourceTable =
SourceTable.builder()
