This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6a4dc73 [HUDI-1218] Introduce BulkInsertSortMode as Independent class
(#2021)
6a4dc73 is described below
commit 6a4dc7384c8f7efbbce729b8e57b3eaaf5cab104
Author: Trevor <[email protected]>
AuthorDate: Tue Aug 25 19:04:13 2020 +0800
[HUDI-1218] Introduce BulkInsertSortMode as Independent class (#2021)
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 2 +-
.../BulkInsertInternalPartitionerFactory.java | 8 +-------
...itionerFactory.java => BulkInsertSortMode.java} | 24 ++--------------------
.../TestBulkInsertInternalPartitioner.java | 8 ++++----
4 files changed, 8 insertions(+), 34 deletions(-)
diff --git
a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 12026a9..249b107 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -27,7 +27,7 @@ import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ReflectionUtils;
-import
org.apache.hudi.execution.bulkinsert.BulkInsertInternalPartitionerFactory.BulkInsertSortMode;
+import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
diff --git
a/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
b/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
index ef4ffb6..aaa7b5b 100644
---
a/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
+++
b/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
@@ -36,13 +36,7 @@ public abstract class BulkInsertInternalPartitionerFactory {
case PARTITION_SORT:
return new RDDPartitionSortPartitioner();
default:
- throw new HoodieException("The bulk insert mode \"" + sortMode.name()
+ "\" is not supported.");
+ throw new HoodieException("The bulk insert sort mode \"" +
sortMode.name() + "\" is not supported.");
}
}
-
- public enum BulkInsertSortMode {
- NONE,
- GLOBAL_SORT,
- PARTITION_SORT
- }
}
diff --git
a/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
b/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
similarity index 53%
copy from
hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
copy to
hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
index ef4ffb6..d171b8c 100644
---
a/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
+++
b/hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
@@ -18,31 +18,11 @@
package org.apache.hudi.execution.bulkinsert;
-import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.table.BulkInsertPartitioner;
-
/**
- * A factory to generate built-in partitioner to repartition input records
into at least
- * expected number of output spark partitions for bulk insert operation.
+ * Bulk insert sort mode.
*/
-public abstract class BulkInsertInternalPartitionerFactory {
-
- public static BulkInsertPartitioner get(BulkInsertSortMode sortMode) {
- switch (sortMode) {
- case NONE:
- return new NonSortPartitioner();
- case GLOBAL_SORT:
- return new GlobalSortPartitioner();
- case PARTITION_SORT:
- return new RDDPartitionSortPartitioner();
- default:
- throw new HoodieException("The bulk insert mode \"" + sortMode.name()
+ "\" is not supported.");
- }
- }
-
- public enum BulkInsertSortMode {
+public enum BulkInsertSortMode {
NONE,
GLOBAL_SORT,
PARTITION_SORT
- }
}
diff --git
a/hudi-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
b/hudi-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
index 3b9df5f..b8ebd5a 100644
---
a/hudi-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
+++
b/hudi-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
@@ -62,9 +62,9 @@ public class TestBulkInsertInternalPartitioner extends
HoodieClientTestBase {
private static Stream<Arguments> configParams() {
Object[][] data = new Object[][] {
- {BulkInsertInternalPartitionerFactory.BulkInsertSortMode.GLOBAL_SORT,
true, true},
-
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.PARTITION_SORT, false,
true},
- {BulkInsertInternalPartitionerFactory.BulkInsertSortMode.NONE, false,
false}
+ {BulkInsertSortMode.GLOBAL_SORT, true, true},
+ {BulkInsertSortMode.PARTITION_SORT, false, true},
+ {BulkInsertSortMode.NONE, false, false}
};
return Stream.of(data).map(Arguments::of);
}
@@ -108,7 +108,7 @@ public class TestBulkInsertInternalPartitioner extends
HoodieClientTestBase {
@ParameterizedTest(name = "[{index}] {0}")
@MethodSource("configParams")
- public void
testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.BulkInsertSortMode
sortMode,
+ public void testBulkInsertInternalPartitioner(BulkInsertSortMode sortMode,
boolean isGloballySorted,
boolean isLocallySorted)
throws Exception {
JavaRDD<HoodieRecord> records1 = generateTestRecordsForBulkInsert(jsc);
