This is an automated email from the ASF dual-hosted git repository.
smarthi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8c30013 HUDI-479: Eliminate or Minimize use of Guava if possible
(#1159)
8c30013 is described below
commit 8c3001363d80b29733470221c192a72f541381c5
Author: Suneel Marthi <[email protected]>
AuthorDate: Sat Mar 28 03:11:32 2020 -0400
HUDI-479: Eliminate or Minimize use of Guava if possible (#1159)
---
.../apache/hudi/cli/commands/RollbacksCommand.java | 4 +-
.../common/HoodieTestCommitMetadataGenerator.java | 20 ++--
.../org/apache/hudi/client/HoodieWriteClient.java | 6 +-
.../hudi/index/bloom/BloomIndexFileInfo.java | 5 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +-
.../apache/hudi/metrics/JmxMetricsReporter.java | 10 +-
.../org/apache/hudi/metrics/JmxReporterServer.java | 18 ++--
.../main/java/org/apache/hudi/metrics/Metrics.java | 5 +-
.../compact/HoodieMergeOnReadTableCompactor.java | 4 +-
.../apache/hudi/table/rollback/RollbackHelper.java | 2 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 12 +--
.../java/org/apache/hudi/table/TestCleaner.java | 51 +++++-----
.../strategy/TestHoodieCompactionStrategy.java | 37 +++++--
.../apache/hudi/avro/MercifulJsonConverter.java | 25 +++--
.../org/apache/hudi/common/model/HoodieRecord.java | 7 +-
.../hudi/common/table/HoodieTableMetaClient.java | 6 +-
.../table/timeline/HoodieActiveTimeline.java | 11 +-
.../table/timeline/HoodieArchivedTimeline.java | 1 +
.../table/timeline/HoodieDefaultTimeline.java | 10 +-
.../hudi/common/table/timeline/HoodieInstant.java | 6 +-
.../IncrementalTimelineSyncFileSystemView.java | 2 +-
.../view/RemoteHoodieTableFileSystemView.java | 2 +-
.../org/apache/hudi/common/util/AvroUtils.java | 18 ++--
.../org/apache/hudi/common/util/CleanerUtils.java | 11 +-
.../apache/hudi/common/util/CollectionUtils.java | 111 +++++++++++++++++++++
.../java/org/apache/hudi/common/util/FSUtils.java | 4 +-
.../org/apache/hudi/common/util/FileIOUtils.java | 25 +++++
.../apache/hudi/common/util/ReflectionUtils.java | 69 +++++++++++--
.../hudi/common/minicluster/HdfsTestService.java | 10 +-
.../common/minicluster/ZookeeperTestService.java | 6 +-
.../common/model/TestHoodieCommitMetadata.java | 1 +
.../table/string/TestHoodieActiveTimeline.java | 12 +--
.../table/view/TestIncrementalFSViewSync.java | 22 ++--
.../view/TestPriorityBasedFileSystemView.java | 5 +-
.../hudi/common/util/CompactionTestUtils.java | 15 +--
.../hudi/common/util/TestCompactionUtils.java | 10 +-
.../org/apache/hudi/common/util/TestFSUtils.java | 2 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 4 +-
.../org/apache/hudi/hive/util/HiveTestService.java | 9 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 40 ++++----
.../org/apache/hudi/HoodieDataSourceHelpers.java | 5 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 4 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 2 +-
.../apache/hudi/utilities/UtilitiesTestBase.java | 4 +-
pom.xml | 7 --
style/checkstyle.xml | 4 +-
46 files changed, 429 insertions(+), 217 deletions(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java
index 4a122c6..3993714 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java
@@ -28,9 +28,9 @@ import
org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.AvroUtils;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.collect.ImmutableSet;
import org.springframework.shell.core.CommandMarker;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
@@ -123,7 +123,7 @@ public class RollbacksCommand implements CommandMarker {
class RollbackTimeline extends HoodieActiveTimeline {
public RollbackTimeline(HoodieTableMetaClient metaClient) {
- super(metaClient,
ImmutableSet.<String>builder().add(HoodieTimeline.ROLLBACK_EXTENSION).build());
+ super(metaClient,
CollectionUtils.createImmutableSet(HoodieTimeline.ROLLBACK_EXTENSION));
}
}
}
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java
index 659b15b..27454ef 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java
@@ -18,8 +18,6 @@
package org.apache.hudi.cli.common;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -30,12 +28,14 @@ import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.exception.HoodieIOException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -91,14 +91,14 @@ public class HoodieTestCommitMetadataGenerator extends
HoodieTestDataGenerator {
* Generate commitMetadata in path.
*/
public static HoodieCommitMetadata generateCommitMetadata(String basePath)
throws IOException {
- String file1P0C0 =
- HoodieTestUtils.createNewDataFile(basePath,
DEFAULT_FIRST_PARTITION_PATH, "000");
- String file1P1C0 =
- HoodieTestUtils.createNewDataFile(basePath,
DEFAULT_SECOND_PARTITION_PATH, "000");
- return generateCommitMetadata(new ImmutableMap.Builder()
- .put(DEFAULT_FIRST_PARTITION_PATH, new
ImmutableList.Builder<>().add(file1P0C0).build())
- .put(DEFAULT_SECOND_PARTITION_PATH, new
ImmutableList.Builder<>().add(file1P1C0).build())
- .build());
+ String file1P0C0 = HoodieTestUtils.createNewDataFile(basePath,
DEFAULT_FIRST_PARTITION_PATH, "000");
+ String file1P1C0 = HoodieTestUtils.createNewDataFile(basePath,
DEFAULT_SECOND_PARTITION_PATH, "000");
+ return generateCommitMetadata(new HashMap<String, List<String>>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P0C0));
+ put(DEFAULT_SECOND_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P1C0));
+ }
+ });
}
/**
diff --git
a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index 5f269a8..d654bc8 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -64,7 +64,6 @@ import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
import com.codahale.metrics.Timer;
-import com.google.common.collect.ImmutableMap;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.Partitioner;
@@ -77,6 +76,7 @@ import org.apache.spark.storage.StorageLevel;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -746,7 +746,7 @@ public class HoodieWriteClient<T extends
HoodieRecordPayload> extends AbstractHo
String startRollbackInstant = HoodieActiveTimeline.createNewInstantTime();
// Start the timer
final Timer.Context context = startContext();
- ImmutableMap.Builder<String, List<HoodieRollbackStat>> instantsToStats =
ImmutableMap.builder();
+ Map<String, List<HoodieRollbackStat>> instantsToStats = new HashMap<>();
table.getActiveTimeline().createNewInstant(
new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION,
startRollbackInstant));
instantsToRollback.forEach(instant -> {
@@ -773,7 +773,7 @@ public class HoodieWriteClient<T extends
HoodieRecordPayload> extends AbstractHo
}
});
try {
- finishRestore(context, instantsToStats.build(),
+ finishRestore(context, Collections.unmodifiableMap(instantsToStats),
instantsToRollback.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()),
startRollbackInstant, instantTime);
} catch (IOException io) {
diff --git
a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java
b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java
index ebd20dc..11ffb78 100644
---
a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java
+++
b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java
@@ -64,9 +64,8 @@ public class BloomIndexFileInfo implements Serializable {
* Does the given key fall within the range (inclusive).
*/
public boolean isKeyInRange(String recordKey) {
- assert minRecordKey != null;
- assert maxRecordKey != null;
- return minRecordKey.compareTo(recordKey) <= 0 &&
maxRecordKey.compareTo(recordKey) >= 0;
+ return Objects.requireNonNull(minRecordKey).compareTo(recordKey) <= 0
+ && Objects.requireNonNull(maxRecordKey).compareTo(recordKey) >= 0;
}
@Override
diff --git
a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index c5be9da..f1bd57c 100644
--- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -54,8 +54,8 @@ import org.apache.spark.util.SizeEstimator;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
diff --git
a/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
index c7c596c..e66738d 100644
--- a/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
+++ b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
@@ -18,19 +18,19 @@
package org.apache.hudi.metrics;
-import com.google.common.base.Preconditions;
-import java.lang.management.ManagementFactory;
-import javax.management.MBeanServer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-
import org.apache.log4j.LogManager;
import com.codahale.metrics.MetricRegistry;
import java.io.Closeable;
+import java.util.Objects;
import java.util.stream.IntStream;
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+
/**
* Implementation of Jmx reporter, which used to report jmx metric.
*/
@@ -92,7 +92,7 @@ public class JmxMetricsReporter extends MetricsReporter {
@Override
public void stop() {
- Preconditions.checkNotNull(jmxReporterServer, "jmxReporterServer is not
running.");
+ Objects.requireNonNull(jmxReporterServer, "jmxReporterServer is not
running.");
try {
jmxReporterServer.stop();
} catch (Exception e) {
diff --git
a/hudi-client/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java
index e055af6..5bb8e8f 100644
--- a/hudi-client/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java
+++ b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java
@@ -18,20 +18,23 @@
package org.apache.hudi.metrics;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.exception.HoodieException;
+
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.jmx.JmxReporter;
-import com.google.common.base.Preconditions;
+
import java.io.IOException;
import java.rmi.NoSuchObjectException;
import java.rmi.registry.LocateRegistry;
import java.rmi.registry.Registry;
import java.rmi.server.UnicastRemoteObject;
+import java.util.Objects;
import javax.management.MBeanServer;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXConnectorServerFactory;
import javax.management.remote.JMXServiceURL;
-import org.apache.hudi.common.util.StringUtils;
-import org.apache.hudi.exception.HoodieException;
/**
* A reporter which publishes metric values to a JMX server.
@@ -78,10 +81,9 @@ public class JmxReporterServer {
}
public JmxReporterServer build() {
- Preconditions.checkNotNull(registry, "registry cannot be null!");
- Preconditions.checkNotNull(mBeanServer, "mBeanServer cannot be null!");
- Preconditions
- .checkArgument(!StringUtils.isNullOrEmpty(host), "host cannot be
null or empty!");
+ Objects.requireNonNull(registry, "registry cannot be null!");
+ Objects.requireNonNull(mBeanServer, "mBeanServer cannot be null!");
+ ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(host), "host
cannot be null or empty!");
return new JmxReporterServer(registry, host, port, mBeanServer);
}
}
@@ -110,7 +112,7 @@ public class JmxReporterServer {
}
public void start() {
- Preconditions.checkArgument(reporter != null && connector != null,
+ ValidationUtils.checkArgument(reporter != null && connector != null,
"reporter or connector cannot be null!");
try {
connector.start();
diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java
index 2538133..b6d2f7a 100644
--- a/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java
+++ b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java
@@ -18,12 +18,12 @@
package org.apache.hudi.metrics;
+import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
-import com.google.common.io.Closeables;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -53,8 +53,7 @@ public class Metrics {
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
reporter.report();
- reporter.stop();
- Closeables.close(reporter.getReporter(), true);
+ FileIOUtils.close(reporter.getReporter(), true);
} catch (Exception e) {
e.printStackTrace();
}
diff --git
a/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
b/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
index 4b09242..355f485 100644
---
a/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
+++
b/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
@@ -32,6 +32,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.TableFileSystemView.SliceView;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.HoodieAvroUtils;
@@ -43,7 +44,6 @@ import
org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.HoodieCopyOnWriteTable;
import org.apache.hudi.table.HoodieTable;
-import com.google.common.collect.Sets;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -113,7 +113,7 @@ public class HoodieMergeOnReadTableCompactor implements
HoodieCompactor {
// loaded and load it using CompositeAvroLogReader
// Since a DeltaCommit is not defined yet, reading all the records.
revisit this soon.
String maxInstantTime = metaClient
-
.getActiveTimeline().getTimelineOfActions(Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
+
.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.ROLLBACK_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION))
.filterCompletedInstants().lastInstant().get().getTimestamp();
LOG.info("MaxMemoryPerCompaction => " +
SparkConfigUtils.getMaxMemoryPerCompaction(config.getProps()));
diff --git
a/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
b/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
index 901c51e..120c00b 100644
---
a/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
+++
b/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
@@ -214,7 +214,7 @@ public class RollbackHelper implements Serializable {
private Map<HeaderMetadataType, String> generateHeader(String commit) {
// generate metadata
- Map<HeaderMetadataType, String> header = new HashMap<>();
+ Map<HeaderMetadataType, String> header = new HashMap<>(3);
header.put(HeaderMetadataType.INSTANT_TIME,
metaClient.getActiveTimeline().lastInstant().get().getTimestamp());
header.put(HeaderMetadataType.TARGET_INSTANT_TIME, commit);
header.put(HeaderMetadataType.COMMAND_BLOCK_TYPE,
diff --git
a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
index ddf2775..2e2121e 100644
---
a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
+++
b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
@@ -249,17 +249,13 @@ public class TestHoodieGlobalBloomIndex extends
HoodieClientTestHarness {
JavaRDD<HoodieRecord> recordRDD = jsc.parallelize(Arrays.asList(record1,
record2, record3, record5));
String filename0 =
- HoodieClientTestUtils.writeParquetFile(basePath, "2016/04/01",
Collections.singletonList(record1),
- schema, null, false);
+ HoodieClientTestUtils.writeParquetFile(basePath, "2016/04/01",
Collections.singletonList(record1), schema, null, false);
String filename1 =
- HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12", new
ArrayList<>(),
- schema, null, false);
+ HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12", new
ArrayList<>(), schema, null, false);
String filename2 =
- HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12",
Collections.singletonList(record2),
- schema, null, false);
+ HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12",
Collections.singletonList(record2), schema, null, false);
String filename3 =
- HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12",
Collections.singletonList(record4),
- schema, null, false);
+ HoodieClientTestUtils.writeParquetFile(basePath, "2015/03/12",
Collections.singletonList(record4), schema, null, false);
// intentionally missed the partition "2015/03/12" to see if the
GlobalBloomIndex can pick it up
metaClient = HoodieTableMetaClient.reload(metaClient);
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index 1af19f5..8502872 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -45,6 +45,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CleanerUtils;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.ConsistencyGuardConfig;
import org.apache.hudi.common.util.FSUtils;
@@ -55,9 +56,6 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.log4j.LogManager;
@@ -290,7 +288,7 @@ public class TestCleaner extends TestHoodieClientBase {
List<String> commitedVersions = new
ArrayList<>(fileIdToVersions.get(fileId));
for (int i = 0; i < dataFiles.size(); i++) {
assertEquals("File " + fileId + " does not have latest
versions on commits" + commitedVersions,
- Iterables.get(dataFiles, i).getCommitTime(),
+ (dataFiles.get(i)).getCommitTime(),
commitedVersions.get(commitedVersions.size() - 1 - i));
}
}
@@ -740,12 +738,14 @@ public class TestCleaner extends TestHoodieClientBase {
String file1P1C0 =
HoodieTestUtils.createNewDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000");
- HoodieCommitMetadata commitMetadata = generateCommitMetadata(new
ImmutableMap.Builder()
- .put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- new ImmutableList.Builder<>().add(file1P0C0).build())
- .put(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH,
- new ImmutableList.Builder<>().add(file1P1C0).build())
- .build());
+ HoodieCommitMetadata commitMetadata = generateCommitMetadata(
+ Collections.unmodifiableMap(new HashMap<String, List<String>>() {
+ {
+ put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P0C0));
+ put(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P1C0));
+ }
+ })
+ );
metaClient.getActiveTimeline().saveAsComplete(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "000"),
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
@@ -778,12 +778,12 @@ public class TestCleaner extends TestHoodieClientBase {
.createDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0); //
update
HoodieTestUtils
.createDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001", file1P1C0); //
update
- commitMetadata = generateCommitMetadata(new ImmutableMap.Builder()
- .put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- new
ImmutableList.Builder<>().add(file1P0C0).add(file2P0C1).build())
- .put(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH,
- new
ImmutableList.Builder<>().add(file1P1C0).add(file2P1C1).build())
- .build());
+ commitMetadata = generateCommitMetadata(new HashMap<String,
List<String>>() {
+ {
+ put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P0C0, file2P0C1));
+ put(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P1C0, file2P1C1));
+ }
+ });
metaClient.getActiveTimeline().saveAsComplete(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "001"),
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
@@ -814,10 +814,9 @@ public class TestCleaner extends TestHoodieClientBase {
String file3P0C2 =
HoodieTestUtils.createNewDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002");
- commitMetadata = generateCommitMetadata(new ImmutableMap.Builder()
- .put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- new
ImmutableList.Builder<>().add(file1P0C0).add(file2P0C1).add(file3P0C2).build())
- .build());
+ commitMetadata = generateCommitMetadata(CollectionUtils
+
.createImmutableMap(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
+ CollectionUtils.createImmutableList(file1P0C0, file2P0C1,
file3P0C2)));
metaClient.getActiveTimeline().saveAsComplete(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "002"),
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
@@ -840,10 +839,8 @@ public class TestCleaner extends TestHoodieClientBase {
.createDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file2P0C1); //
update
String file4P0C3 =
HoodieTestUtils.createNewDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003");
- commitMetadata = generateCommitMetadata(new ImmutableMap.Builder()
- .put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- new
ImmutableList.Builder<>().add(file1P0C0).add(file2P0C1).add(file4P0C3).build())
- .build());
+ commitMetadata = generateCommitMetadata(CollectionUtils.createImmutableMap(
+ HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
CollectionUtils.createImmutableList(file1P0C0, file2P0C1, file4P0C3)));
metaClient.getActiveTimeline().saveAsComplete(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "003"),
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
@@ -871,10 +868,8 @@ public class TestCleaner extends TestHoodieClientBase {
// No cleaning on partially written file, with no commit.
HoodieTestUtils
.createDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "004", file3P0C2); //
update
- commitMetadata = generateCommitMetadata(new ImmutableMap.Builder()
- .put(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- new ImmutableList.Builder<>().add(file3P0C2).build())
- .build());
+ commitMetadata =
generateCommitMetadata(CollectionUtils.createImmutableMap(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
+ CollectionUtils.createImmutableList(file3P0C2)));
metaClient.getActiveTimeline().createNewInstant(
new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION,
"004"));
metaClient.getActiveTimeline().transitionRequestedToInflight(
diff --git
a/hudi-client/src/test/java/org/apache/hudi/table/compact/strategy/TestHoodieCompactionStrategy.java
b/hudi-client/src/test/java/org/apache/hudi/table/compact/strategy/TestHoodieCompactionStrategy.java
index 1d386e5..6491148 100644
---
a/hudi-client/src/test/java/org/apache/hudi/table/compact/strategy/TestHoodieCompactionStrategy.java
+++
b/hudi-client/src/test/java/org/apache/hudi/table/compact/strategy/TestHoodieCompactionStrategy.java
@@ -27,7 +27,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
-import com.google.common.collect.ImmutableMap;
import org.junit.Assert;
import org.junit.Test;
@@ -122,8 +121,14 @@ public class TestHoodieCompactionStrategy {
sizesMap.put(100 * MB, Collections.singletonList(MB));
sizesMap.put(90 * MB, Collections.singletonList(1024 * MB));
- Map<Long, String> keyToPartitionMap = new ImmutableMap.Builder().put(120 *
MB, partitionPaths[2])
- .put(110 * MB, partitionPaths[2]).put(100 * MB,
partitionPaths[1]).put(90 * MB, partitionPaths[0]).build();
+ Map<Long, String> keyToPartitionMap = Collections.unmodifiableMap(new
HashMap<Long,String>() {
+ {
+ put(120 * MB, partitionPaths[2]);
+ put(110 * MB, partitionPaths[2]);
+ put(100 * MB, partitionPaths[1]);
+ put(90 * MB, partitionPaths[0]);
+ }
+ });
DayBasedCompactionStrategy strategy = new DayBasedCompactionStrategy();
HoodieWriteConfig writeConfig =
@@ -162,9 +167,16 @@ public class TestHoodieCompactionStrategy {
String currentDayPlus1 =
format.format(BoundedPartitionAwareCompactionStrategy.getDateAtOffsetFromToday(1));
String currentDayPlus5 =
format.format(BoundedPartitionAwareCompactionStrategy.getDateAtOffsetFromToday(5));
- Map<Long, String> keyToPartitionMap = new ImmutableMap.Builder().put(120 *
MB, currentDay)
- .put(110 * MB, currentDayMinus1).put(100 * MB,
currentDayMinus2).put(80 * MB, currentDayMinus3)
- .put(90 * MB, currentDayPlus1).put(70 * MB, currentDayPlus5).build();
+ Map<Long, String> keyToPartitionMap = Collections.unmodifiableMap(new
HashMap<Long,String>() {
+ {
+ put(120 * MB, currentDay);
+ put(110 * MB, currentDayMinus1);
+ put(100 * MB, currentDayMinus2);
+ put(80 * MB, currentDayMinus3);
+ put(90 * MB, currentDayPlus1);
+ put(70 * MB, currentDayPlus5);
+ }
+ });
BoundedPartitionAwareCompactionStrategy strategy = new
BoundedPartitionAwareCompactionStrategy();
HoodieWriteConfig writeConfig =
@@ -204,9 +216,16 @@ public class TestHoodieCompactionStrategy {
String currentDayPlus1 =
format.format(BoundedPartitionAwareCompactionStrategy.getDateAtOffsetFromToday(1));
String currentDayPlus5 =
format.format(BoundedPartitionAwareCompactionStrategy.getDateAtOffsetFromToday(5));
- Map<Long, String> keyToPartitionMap = new ImmutableMap.Builder().put(120 *
MB, currentDay)
- .put(110 * MB, currentDayMinus1).put(100 * MB,
currentDayMinus2).put(80 * MB, currentDayMinus3)
- .put(90 * MB, currentDayPlus1).put(70 * MB, currentDayPlus5).build();
+ Map<Long, String> keyToPartitionMap = Collections.unmodifiableMap(new
HashMap<Long,String>() {
+ {
+ put(120 * MB, currentDay);
+ put(110 * MB, currentDayMinus1);
+ put(100 * MB, currentDayMinus2);
+ put(80 * MB, currentDayMinus3);
+ put(90 * MB, currentDayPlus1);
+ put(70 * MB, currentDayPlus5);
+ }
+ });
UnBoundedPartitionAwareCompactionStrategy strategy = new
UnBoundedPartitionAwareCompactionStrategy();
HoodieWriteConfig writeConfig =
diff --git
a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
index 3f5df01..734c631 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
@@ -23,7 +23,6 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericData;
@@ -32,6 +31,7 @@ import org.apache.avro.generic.GenericRecord;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -49,13 +49,22 @@ public class MercifulJsonConverter {
* Build type processor map for each avro type.
*/
private static Map<Schema.Type, JsonToAvroFieldProcessor>
getFieldTypeProcessors() {
- return new ImmutableMap.Builder<Type,
JsonToAvroFieldProcessor>().put(Type.STRING, generateStringTypeHandler())
- .put(Type.BOOLEAN, generateBooleanTypeHandler()).put(Type.DOUBLE,
generateDoubleTypeHandler())
- .put(Type.FLOAT, generateFloatTypeHandler()).put(Type.INT,
generateIntTypeHandler())
- .put(Type.LONG, generateLongTypeHandler()).put(Type.ARRAY,
generateArrayTypeHandler())
- .put(Type.RECORD, generateRecordTypeHandler()).put(Type.ENUM,
generateEnumTypeHandler())
- .put(Type.MAP, generateMapTypeHandler()).put(Type.BYTES,
generateBytesTypeHandler())
- .put(Type.FIXED, generateFixedTypeHandler()).build();
+ return Collections.unmodifiableMap(new HashMap<Schema.Type,
JsonToAvroFieldProcessor>() {
+ {
+ put(Type.STRING, generateStringTypeHandler());
+ put(Type.BOOLEAN, generateBooleanTypeHandler());
+ put(Type.DOUBLE, generateDoubleTypeHandler());
+ put(Type.FLOAT, generateFloatTypeHandler());
+ put(Type.INT, generateIntTypeHandler());
+ put(Type.LONG, generateLongTypeHandler());
+ put(Type.ARRAY, generateArrayTypeHandler());
+ put(Type.RECORD, generateRecordTypeHandler());
+ put(Type.ENUM, generateEnumTypeHandler());
+ put(Type.MAP, generateMapTypeHandler());
+ put(Type.BYTES, generateBytesTypeHandler());
+ put(Type.FIXED, generateFixedTypeHandler());
+ }
+ });
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
index c0b41ae..57f5fa7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
@@ -18,10 +18,9 @@
package org.apache.hudi.common.model;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
-import com.google.common.collect.ImmutableList;
-
import java.io.Serializable;
import java.util.List;
import java.util.Objects;
@@ -38,8 +37,8 @@ public class HoodieRecord<T extends HoodieRecordPayload>
implements Serializable
public static String FILENAME_METADATA_FIELD = "_hoodie_file_name";
public static final List<String> HOODIE_META_COLUMNS =
- new
ImmutableList.Builder<String>().add(COMMIT_TIME_METADATA_FIELD).add(COMMIT_SEQNO_METADATA_FIELD)
-
.add(RECORD_KEY_METADATA_FIELD).add(PARTITION_PATH_METADATA_FIELD).add(FILENAME_METADATA_FIELD).build();
+ CollectionUtils.createImmutableList(COMMIT_TIME_METADATA_FIELD,
COMMIT_SEQNO_METADATA_FIELD,
+ RECORD_KEY_METADATA_FIELD, PARTITION_PATH_METADATA_FIELD,
FILENAME_METADATA_FIELD);
/**
* Identifies the record across the table.
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 5c56d17..2bff9b7 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -26,13 +26,13 @@ import
org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.ConsistencyGuardConfig;
-import org.apache.hudi.common.util.FailSafeConsistencyGuard;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.FailSafeConsistencyGuard;
import org.apache.hudi.common.util.NoOpConsistencyGuard;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
-import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -246,7 +246,7 @@ public class HoodieTableMetaClient implements Serializable {
/**
* Return raw file-system.
*
- * @return
+ * @return fs
*/
public FileSystem getRawFs() {
return getFs().getFileSystem();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 389314f..837c4a1 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -26,7 +26,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -38,8 +37,10 @@ import java.io.IOException;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
@@ -106,13 +107,11 @@ public class HoodieActiveTimeline extends
HoodieDefaultTimeline {
}
public HoodieActiveTimeline(HoodieTableMetaClient metaClient) {
- this(metaClient, new
ImmutableSet.Builder<String>().addAll(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE).build());
+ this(metaClient,
Collections.unmodifiableSet(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE));
}
public HoodieActiveTimeline(HoodieTableMetaClient metaClient, boolean
applyLayoutFilter) {
- this(metaClient,
- new ImmutableSet.Builder<String>()
- .addAll(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE).build(),
applyLayoutFilter);
+ this(metaClient,
Collections.unmodifiableSet(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE),
applyLayoutFilter);
}
/**
@@ -166,7 +165,7 @@ public class HoodieActiveTimeline extends
HoodieDefaultTimeline {
public void deleteCompactionRequested(HoodieInstant instant) {
ValidationUtils.checkArgument(instant.isRequested());
-
ValidationUtils.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(Objects.equals(instant.getAction(),
HoodieTimeline.COMPACTION_ACTION));
deleteInstantFile(instant);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
index d544628..76d7422 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
+
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
index b3d9ae5..bc21c29 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
@@ -20,11 +20,11 @@ package org.apache.hudi.common.table.timeline;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.collect.Sets;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -113,7 +113,7 @@ public class HoodieDefaultTimeline implements
HoodieTimeline {
@Override
public HoodieTimeline getCommitsAndCompactionTimeline() {
- Set<String> validActions = Sets.newHashSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION, COMPACTION_ACTION);
+ Set<String> validActions = CollectionUtils.createSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION, COMPACTION_ACTION);
return new HoodieDefaultTimeline(instants.stream().filter(s ->
validActions.contains(s.getAction())), details);
}
@@ -145,7 +145,7 @@ public class HoodieDefaultTimeline implements
HoodieTimeline {
* Get all instants (commits, delta commits) that produce new data, in the
active timeline.
*/
public HoodieTimeline getCommitsTimeline() {
- return getTimelineOfActions(Sets.newHashSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION));
+ return getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION));
}
/**
@@ -153,8 +153,8 @@ public class HoodieDefaultTimeline implements
HoodieTimeline {
* timeline.
*/
public HoodieTimeline getAllCommitsTimeline() {
- return getTimelineOfActions(Sets.newHashSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION, CLEAN_ACTION, COMPACTION_ACTION,
- SAVEPOINT_ACTION, ROLLBACK_ACTION));
+ return getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION,
DELTA_COMMIT_ACTION,
+ CLEAN_ACTION, COMPACTION_ACTION, SAVEPOINT_ACTION,
ROLLBACK_ACTION));
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java
index 460931b..e98e5bf 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java
@@ -19,9 +19,9 @@
package org.apache.hudi.common.table.timeline;
import org.apache.hudi.common.table.HoodieTimeline;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FSUtils;
-import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.FileStatus;
import java.io.Serializable;
@@ -41,8 +41,8 @@ public class HoodieInstant implements Serializable,
Comparable<HoodieInstant> {
* A COMPACTION action eventually becomes COMMIT when completed. So, when
grouping instants
* for state transitions, this needs to be taken into account
*/
- private static final Map<String, String> COMPARABLE_ACTIONS = new
ImmutableMap.Builder<String, String>()
- .put(HoodieTimeline.COMPACTION_ACTION,
HoodieTimeline.COMMIT_ACTION).build();
+ private static final Map<String, String> COMPARABLE_ACTIONS =
+ CollectionUtils.createImmutableMap(HoodieTimeline.COMPACTION_ACTION,
HoodieTimeline.COMMIT_ACTION);
public static final Comparator<HoodieInstant> ACTION_COMPARATOR =
Comparator.comparing(instant ->
getComparableAction(instant.getAction()));
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
index 07b262d..3ce40bf 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
@@ -24,8 +24,8 @@ import org.apache.hudi.avro.model.HoodieRestoreMetadata;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
-import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.table.HoodieTimeline;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
index 01010a2..256fbd4 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
@@ -26,8 +26,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.SyncableFileSystemView;
import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO;
import org.apache.hudi.common.table.timeline.dto.BaseFileDTO;
+import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO;
import org.apache.hudi.common.table.timeline.dto.FileGroupDTO;
import org.apache.hudi.common.table.timeline.dto.FileSliceDTO;
import org.apache.hudi.common.table.timeline.dto.InstantDTO;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
index 683a4dd..3cf6bc8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
@@ -28,7 +28,6 @@ import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.avro.model.HoodieSavepointPartitionMetadata;
import org.apache.hudi.common.HoodieRollbackStat;
-import com.google.common.collect.ImmutableMap;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.file.FileReader;
@@ -42,6 +41,7 @@ import org.apache.avro.specific.SpecificRecordBase;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,18 +54,18 @@ public class AvroUtils {
public static HoodieRestoreMetadata convertRestoreMetadata(String
startRestoreTime, Option<Long> durationInMs,
List<String> commits, Map<String, List<HoodieRollbackStat>>
commitToStats) {
- ImmutableMap.Builder<String, List<HoodieRollbackMetadata>>
commitToStatBuilder = ImmutableMap.builder();
+ Map<String, List<HoodieRollbackMetadata>> commitToStatsMap = new
HashMap<>();
for (Map.Entry<String, List<HoodieRollbackStat>> commitToStat :
commitToStats.entrySet()) {
- commitToStatBuilder.put(commitToStat.getKey(),
+ commitToStatsMap.put(commitToStat.getKey(),
Collections.singletonList(convertRollbackMetadata(startRestoreTime,
durationInMs, commits, commitToStat.getValue())));
}
return new HoodieRestoreMetadata(startRestoreTime,
durationInMs.orElseGet(() -> -1L), commits,
- commitToStatBuilder.build(), DEFAULT_VERSION);
+ Collections.unmodifiableMap(commitToStatsMap), DEFAULT_VERSION);
}
public static HoodieRollbackMetadata convertRollbackMetadata(String
startRollbackTime, Option<Long> durationInMs,
List<String> commits, List<HoodieRollbackStat> rollbackStats) {
- ImmutableMap.Builder<String, HoodieRollbackPartitionMetadata>
partitionMetadataBuilder = ImmutableMap.builder();
+ Map<String, HoodieRollbackPartitionMetadata> partitionMetadataBuilder =
new HashMap<>();
int totalDeleted = 0;
for (HoodieRollbackStat stat : rollbackStats) {
HoodieRollbackPartitionMetadata metadata = new
HoodieRollbackPartitionMetadata(stat.getPartitionPath(),
@@ -75,18 +75,18 @@ public class AvroUtils {
}
return new HoodieRollbackMetadata(startRollbackTime,
durationInMs.orElseGet(() -> -1L), totalDeleted, commits,
- partitionMetadataBuilder.build(), DEFAULT_VERSION);
+ Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION);
}
public static HoodieSavepointMetadata convertSavepointMetadata(String user,
String comment,
Map<String, List<String>> latestFiles) {
- ImmutableMap.Builder<String, HoodieSavepointPartitionMetadata>
partitionMetadataBuilder = ImmutableMap.builder();
+ Map<String, HoodieSavepointPartitionMetadata> partitionMetadataBuilder =
new HashMap<>();
for (Map.Entry<String, List<String>> stat : latestFiles.entrySet()) {
HoodieSavepointPartitionMetadata metadata = new
HoodieSavepointPartitionMetadata(stat.getKey(), stat.getValue());
partitionMetadataBuilder.put(stat.getKey(), metadata);
}
- return new HoodieSavepointMetadata(user, System.currentTimeMillis(),
comment, partitionMetadataBuilder.build(),
- DEFAULT_VERSION);
+ return new HoodieSavepointMetadata(user, System.currentTimeMillis(),
comment,
+ Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION);
}
public static Option<byte[]> serializeCompactionPlan(HoodieCompactionPlan
compactionWorkload) throws IOException {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
index 1d32b64..263a34d 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
@@ -28,10 +28,10 @@ import
org.apache.hudi.common.versioning.clean.CleanMetadataMigrator;
import org.apache.hudi.common.versioning.clean.CleanV1MigrationHandler;
import org.apache.hudi.common.versioning.clean.CleanV2MigrationHandler;
-import com.google.common.collect.ImmutableMap;
-
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class CleanerUtils {
public static final Integer CLEAN_METADATA_VERSION_1 =
CleanV1MigrationHandler.VERSION;
@@ -40,14 +40,14 @@ public class CleanerUtils {
public static HoodieCleanMetadata convertCleanMetadata(HoodieTableMetaClient
metaClient,
String startCleanTime, Option<Long> durationInMs, List<HoodieCleanStat>
cleanStats) {
- ImmutableMap.Builder<String, HoodieCleanPartitionMetadata>
partitionMetadataBuilder = ImmutableMap.builder();
+ Map<String, HoodieCleanPartitionMetadata> partitionMetadataMap = new
HashMap<>();
int totalDeleted = 0;
String earliestCommitToRetain = null;
for (HoodieCleanStat stat : cleanStats) {
HoodieCleanPartitionMetadata metadata =
new HoodieCleanPartitionMetadata(stat.getPartitionPath(),
stat.getPolicy().name(),
stat.getDeletePathPatterns(), stat.getSuccessDeleteFiles(),
stat.getFailedDeleteFiles());
- partitionMetadataBuilder.put(stat.getPartitionPath(), metadata);
+ partitionMetadataMap.put(stat.getPartitionPath(), metadata);
totalDeleted += stat.getSuccessDeleteFiles().size();
if (earliestCommitToRetain == null) {
// This will be the same for all partitions
@@ -56,8 +56,7 @@ public class CleanerUtils {
}
return new HoodieCleanMetadata(startCleanTime,
- durationInMs.orElseGet(() -> -1L), totalDeleted,
earliestCommitToRetain,
- partitionMetadataBuilder.build(), CLEAN_METADATA_VERSION_2);
+ durationInMs.orElseGet(() -> -1L), totalDeleted,
earliestCommitToRetain, partitionMetadataMap, CLEAN_METADATA_VERSION_2);
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
new file mode 100644
index 0000000..0fb238e
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util;
+
+import org.apache.hudi.common.util.collection.Pair;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CollectionUtils {
+ /**
+ * Determines whether two iterators contain equal elements in the same
order. More specifically,
+ * this method returns {@code true} if {@code iterator1} and {@code
iterator2} contain the same
+ * number of elements and every element of {@code iterator1} is equal to the
corresponding element
+ * of {@code iterator2}.
+ *
+ * <p>Note that this will modify the supplied iterators, since they will
have been advanced some
+ * number of elements forward.
+ */
+ public static boolean elementsEqual(Iterator<?> iterator1, Iterator<?>
iterator2) {
+ while (iterator1.hasNext()) {
+ if (!iterator2.hasNext()) {
+ return false;
+ }
+ Object o1 = iterator1.next();
+ Object o2 = iterator2.next();
+ if (!Objects.equals(o1, o2)) {
+ return false;
+ }
+ }
+ return !iterator2.hasNext();
+ }
+
+ @SafeVarargs
+ public static <T> Set<T> createSet(final T... elements) {
+ return Stream.of(elements).collect(Collectors.toSet());
+ }
+
+ public static <K,V> Map<K, V> createImmutableMap(final K key, final V value)
{
+ return Collections.unmodifiableMap(Collections.singletonMap(key, value));
+ }
+
+ @SafeVarargs
+ public static <T> List<T> createImmutableList(final T... elements) {
+ return
Collections.unmodifiableList(Stream.of(elements).collect(Collectors.toList()));
+ }
+
+ public static <K,V> Map<K,V> createImmutableMap(final Map<K,V> map) {
+ return Collections.unmodifiableMap(map);
+ }
+
+ @SafeVarargs
+ public static <K,V> Map<K,V> createImmutableMap(final Pair<K,V>... elements)
{
+ Map<K,V> map = new HashMap<>();
+ for (Pair<K,V> pair: elements) {
+ map.put(pair.getLeft(), pair.getRight());
+ }
+ return Collections.unmodifiableMap(map);
+ }
+
+ @SafeVarargs
+ public static <T> Set<T> createImmutableSet(final T... elements) {
+ return Collections.unmodifiableSet(createSet(elements));
+ }
+
+ public static <T> Set<T> createImmutableSet(final Set<T> set) {
+ return Collections.unmodifiableSet(set);
+ }
+
+ public static <T> List<T> createImmutableList(final List<T> list) {
+ return Collections.unmodifiableList(list);
+ }
+
+ private static Object[] checkElementsNotNull(Object... array) {
+ return checkElementsNotNull(array, array.length);
+ }
+
+ private static Object[] checkElementsNotNull(Object[] array, int length) {
+ for (int i = 0; i < length; i++) {
+ checkElementNotNull(array[i], i);
+ }
+ return array;
+ }
+
+ private static Object checkElementNotNull(Object element, int index) {
+ return Objects.requireNonNull(element, "Element is null at index " +
index);
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
index 5eff951..7d207e2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
@@ -46,8 +46,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
-import java.util.Objects;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.UUID;
import java.util.function.Function;
import java.util.regex.Matcher;
@@ -213,7 +213,7 @@ public class FSUtils {
* @param basePathStr Base-Path
* @param consumer Callback for processing
* @param excludeMetaFolder Exclude .hoodie folder
- * @throws IOException
+ * @throws IOException -
*/
static void processFiles(FileSystem fs, String basePathStr,
Function<FileStatus, Boolean> consumer,
boolean excludeMetaFolder) throws IOException {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java
index f1095b6..65a28b0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java
@@ -18,7 +18,10 @@
package org.apache.hudi.common.util;
+import javax.annotation.Nullable;
+
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -91,4 +94,26 @@ public class FileIOUtils {
out.flush();
out.close();
}
+
+ /**
+ * Closes a {@link Closeable}, with control over whether an {@code
IOException} may be thrown.
+ * @param closeable the {@code Closeable} object to be closed, or null,
+ * in which case this method does nothing.
+ * @param swallowIOException if true, don't propagate IO exceptions thrown
by the {@code close} methods.
+ *
+ * @throws IOException if {@code swallowIOException} is false and {@code
close} throws an {@code IOException}.
+ */
+ public static void close(@Nullable Closeable closeable, boolean
swallowIOException)
+ throws IOException {
+ if (closeable == null) {
+ return;
+ }
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ if (!swallowIOException) {
+ throw e;
+ }
+ }
+ }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java
index 0a69d68..23a87e7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java
@@ -21,14 +21,21 @@ package org.apache.hudi.common.util;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.reflect.ClassPath;
-import com.google.common.reflect.ClassPath.ClassInfo;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Enumeration;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Stream;
/**
@@ -36,6 +43,8 @@ import java.util.stream.Stream;
*/
public class ReflectionUtils {
+ private static final Logger LOG =
LogManager.getLogger(ReflectionUtils.class);
+
private static Map<String, Class<?>> clazzCache = new HashMap<>();
private static Class<?> getClass(String clazzName) {
@@ -90,16 +99,58 @@ public class ReflectionUtils {
}
/**
- * Return stream of top level class names in the same class path as
passed-in class.
- *
- * @param clazz
+ * Scans all classes accessible from the context class loader
+ * which belong to the given package and subpackages.
+ *
+ * @param clazz class
+ * @return Stream of Class names in package
*/
- public static Stream<String> getTopLevelClassesInClasspath(Class clazz) {
+ public static Stream<String> getTopLevelClassesInClasspath(Class<?> clazz) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ String packageName = clazz.getPackage().getName();
+ String path = packageName.replace('.', '/');
+ Enumeration<URL> resources = null;
try {
- ClassPath classPath = ClassPath.from(clazz.getClassLoader());
- return classPath.getTopLevelClasses().stream().map(ClassInfo::getName);
+ resources = classLoader.getResources(path);
} catch (IOException e) {
- throw new RuntimeException("Got exception while dumping top level
classes", e);
+ LOG.error("Unable to fetch Resources in package " + e.getMessage());
+ }
+ List<File> directories = new ArrayList<>();
+ while (Objects.requireNonNull(resources).hasMoreElements()) {
+ URL resource = resources.nextElement();
+ try {
+ directories.add(new File(resource.toURI()));
+ } catch (URISyntaxException e) {
+ LOG.error("Unable to get " + e.getMessage());
+ }
+ }
+ List<String> classes = new ArrayList<>();
+ for (File directory : directories) {
+ classes.addAll(findClasses(directory, packageName));
+ }
+ return classes.stream();
+ }
+
+ /**
+ * Recursive method used to find all classes in a given directory and
subdirs.
+ *
+ * @param directory The base directory
+ * @param packageName The package name for classes found inside the base
directory
+ * @return classes in the package
+ */
+ private static List<String> findClasses(File directory, String packageName) {
+ List<String> classes = new ArrayList<>();
+ if (!directory.exists()) {
+ return classes;
+ }
+ File[] files = directory.listFiles();
+ for (File file : Objects.requireNonNull(files)) {
+ if (file.isDirectory()) {
+ classes.addAll(findClasses(file, packageName + "." + file.getName()));
+ } else if (file.getName().endsWith(".class")) {
+ classes.add(packageName + '.' + file.getName().substring(0,
file.getName().length() - 6));
+ }
}
+ return classes;
}
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
index ed4750a..38e6218 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
@@ -20,20 +20,20 @@ package org.apache.hudi.common.minicluster;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
+import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.IOException;
-import java.util.Objects;
import java.net.ServerSocket;
+import java.nio.file.Files;
+import java.util.Objects;
/**
* An HDFS minicluster service implementation.
@@ -53,8 +53,8 @@ public class HdfsTestService {
*/
private MiniDFSCluster miniDfsCluster;
- public HdfsTestService() {
- workDir = Files.createTempDir().getAbsolutePath();
+ public HdfsTestService() throws IOException {
+ workDir = Files.createTempDirectory("temp").getName(0).toString();
}
public Configuration getHadoopConf() {
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
index 514e298..5c837dd 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
@@ -18,7 +18,6 @@
package org.apache.hudi.common.minicluster;
-import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.log4j.LogManager;
@@ -35,6 +34,7 @@ import java.io.OutputStream;
import java.io.Reader;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.nio.file.Files;
import java.util.Objects;
/**
@@ -75,8 +75,8 @@ public class ZookeeperTestService {
private ZooKeeperServer zooKeeperServer;
private boolean started = false;
- public ZookeeperTestService(Configuration config) {
- this.workDir = Files.createTempDir().getAbsolutePath();
+ public ZookeeperTestService(Configuration config) throws IOException {
+ this.workDir = Files.createTempDirectory(System.currentTimeMillis() +
"-").toFile().getAbsolutePath();
this.hadoopConf = config;
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
index e6b395b..c845f33 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.model;
import org.apache.hudi.common.util.FileIOUtils;
+
import org.junit.Assert;
import org.junit.Test;
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
index 8a41f8c..7a9dc74 100755
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hadoop.fs.Path;
@@ -34,7 +35,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import com.google.common.collect.Sets;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -195,9 +195,9 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
// Test that various types of getXXX operations from HoodieActiveTimeline
// return the correct set of Instant
checkTimeline.accept(timeline.getCommitsTimeline(),
- Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION));
+ CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION));
checkTimeline.accept(timeline.getCommitsAndCompactionTimeline(),
- Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION, HoodieTimeline.COMPACTION_ACTION));
+ CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION, HoodieTimeline.COMPACTION_ACTION));
checkTimeline.accept(timeline.getCommitTimeline(),
Collections.singleton(HoodieTimeline.COMMIT_ACTION));
checkTimeline.accept(timeline.getDeltaCommitTimeline(),
Collections.singleton(HoodieTimeline.DELTA_COMMIT_ACTION));
checkTimeline.accept(timeline.getCleanerTimeline(),
Collections.singleton(HoodieTimeline.CLEAN_ACTION));
@@ -205,7 +205,7 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
checkTimeline.accept(timeline.getRestoreTimeline(),
Collections.singleton(HoodieTimeline.RESTORE_ACTION));
checkTimeline.accept(timeline.getSavePointTimeline(),
Collections.singleton(HoodieTimeline.SAVEPOINT_ACTION));
checkTimeline.accept(timeline.getAllCommitsTimeline(),
- Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION,
+ CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.CLEAN_ACTION,
HoodieTimeline.COMPACTION_ACTION,
HoodieTimeline.SAVEPOINT_ACTION,
HoodieTimeline.ROLLBACK_ACTION));
@@ -380,12 +380,12 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
checkFilter.accept(timeline.filter(i -> false), new HashSet<>());
checkFilter.accept(timeline.filterInflights(),
Collections.singleton(State.INFLIGHT));
checkFilter.accept(timeline.filterInflightsAndRequested(),
- Sets.newHashSet(State.INFLIGHT, State.REQUESTED));
+ CollectionUtils.createSet(State.INFLIGHT, State.REQUESTED));
// filterCompletedAndCompactionInstants
// This cannot be done using checkFilter as it involves both states and
actions
final HoodieTimeline t1 = timeline.filterCompletedAndCompactionInstants();
- final Set<State> states = Sets.newHashSet(State.REQUESTED,
State.COMPLETED);
+ final Set<State> states = CollectionUtils.createSet(State.REQUESTED,
State.COMPLETED);
final Set<String> actions =
Collections.singleton(HoodieTimeline.COMPACTION_ACTION);
sup.get().filter(i -> states.contains(i.getState()) ||
actions.contains(i.getAction()))
.forEach(i -> assertTrue(t1.containsInstant(i)));
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
index c00c6a1..08f3174 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
@@ -27,9 +27,9 @@ import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieTableType;
@@ -41,6 +41,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CleanerUtils;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
@@ -48,8 +49,6 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterators;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -135,8 +134,11 @@ public class TestIncrementalFSViewSync extends
HoodieCommonTestHarness {
// Clean first slice
testCleans(view, Collections.singletonList("21"),
- new ImmutableMap.Builder<String, List<String>>().put("11",
Arrays.asList("12", "13", "15")).build(),
- instantsToFiles, Collections.singletonList("11"));
+ new HashMap<String, List<String>>() {
+ {
+ put("11", Arrays.asList("12", "13", "15"));
+ }
+ }, instantsToFiles, Collections.singletonList("11"));
// Add one more ingestion instant. This should be 2nd slice now
instantsToFiles.putAll(testMultipleWriteSteps(view,
Collections.singletonList("22"), true, "19", 2));
@@ -251,7 +253,11 @@ public class TestIncrementalFSViewSync extends
HoodieCommonTestHarness {
* Case where a clean happened and then rounds of ingestion and compaction
happened
*/
testCleans(view2, Collections.singletonList("19"),
- new ImmutableMap.Builder<String, List<String>>().put("11",
Arrays.asList("12", "13", "14")).build(),
+ new HashMap<String, List<String>>() {
+ {
+ put("11", Arrays.asList("12", "13", "14"));
+ }
+ },
instantsToFiles, Collections.singletonList("11"));
scheduleCompaction(view2, "20");
instantsToFiles.putAll(testMultipleWriteSteps(view2, Arrays.asList("21",
"22"), true, "20", 2));
@@ -439,7 +445,7 @@ public class TestIncrementalFSViewSync extends
HoodieCommonTestHarness {
List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
rollbackM.add(rollbackMetadata);
- metadata.setHoodieRestoreMetadata(new
ImmutableMap.Builder().put(rollbackInstant, rollbackM).build());
+
metadata.setHoodieRestoreMetadata(CollectionUtils.createImmutableMap(rollbackInstant,
rollbackM));
List<String> rollbackInstants = new ArrayList<>();
rollbackInstants.add(rollbackInstant);
metadata.setInstantsToRollback(rollbackInstants);
@@ -646,7 +652,7 @@ public class TestIncrementalFSViewSync extends
HoodieCommonTestHarness {
HoodieTimeline timeline1 = view1.getTimeline();
HoodieTimeline timeline2 = view2.getTimeline();
Assert.assertEquals(view1.getLastInstant(), view2.getLastInstant());
- Iterators.elementsEqual(timeline1.getInstants().iterator(),
timeline2.getInstants().iterator());
+ CollectionUtils.elementsEqual(timeline1.getInstants().iterator(),
timeline2.getInstants().iterator());
// View Checks
Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap1 =
partitions.stream().flatMap(view1::getAllFileGroups)
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
index 79e6a12..d94177e 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
@@ -18,7 +18,6 @@
package org.apache.hudi.common.table.view;
-import junit.framework.TestCase;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -30,6 +29,8 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
+
+import junit.framework.TestCase;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -45,10 +46,10 @@ import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
-import static org.mockito.Mockito.when;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class TestPriorityBasedFileSystemView extends TestCase {
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
index 46783ef..228e21d 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
@@ -31,8 +31,6 @@ import
org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
@@ -82,12 +80,17 @@ public class CompactionTestUtils {
createDeltaCommit(metaClient, "004");
createDeltaCommit(metaClient, "006");
- Map<String, String> baseInstantsToCompaction = new
ImmutableMap.Builder<String, String>().put("000", "001")
- .put("002", "003").put("004", "005").put("006", "007").build();
+ Map<String, String> baseInstantsToCompaction = new HashMap<String,
String>() {
+ {
+ put("000", "001");
+ put("002", "003");
+ put("004", "005");
+ put("006", "007");
+ }
+ };
List<Integer> expectedNumEntries =
Arrays.asList(numEntriesInPlan1, numEntriesInPlan2, numEntriesInPlan3,
numEntriesInPlan4);
- List<HoodieCompactionPlan> plans =
- new ImmutableList.Builder<HoodieCompactionPlan>().add(plan1, plan2,
plan3, plan4).build();
+ List<HoodieCompactionPlan> plans =
CollectionUtils.createImmutableList(plan1, plan2, plan3, plan4);
IntStream.range(0, 4).boxed().forEach(idx -> {
if (expectedNumEntries.get(idx) > 0) {
Assert.assertEquals("check if plan " + idx + " has exp entries",
expectedNumEntries.get(idx).longValue(),
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
index 034ac3c..eb6b73c 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
@@ -31,7 +31,6 @@ import
org.apache.hudi.common.util.CompactionTestUtils.TestHoodieBaseFile;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.versioning.compaction.CompactionPlanMigrator;
-import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Before;
@@ -39,6 +38,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@@ -59,8 +59,12 @@ public class TestCompactionUtils extends
HoodieCommonTestHarness {
private static String TEST_WRITE_TOKEN = "1-0-1";
- private static final Map<String, Double> METRICS =
- new ImmutableMap.Builder<String, Double>().put("key1", 1.0).put("key2",
3.0).build();
+ private static final Map<String, Double> METRICS = new HashMap<String,
Double>() {
+ {
+ put("key1", 1.0);
+ put("key2", 3.0);
+ }
+ };
private Function<Pair<String, FileSlice>, Map<String, Double>>
metricsCaptureFn = (partitionFileSlice) -> METRICS;
@Before
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
index a77c61b..9f9562b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
@@ -23,8 +23,8 @@ import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
-import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
index 4fe041a..265bc5c 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
@@ -33,7 +34,6 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.UseFileSplitsFromInputFormat;
-import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -135,7 +135,7 @@ public class HoodieParquetRealtimeInputFormat extends
HoodieParquetInputFormat i
// Get the maxCommit from the last delta or compaction or commit
- when
// bootstrapped from COW table
String maxCommitTime = metaClient
-
.getActiveTimeline().getTimelineOfActions(Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
+
.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.ROLLBACK_ACTION,
HoodieTimeline.DELTA_COMMIT_ACTION))
.filterCompletedInstants().lastInstant().get().getTimestamp();
rtSplits.add(new HoodieRealtimeFileSplit(split,
metaClient.getBasePath(), logFilePaths, maxCommitTime));
diff --git
a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
index 0cef82b..d2808d6 100644
---
a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
+++
b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
@@ -21,7 +21,6 @@ package org.apache.hudi.hive.util;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
-import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -51,6 +50,7 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketException;
+import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@@ -78,8 +78,8 @@ public class HiveTestService {
private TServer tServer;
private HiveServer2 hiveServer;
- public HiveTestService(Configuration configuration) {
- this.workDir = Files.createTempDir().getAbsolutePath();
+ public HiveTestService(Configuration configuration) throws IOException {
+ this.workDir = Files.createTempDirectory(System.currentTimeMillis() +
"-").toFile().getAbsolutePath();
}
public Configuration getHadoopConf() {
@@ -139,7 +139,8 @@ public class HiveTestService {
File derbyLogFile = new File(localHiveDir, "derby.log");
derbyLogFile.createNewFile();
setSystemProperty("derby.stream.error.file", derbyLogFile.getPath());
- conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
Files.createTempDir().getAbsolutePath());
+ conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
+ Files.createTempDirectory(System.currentTimeMillis() +
"-").toFile().getAbsolutePath());
conf.set("datanucleus.schema.autoCreateTables", "true");
conf.set("hive.metastore.schema.verification", "false");
setSystemProperty("derby.stream.error.file", derbyLogFile.getPath());
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
index f61028e..5e6bc33 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
@@ -18,11 +18,12 @@
package org.apache.hudi.integ;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.collect.ImmutableList;
import org.junit.Test;
+import java.util.Collections;
import java.util.List;
/**
@@ -97,16 +98,15 @@ public class ITTestHoodieDemo extends ITTestBase {
}
private void setupDemo() throws Exception {
- List<String> cmds = new ImmutableList.Builder<String>()
- .add("hdfs dfsadmin -safemode wait") // handle NN going into safe mode
at times
- .add("hdfs dfs -mkdir -p " + HDFS_DATA_DIR)
- .add("hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " +
HDFS_BATCH_PATH1)
- .add("/bin/bash " + DEMO_CONTAINER_SCRIPT).build();
+ List<String> cmds = CollectionUtils.createImmutableList("hdfs dfsadmin
-safemode wait",
+ "hdfs dfs -mkdir -p " + HDFS_DATA_DIR,
+ "hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " +
HDFS_BATCH_PATH1,
+ "/bin/bash " + DEMO_CONTAINER_SCRIPT);
+
executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
// create input dir in presto coordinator
- cmds = new ImmutableList.Builder<String>()
- .add("mkdir -p " + HDFS_DATA_DIR).build();
+ cmds = Collections.singletonList("mkdir -p " + HDFS_DATA_DIR);
executeCommandStringsInDocker(PRESTO_COORDINATOR, cmds);
// copy presto sql files to presto coordinator
@@ -116,22 +116,21 @@ public class ITTestHoodieDemo extends ITTestBase {
}
private void ingestFirstBatchAndHiveSync() throws Exception {
- List<String> cmds = new ImmutableList.Builder<String>()
- .add("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ List<String> cmds = CollectionUtils.createImmutableList(
+ "spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ " --table-type COPY_ON_WRITE "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource
--source-ordering-field ts "
+ " --target-base-path " + COW_BASE_PATH + " --target-table " +
COW_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties "
+ " --schemaprovider-class
org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
- + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME))
- .add("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME),
+ ("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ " --table-type MERGE_ON_READ "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource
--source-ordering-field ts "
+ " --target-base-path " + MOR_BASE_PATH + " --target-table " +
MOR_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties "
+ " --schemaprovider-class
org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
- + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT,
"dt", MOR_TABLE_NAME))
- .build();
+ + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT,
"dt", MOR_TABLE_NAME)));
executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
}
@@ -168,23 +167,22 @@ public class ITTestHoodieDemo extends ITTestBase {
}
private void ingestSecondBatchAndHiveSync() throws Exception {
- List<String> cmds = new ImmutableList.Builder<String>()
- .add("hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH2 + " " +
HDFS_BATCH_PATH2)
- .add("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ List<String> cmds = CollectionUtils.createImmutableList(
+ ("hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH2 + " " +
HDFS_BATCH_PATH2),
+ ("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ " --table-type COPY_ON_WRITE "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource
--source-ordering-field ts "
+ " --target-base-path " + COW_BASE_PATH + " --target-table " +
COW_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties "
+ " --schemaprovider-class
org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
- + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME))
- .add("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME)),
+ ("spark-submit --class
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " +
HUDI_UTILITIES_BUNDLE
+ " --table-type MERGE_ON_READ "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource
--source-ordering-field ts "
+ " --target-base-path " + MOR_BASE_PATH + " --target-table " +
MOR_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties "
+ " --schemaprovider-class
org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
- + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT,
"dt", MOR_TABLE_NAME))
- .build();
+ + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT,
"dt", MOR_TABLE_NAME)));
executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
}
diff --git
a/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java
b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java
index 8784526..ef9825b 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java
@@ -23,8 +23,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.CollectionUtils;
-import com.google.common.collect.Sets;
import org.apache.hadoop.fs.FileSystem;
import java.util.List;
@@ -68,7 +68,8 @@ public class HoodieDataSourceHelpers {
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(),
basePath, true);
if (metaClient.getTableType().equals(HoodieTableType.MERGE_ON_READ)) {
return metaClient.getActiveTimeline().getTimelineOfActions(
- Sets.newHashSet(HoodieActiveTimeline.COMMIT_ACTION,
HoodieActiveTimeline.DELTA_COMMIT_ACTION));
+ CollectionUtils.createSet(HoodieActiveTimeline.COMMIT_ACTION,
+ HoodieActiveTimeline.DELTA_COMMIT_ACTION));
} else {
return metaClient.getCommitTimeline().filterCompletedInstants();
}
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
index 7df630a..7d48d55 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.TableFileSystemView.BaseFileOnlyView;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
@@ -38,7 +39,6 @@ import com.beust.jcommander.IValueValidator;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
-import com.google.common.collect.ImmutableList;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -83,7 +83,7 @@ public class HoodieSnapshotExporter {
public static class OutputFormatValidator implements IValueValidator<String>
{
static final String HUDI = "hudi";
- static final List<String> FORMATS = ImmutableList.of("json", "parquet",
HUDI);
+ static final List<String> FORMATS =
CollectionUtils.createImmutableList("json", "parquet", HUDI);
@Override
public void validate(String name, String value) {
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
index deb26b5..0b03507 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
@@ -135,7 +135,7 @@ public class HoodieIncrSource extends RowSource {
* instantEndpts.getValue()); if (!partitionFields.isEmpty()) { //
_hoodie_partition_path String hoodiePartitionPath
* = row.getString(3); List<Object> partitionVals =
* extractor.extractPartitionValuesInPath(hoodiePartitionPath).stream()
.map(o -> (Object)
- * o).collect(Collectors.toList());
Preconditions.checkArgument(partitionVals.size() == partitionFields.size(),
+ * o).collect(Collectors.toList());
ValidationUtils.checkArgument(partitionVals.size() == partitionFields.size(),
* "#partition-fields != #partition-values-extracted"); List<Object>
rowObjs = new
*
ArrayList<>(scala.collection.JavaConversions.seqAsJavaList(row.toSeq()));
rowObjs.addAll(partitionVals); return
* RowFactory.create(rowObjs.toArray()); } return row; },
RowEncoder.apply(newSchema));
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
index abf6578..ffb8e47 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.TypedProperties;
import org.apache.hudi.exception.HoodieIOException;
@@ -41,7 +42,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import com.fasterxml.jackson.dataformat.csv.CsvSchema.Builder;
-import com.google.common.collect.ImmutableList;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileSystem;
@@ -146,7 +146,7 @@ public class UtilitiesTestBase {
hiveSyncConfig.basePath = basePath;
hiveSyncConfig.assumeDatePartitioning = false;
hiveSyncConfig.usePreApacheInputFormat = false;
- hiveSyncConfig.partitionFields = new
ImmutableList.Builder<String>().add("datestr").build();
+ hiveSyncConfig.partitionFields =
CollectionUtils.createImmutableList("datestr");
return hiveSyncConfig;
}
diff --git a/pom.xml b/pom.xml
index 758a049..2274147 100644
--- a/pom.xml
+++ b/pom.xml
@@ -534,13 +534,6 @@
<version>${joda.version}</version>
</dependency>
- <!-- we have to stay at <= 16.0, due to issues with HBase client -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>15.0</version>
- </dependency>
-
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
diff --git a/style/checkstyle.xml b/style/checkstyle.xml
index 2a5e831..f0c423c 100644
--- a/style/checkstyle.xml
+++ b/style/checkstyle.xml
@@ -264,8 +264,8 @@
</module>
<module name="CommentsIndentation"/>
<module name="IllegalImport">
- <property name="illegalPkgs" value="org.apache.commons" />
- <property name="illegalClasses" value="java.util.Optional,
com.google.common.base.Optional" />
+ <property name="illegalPkgs" value="org.apache.commons,
com.google.common" />
+ <property name="illegalClasses" value="java.util.Optional" />
</module>
<module name="EmptyStatement" />
