This is an automated email from the ASF dual-hosted git repository.
forwardxu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ef3a17e3d97 [HUDI-5692] SpillableMapBasePath should be lazily loaded
(#7837)
ef3a17e3d97 is described below
commit ef3a17e3d97428a6f0d6e7ac888747d65a8792c5
Author: Rex(Hui) An <[email protected]>
AuthorDate: Sat Feb 4 15:14:52 2023 +0800
[HUDI-5692] SpillableMapBasePath should be lazily loaded (#7837)
---
.../apache/hudi/cli/commands/HoodieLogFileCommand.java | 2 +-
.../hudi/cli/commands/TestHoodieLogFileCommand.java | 2 +-
.../java/org/apache/hudi/config/HoodieMemoryConfig.java | 16 ++++++++--------
.../java/org/apache/hudi/config/HoodieWriteConfig.java | 3 ++-
.../apache/hudi/sink/compact/FlinkCompactionConfig.java | 2 +-
.../testsuite/reader/DFSHoodieDatasetInputReader.java | 2 +-
.../procedures/ShowHoodieLogFileRecordsProcedure.scala | 2 +-
7 files changed, 15 insertions(+), 14 deletions(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
index 075b809e05c..d85d1c36fd0 100644
---
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
+++
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
@@ -218,7 +218,7 @@ public class HoodieLogFileCommand {
.withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue())
.withMaxMemorySizeInBytes(
HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES)
-
.withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue())
+
.withSpillableMapBasePath(HoodieMemoryConfig.getDefaultSpillableMapBasePath())
.withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue())
.withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue())
.withRecordMerger(HoodieRecordUtils.loadRecordMerger(HoodieAvroRecordMerger.class.getName()))
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
index 261002c9327..d964a127683 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
@@ -229,7 +229,7 @@ public class TestHoodieLogFileCommand extends
CLIFunctionalTestHarness {
Boolean.parseBoolean(
HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue()))
.withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue())
-
.withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue())
+
.withSpillableMapBasePath(HoodieMemoryConfig.getDefaultSpillableMapBasePath())
.withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue())
.withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue())
.withRecordMerger(HoodieRecordUtils.loadRecordMerger(HoodieAvroRecordMerger.class.getName()))
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
index 960ec61dc0b..480cf1797f7 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
@@ -23,7 +23,6 @@ import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.util.FileIOUtils;
-import org.apache.hudi.common.util.Option;
import javax.annotation.concurrent.Immutable;
import java.io.File;
@@ -80,11 +79,7 @@ public class HoodieMemoryConfig extends HoodieConfig {
public static final ConfigProperty<String> SPILLABLE_MAP_BASE_PATH =
ConfigProperty
.key("hoodie.memory.spillable.map.path")
- .defaultValue("/tmp/")
- .withInferFunction(cfg -> {
- String[] localDirs = FileIOUtils.getConfiguredLocalDirs();
- return (localDirs != null && localDirs.length > 0) ?
Option.of(localDirs[0]) : Option.empty();
- })
+ .noDefaultValue()
.withDocumentation("Default file path for spillable map");
public static final ConfigProperty<Double> WRITESTATUS_FAILURE_FRACTION =
ConfigProperty
@@ -121,9 +116,9 @@ public class HoodieMemoryConfig extends HoodieConfig {
/** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead
*/
@Deprecated
public static final String SPILLABLE_MAP_BASE_PATH_PROP =
SPILLABLE_MAP_BASE_PATH.key();
- /** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead
*/
+ /** @deprecated Use getDefaultSpillableMapBasePath() instead */
@Deprecated
- public static final String DEFAULT_SPILLABLE_MAP_BASE_PATH =
SPILLABLE_MAP_BASE_PATH.defaultValue();
+ public static final String DEFAULT_SPILLABLE_MAP_BASE_PATH =
getDefaultSpillableMapBasePath();
/** @deprecated Use {@link #WRITESTATUS_FAILURE_FRACTION} and its methods
instead */
@Deprecated
public static final String WRITESTATUS_FAILURE_FRACTION_PROP =
WRITESTATUS_FAILURE_FRACTION.key();
@@ -135,6 +130,11 @@ public class HoodieMemoryConfig extends HoodieConfig {
super();
}
+ public static String getDefaultSpillableMapBasePath() {
+ String[] localDirs = FileIOUtils.getConfiguredLocalDirs();
+ return (localDirs != null && localDirs.length > 0) ? localDirs[0] :
"/tmp/";
+ }
+
public static HoodieMemoryConfig.Builder newBuilder() {
return new Builder();
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index f56defe7eac..7988f7befef 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2090,7 +2090,8 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public String getSpillableMapBasePath() {
- return getString(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH);
+ return
Option.ofNullable(getString(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH))
+ .orElseGet(HoodieMemoryConfig::getDefaultSpillableMapBasePath);
}
public double getWriteStatusFailureFraction() {
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/FlinkCompactionConfig.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/FlinkCompactionConfig.java
index 0308d246333..068c39d838c 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/FlinkCompactionConfig.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/FlinkCompactionConfig.java
@@ -123,7 +123,7 @@ public class FlinkCompactionConfig extends Configuration {
public String compactionPlanInstant;
@Parameter(names = {"--spillable_map_path"}, description = "Default file
path prefix for spillable map.")
- public String spillableMapPath =
HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue();
+ public String spillableMapPath =
HoodieMemoryConfig.getDefaultSpillableMapBasePath();
/**
* Transforms a {@code HoodieFlinkCompaction.config} into {@code
Configuration}.
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
index 0e3f27e2e3b..5703cc6cead 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
@@ -287,7 +287,7 @@ public class DFSHoodieDatasetInputReader extends
DFSDeltaInputReader {
.withReadBlocksLazily(true)
.withReverseReader(false)
.withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue())
-
.withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue())
+
.withSpillableMapBasePath(HoodieMemoryConfig.getDefaultSpillableMapBasePath())
.withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue())
.withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue())
.withOptimizedLogBlocksScan(Boolean.parseBoolean(HoodieCompactionConfig.ENABLE_OPTIMIZED_LOG_BLOCKS_SCAN.defaultValue()))
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
index 9c3e2e469b0..ebb325f1936 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
@@ -73,7 +73,7 @@ class ShowHoodieLogFileRecordsProcedure extends BaseProcedure
with ProcedureBuil
.withReverseReader(java.lang.Boolean.parseBoolean(HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue))
.withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue)
.withMaxMemorySizeInBytes(HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES)
-
.withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue)
+
.withSpillableMapBasePath(HoodieMemoryConfig.getDefaultSpillableMapBasePath)
.withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue)
.withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue)
.withRecordMerger(HoodieRecordUtils.loadRecordMerger(classOf[HoodieAvroRecordMerger].getName))
