This is an automated email from the ASF dual-hosted git repository. forwardxu pushed a commit to branch release-0.12.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 257b09003b4473bfce2c7f4731dee7bd24cbfd4f Author: XuQianJin-Stars <[email protected]> AuthorDate: Wed Jan 4 16:04:09 2023 +0800 [MINOR] Improve spillableMapBasePath when disk directory is full --- .../src/main/java/org/apache/hudi/io/HoodieMergeHandle.java | 5 ++++- .../apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java index 17bbb2f7f0e..a6d3b9df930 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java @@ -36,6 +36,7 @@ import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; import org.apache.hudi.common.model.IOType; import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; @@ -226,7 +227,9 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload, I, K, O> extends H // Load the new records in a map long memoryForMerge = IOUtils.getMaxMemoryPerPartitionMerge(taskContextSupplier, config); LOG.info("MaxMemoryPerPartitionMerge => " + memoryForMerge); - this.keyToNewRecords = new ExternalSpillableMap<>(memoryForMerge, config.getSpillableMapBasePath(), + String[] localDirs = FileIOUtils.getConfiguredLocalDirs(); + String spillableMapBasePath = localDirs.length > 0 ? localDirs[0] : config.getSpillableMapBasePath(); + this.keyToNewRecords = new ExternalSpillableMap<>(memoryForMerge, spillableMapBasePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(tableSchema), config.getCommonConfig().getSpillableDiskMapType(), config.getCommonConfig().isBitCaskDiskMapCompressionEnabled()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index 45975fbfdee..ead387b214d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; @@ -90,6 +91,8 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordReader instantRange, withOperationField, forceFullScan, partitionName, internalSchema); try { + String[] localDirs = FileIOUtils.getConfiguredLocalDirs(); + spillableMapBasePath = localDirs.length > 0 ? localDirs[0] : spillableMapBasePath; // Store merged records for all versions for this log file, set the in-memory footprint to maxInMemoryMapSize this.records = new ExternalSpillableMap<>(maxMemorySizeInBytes, spillableMapBasePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(readerSchema), diskMapType, isBitCaskDiskMapCompressionEnabled);
