This is an automated email from the ASF dual-hosted git repository. codope pushed a commit to branch release-0.12.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 92843dc4095e497d684d51318708611163d6fd8b Author: 冯健 <[email protected]> AuthorDate: Wed Aug 3 07:09:33 2022 +0800 [HUDI-4516] fix Task not serializable error when run HoodieCleaner after one failure (#6265) Co-authored-by: jian.feng <[email protected]> --- .../main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java | 4 +++- .../java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java | 4 +++- .../org/apache/hudi/common/util/collection/ExternalSpillableMap.java | 2 +- .../java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java index 0e6d78f1f2..8032a04df4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.util; +import java.io.Serializable; + /** * Default implementation of size-estimator that uses Twitter's ObjectSizeCalculator. * * @param <T> */ -public class DefaultSizeEstimator<T> implements SizeEstimator<T> { +public class DefaultSizeEstimator<T> implements SizeEstimator<T>, Serializable { @Override public long sizeEstimate(T t) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java index 78f7b1b9bb..88a0f70cd8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java @@ -26,12 +26,14 @@ import org.apache.avro.Schema; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.Serializable; + /** * Size Estimator for Hoodie record payload. * * @param <T> */ -public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> implements SizeEstimator<HoodieRecord<T>> { +public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> implements SizeEstimator<HoodieRecord<T>>, Serializable { private static final Logger LOG = LogManager.getLogger(HoodieRecordSizeEstimator.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java index a6e8d5cfb3..8d2707d604 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java @@ -52,7 +52,7 @@ import java.util.stream.Stream; * map may occupy more memory than is available, resulting in OOM. However, if the spill threshold is too low, we spill * frequently and incur unnecessary disk writes. */ -public class ExternalSpillableMap<T extends Serializable, R extends Serializable> implements Map<T, R> { +public class ExternalSpillableMap<T extends Serializable, R extends Serializable> implements Map<T, R>, Serializable { // Find the actual estimated payload size after inserting N records private static final int NUMBER_OF_RECORDS_TO_ESTIMATE_PAYLOAD_SIZE = 100; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java index c4fbd3a8cb..67b4ffabe2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java @@ -29,12 +29,12 @@ import java.util.Set; /** * A map's implementation based on RocksDB. */ -public final class RocksDBBasedMap<K extends Serializable, R extends Serializable> implements Map<K, R> { +public final class RocksDBBasedMap<K extends Serializable, R extends Serializable> implements Map<K, R>, Serializable { private static final String COL_FAMILY_NAME = "map_handle"; private final String rocksDbStoragePath; - private RocksDBDAO rocksDBDAO; + private transient RocksDBDAO rocksDBDAO; private final String columnFamilyName; public RocksDBBasedMap(String rocksDbStoragePath) {
