[hudi] branch master updated: [HUDI-4516] fix Task not serializable error when run HoodieCleaner after one failure (#6265)

[sivabalan]

This is an automated email from the ASF dual-hosted git repository.

sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 4ec060129f [HUDI-4516] fix Task not serializable error when run 
HoodieCleaner after one failure (#6265)
4ec060129f is described below

commit 4ec060129fb66dfd753a61266ef2436fa6748168
Author: 冯健 <fengjian...@gmail.com>
AuthorDate: Wed Aug 3 07:09:33 2022 +0800

    [HUDI-4516] fix Task not serializable error when run HoodieCleaner after 
one failure (#6265)
    
    
    Co-authored-by: jian.feng <jian.f...@shopee.com>
---
 .../main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java   | 4 +++-
 .../java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java   | 4 +++-
 .../org/apache/hudi/common/util/collection/ExternalSpillableMap.java  | 2 +-
 .../java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java  | 4 ++--
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
index 0e6d78f1f2..8032a04df4 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
@@ -18,12 +18,14 @@
 
 package org.apache.hudi.common.util;
 
+import java.io.Serializable;
+
 /**
  * Default implementation of size-estimator that uses Twitter's 
ObjectSizeCalculator.
  * 
  * @param <T>
  */
-public class DefaultSizeEstimator<T> implements SizeEstimator<T> {
+public class DefaultSizeEstimator<T> implements SizeEstimator<T>, Serializable 
{
 
   @Override
   public long sizeEstimate(T t) {
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
index 78f7b1b9bb..88a0f70cd8 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
@@ -26,12 +26,14 @@ import org.apache.avro.Schema;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 
+import java.io.Serializable;
+
 /**
  * Size Estimator for Hoodie record payload.
  * 
  * @param <T>
  */
-public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> 
implements SizeEstimator<HoodieRecord<T>> {
+public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> 
implements SizeEstimator<HoodieRecord<T>>, Serializable {
 
   private static final Logger LOG = 
LogManager.getLogger(HoodieRecordSizeEstimator.class);
 
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
index a6e8d5cfb3..8d2707d604 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
@@ -52,7 +52,7 @@ import java.util.stream.Stream;
  * map may occupy more memory than is available, resulting in OOM. However, if 
the spill threshold is too low, we spill
  * frequently and incur unnecessary disk writes.
  */
-public class ExternalSpillableMap<T extends Serializable, R extends 
Serializable> implements Map<T, R> {
+public class ExternalSpillableMap<T extends Serializable, R extends 
Serializable> implements Map<T, R>, Serializable {
 
   // Find the actual estimated payload size after inserting N records
   private static final int NUMBER_OF_RECORDS_TO_ESTIMATE_PAYLOAD_SIZE = 100;
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
index c4fbd3a8cb..67b4ffabe2 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
@@ -29,12 +29,12 @@ import java.util.Set;
 /**
  * A map's implementation based on RocksDB.
  */
-public final class RocksDBBasedMap<K extends Serializable, R extends 
Serializable> implements Map<K, R> {
+public final class RocksDBBasedMap<K extends Serializable, R extends 
Serializable> implements Map<K, R>, Serializable {
 
   private static final String COL_FAMILY_NAME = "map_handle";
 
   private final String rocksDbStoragePath;
-  private RocksDBDAO rocksDBDAO;
+  private transient RocksDBDAO rocksDBDAO;
   private final String columnFamilyName;
 
   public RocksDBBasedMap(String rocksDbStoragePath) {