This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e8494df5704f0bbc62c71a1e9a1e6dba02b09f06 Author: Y Ethan Guo <[email protected]> AuthorDate: Thu Apr 25 14:51:58 2024 -0700 [HUDI-7666] Fix serializable implementation of StorageConfiguration class (#11091) --- .../storage/hadoop/HadoopStorageConfiguration.java | 41 +++++++++++++++------- .../apache/hudi/storage/StorageConfiguration.java | 23 +----------- .../io/storage/BaseTestStorageConfiguration.java | 21 +++++++++++ 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java index 9c5696c01ab..a0009aaf75a 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.Serializable; /** * Implementation of {@link StorageConfiguration} providing Hadoop's {@link Configuration}. @@ -66,18 +67,6 @@ public class HadoopStorageConfiguration extends StorageConfiguration<Configurati return new Configuration(configuration); } - @Override - public void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - configuration.write(out); - } - - @Override - public void readObject(ObjectInputStream in) throws IOException { - configuration = new Configuration(false); - configuration.readFields(in); - } - @Override public void set(String key, String value) { configuration.set(key, value); @@ -95,4 +84,32 @@ public class HadoopStorageConfiguration extends StorageConfiguration<Configurati e -> stringBuilder.append(String.format("%s => %s \n", e.getKey(), e.getValue()))); return stringBuilder.toString(); } + + /** + * Serializes the storage configuration. + * DO NOT change the signature, as required by {@link Serializable}. + * This method has to be private; otherwise, serde of the object of this class + * in Spark does not work. + * + * @param out stream to write. + * @throws IOException on I/O error. + */ + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + configuration.write(out); + } + + /** + * Deserializes the storage configuration. + * DO NOT change the signature, as required by {@link Serializable}. + * This method has to be private; otherwise, serde of the object of this class + * in Spark does not work. + * + * @param in stream to read. + * @throws IOException on I/O error. + */ + private void readObject(ObjectInputStream in) throws IOException { + configuration = new Configuration(false); + configuration.readFields(in); + } } diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java b/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java index 4b81347bf3e..d92eeab8bed 100644 --- a/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java +++ b/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java @@ -22,9 +22,6 @@ package org.apache.hudi.storage; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.Serializable; /** @@ -42,25 +39,7 @@ public abstract class StorageConfiguration<T> implements Serializable { * @return a new copy of the storage configuration. */ public abstract T newCopy(); - - /** - * Serializes the storage configuration. - * DO NOT change the signature, as required by {@link Serializable}. - * - * @param out stream to write. - * @throws IOException on I/O error. - */ - public abstract void writeObject(ObjectOutputStream out) throws IOException; - - /** - * Deserializes the storage configuration. - * DO NOT change the signature, as required by {@link Serializable}. - * - * @param in stream to read. - * @throws IOException on I/O error. - */ - public abstract void readObject(ObjectInputStream in) throws IOException; - + /** * Sets the configuration key-value pair. * diff --git a/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java b/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java index 6828e3c766e..19ae29da985 100644 --- a/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java +++ b/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java @@ -24,11 +24,17 @@ import org.apache.hudi.storage.StorageConfiguration; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -87,6 +93,21 @@ public abstract class BaseTestStorageConfiguration<T> { validateConfigs(storageConf); } + @Test + public void testSerializability() throws IOException, ClassNotFoundException { + StorageConfiguration<?> storageConf = getStorageConfiguration(getConf(prepareConfigs())); + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(storageConf); + try (ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + ObjectInputStream ois = new ObjectInputStream(bais)) { + StorageConfiguration<?> deserialized = (StorageConfiguration) ois.readObject(); + assertNotNull(deserialized.get()); + validateConfigs(deserialized); + } + } + } + private Map<String, String> prepareConfigs() { Map<String, String> conf = new HashMap<>(); conf.put(KEY_STRING, VALUE_STRING);
