This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 667f93b1fff97fac8803878ad451ffc5ca59401a Author: Y Ethan Guo <[email protected]> AuthorDate: Wed Apr 17 21:39:28 2024 -0700 [HUDI-7636] Make StoragePath Serializable (#11049) --- .../java/org/apache/hudi/storage/StoragePath.java | 14 +++++++++-- .../apache/hudi/io/storage/TestStoragePath.java | 28 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/StoragePath.java b/hudi-io/src/main/java/org/apache/hudi/storage/StoragePath.java index f3a88f7c89b..24bf77e76ad 100644 --- a/hudi-io/src/main/java/org/apache/hudi/storage/StoragePath.java +++ b/hudi-io/src/main/java/org/apache/hudi/storage/StoragePath.java @@ -23,6 +23,9 @@ import org.apache.hudi.ApiMaturityLevel; import org.apache.hudi.PublicAPIClass; import org.apache.hudi.PublicAPIMethod; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; @@ -33,12 +36,11 @@ import java.net.URISyntaxException; * The APIs are mainly based on {@code org.apache.hadoop.fs.Path} class. */ @PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING) -// StoragePath public class StoragePath implements Comparable<StoragePath>, Serializable { public static final char SEPARATOR_CHAR = '/'; public static final char COLON_CHAR = ':'; public static final String SEPARATOR = "" + SEPARATOR_CHAR; - private final URI uri; + private URI uri; private transient volatile StoragePath cachedParent; private transient volatile String cachedName; private transient volatile String uriString; @@ -306,4 +308,12 @@ public class StoragePath implements Comparable<StoragePath>, Serializable { } return path.substring(0, indexOfLastSlash); } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.writeObject(uri); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + uri = (URI) in.readObject(); + } } diff --git a/hudi-io/src/test/java/org/apache/hudi/io/storage/TestStoragePath.java b/hudi-io/src/test/java/org/apache/hudi/io/storage/TestStoragePath.java index 9195ebec9fd..e7ce6ecc838 100644 --- a/hudi-io/src/test/java/org/apache/hudi/io/storage/TestStoragePath.java +++ b/hudi-io/src/test/java/org/apache/hudi/io/storage/TestStoragePath.java @@ -22,7 +22,14 @@ package org.apache.hudi.io.storage; import org.apache.hudi.storage.StoragePath; import org.junit.jupiter.api.Test; - +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; @@ -197,6 +204,25 @@ public class TestStoragePath { () -> new StoragePath("a").makeQualified(defaultUri)); } + @ParameterizedTest + @ValueSource(strings = { + "/x/y/1.file#bar", + "s3://foo/bar/1%2F2%2F3", + "hdfs://host1/a/b/c" + }) + public void testSerializability(String pathStr) throws IOException, ClassNotFoundException { + StoragePath path = new StoragePath(pathStr); + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(path); + try (ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + ObjectInputStream ois = new ObjectInputStream(bais)) { + StoragePath deserialized = (StoragePath) ois.readObject(); + assertEquals(path.toUri(), deserialized.toUri()); + } + } + } + @Test public void testEquals() { assertEquals(new StoragePath("/foo"), new StoragePath("/foo"));
