This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 658e964cf77 [HUDI-7668] Add and rename APIs in StorageConfiguration
(#11102)
658e964cf77 is described below
commit 658e964cf7711f61d476e4e9bc5f67eff817c3ab
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Apr 26 16:25:13 2024 -0700
[HUDI-7668] Add and rename APIs in StorageConfiguration (#11102)
---
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 2 +-
.../storage/hadoop/HadoopStorageConfiguration.java | 11 ++++++---
.../apache/hudi/storage/StorageConfiguration.java | 27 ++++++++++++++++++----
.../io/storage/BaseTestStorageConfiguration.java | 18 +++++++++++----
4 files changed, 46 insertions(+), 12 deletions(-)
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
index 78b293ee75f..80d881a45fa 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
@@ -80,7 +80,7 @@ public class HadoopFSUtils {
}
public static <T> FileSystem getFs(Path path, StorageConfiguration<T>
storageConf, boolean newCopy) {
- T conf = newCopy ? storageConf.newCopy() : storageConf.get();
+ T conf = newCopy ? storageConf.unwrapCopy() : storageConf.unwrap();
ValidationUtils.checkArgument(conf instanceof Configuration);
return getFs(path, (Configuration) conf);
}
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java
b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java
index a0009aaf75a..f272f8333eb 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HadoopStorageConfiguration.java
@@ -54,16 +54,21 @@ public class HadoopStorageConfiguration extends
StorageConfiguration<Configurati
}
public HadoopStorageConfiguration(HadoopStorageConfiguration configuration) {
- this.configuration = configuration.newCopy();
+ this.configuration = configuration.unwrapCopy();
}
@Override
- public Configuration get() {
+ public StorageConfiguration<Configuration> newInstance() {
+ return new HadoopStorageConfiguration(this);
+ }
+
+ @Override
+ public Configuration unwrap() {
return configuration;
}
@Override
- public Configuration newCopy() {
+ public Configuration unwrapCopy() {
return new Configuration(configuration);
}
diff --git
a/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java
b/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java
index d92eeab8bed..c0a60490f21 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/StorageConfiguration.java
@@ -31,14 +31,20 @@ import java.io.Serializable;
*/
public abstract class StorageConfiguration<T> implements Serializable {
/**
- * @return the storage configuration.
+ * @return a new {@link StorageConfiguration} instance with a new copy of
+ * the configuration of type {@link T}.
*/
- public abstract T get();
+ public abstract StorageConfiguration<T> newInstance();
/**
- * @return a new copy of the storage configuration.
+ * @return the underlying configuration of type {@link T}.
*/
- public abstract T newCopy();
+ public abstract T unwrap();
+
+ /**
+ * @return a new copy of the underlying configuration of type {@link T}.
+ */
+ public abstract T unwrapCopy();
/**
* Sets the configuration key-value pair.
@@ -108,4 +114,17 @@ public abstract class StorageConfiguration<T> implements
Serializable {
? Enum.valueOf(defaultValue.getDeclaringClass(), value.get())
: defaultValue;
}
+
+ /**
+ * Sets a property key with a value in the configuration, if the property key
+ * does not already exist.
+ *
+ * @param key property key.
+ * @param value property value.
+ */
+ public final void setIfUnset(String key, String value) {
+ if (getString(key).isEmpty()) {
+ set(key, value);
+ }
+ }
}
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java
b/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java
index 19ae29da985..1d6a3d338e4 100644
---
a/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java
+++
b/hudi-io/src/test/java/org/apache/hudi/io/storage/BaseTestStorageConfiguration.java
@@ -47,11 +47,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public abstract class BaseTestStorageConfiguration<T> {
private static final Map<String, String> EMPTY_MAP = new HashMap<>();
private static final String KEY_STRING = "hudi.key.string";
+ private static final String KEY_STRING_OTHER = "hudi.key.string.other";
private static final String KEY_BOOLEAN = "hudi.key.boolean";
private static final String KEY_LONG = "hudi.key.long";
private static final String KEY_ENUM = "hudi.key.enum";
private static final String KEY_NON_EXISTENT = "hudi.key.non_existent";
private static final String VALUE_STRING = "string_value";
+ private static final String VALUE_STRING_1 = "string_value_1";
private static final String VALUE_BOOLEAN = "true";
private static final String VALUE_LONG = "12309120";
private static final String VALUE_ENUM = TestEnum.ENUM2.toString();
@@ -68,11 +70,14 @@ public abstract class BaseTestStorageConfiguration<T> {
protected abstract T getConf(Map<String, String> mapping);
@Test
- public void testConstructorGetNewCopy() {
+ public void testConstructorNewInstanceUnwrapCopy() {
T conf = getConf(EMPTY_MAP);
StorageConfiguration<T> storageConf = getStorageConfiguration(conf);
- assertSame(storageConf.get(), storageConf.get());
- assertNotSame(storageConf.get(), storageConf.newCopy());
+ StorageConfiguration<T> newStorageConf = storageConf.newInstance();
+ assertNotSame(storageConf, newStorageConf);
+ assertNotSame(storageConf.unwrap(), newStorageConf.unwrap());
+ assertSame(storageConf.unwrap(), storageConf.unwrap());
+ assertNotSame(storageConf.unwrap(), storageConf.unwrapCopy());
}
@Test
@@ -85,6 +90,11 @@ public abstract class BaseTestStorageConfiguration<T> {
storageConf.set(KEY_BOOLEAN, VALUE_BOOLEAN);
assertEquals(Option.of(VALUE_STRING), storageConf.getString(KEY_STRING));
assertTrue(storageConf.getBoolean(KEY_BOOLEAN, false));
+
+ storageConf.setIfUnset(KEY_STRING, VALUE_STRING + "_1");
+ storageConf.setIfUnset(KEY_STRING_OTHER, VALUE_STRING_1);
+ assertEquals(Option.of(VALUE_STRING), storageConf.getString(KEY_STRING));
+ assertEquals(Option.of(VALUE_STRING_1),
storageConf.getString(KEY_STRING_OTHER));
}
@Test
@@ -102,7 +112,7 @@ public abstract class BaseTestStorageConfiguration<T> {
try (ByteArrayInputStream bais = new
ByteArrayInputStream(baos.toByteArray());
ObjectInputStream ois = new ObjectInputStream(bais)) {
StorageConfiguration<?> deserialized = (StorageConfiguration)
ois.readObject();
- assertNotNull(deserialized.get());
+ assertNotNull(deserialized.unwrap());
validateConfigs(deserialized);
}
}
