This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9ee96655094 [HUDI-7979] Adjusting defaults with spillable map memory
(#11611)
9ee96655094 is described below
commit 9ee9665509451c58736b37daa686510fbd1ad536
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Thu Jul 11 22:11:19 2024 -0700
[HUDI-7979] Adjusting defaults with spillable map memory (#11611)
---
.../apache/hudi/config/TestHoodieWriteConfig.java | 21 +++++++
.../common/config/SerializableConfiguration.java | 69 ++++++++++++++++++++++
.../table/view/FileSystemViewStorageConfig.java | 11 ++--
.../org/apache/hudi/exception/HoodieException.java | 48 +++++++++++++++
.../apache/hudi/exception/HoodieIOException.java | 44 ++++++++++++++
.../HoodieIncompatibleSchemaException.java | 33 +++++++++++
6 files changed, 221 insertions(+), 5 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
index 00c9bfbd163..b39ac241407 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.marker.MarkerType;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig.Builder;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
@@ -553,6 +554,26 @@ public class TestHoodieWriteConfig {
"Non-blocking concurrency control requires the MOR table with simple
bucket index");
}
+ @Test
+ public void testFileSystemViewStorageConfigDefaults() {
+ HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath("/tmp").build();
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() *
FileSystemViewStorageConfig.BOOTSTRAP_BASE_FILE_MEM_FRACTION.defaultValue(),
+ writeConfig.getViewStorageConfig().getMaxMemoryForBootstrapBaseFile());
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() *
FileSystemViewStorageConfig.SPILLABLE_COMPACTION_MEM_FRACTION.defaultValue(),
+ writeConfig.getViewStorageConfig().getMaxMemoryForPendingCompaction());
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() *
FileSystemViewStorageConfig.SPILLABLE_LOG_COMPACTION_MEM_FRACTION.defaultValue(),
+
writeConfig.getViewStorageConfig().getMaxMemoryForPendingLogCompaction());
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() *
FileSystemViewStorageConfig.SPILLABLE_CLUSTERING_MEM_FRACTION.defaultValue(),
+
writeConfig.getViewStorageConfig().getMaxMemoryForPendingClusteringFileGroups());
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() *
FileSystemViewStorageConfig.SPILLABLE_REPLACED_MEM_FRACTION.defaultValue(),
+
writeConfig.getViewStorageConfig().getMaxMemoryForReplacedFileGroups());
+ assertEquals(FileSystemViewStorageConfig.SPILLABLE_MEMORY.defaultValue() -
writeConfig.getViewStorageConfig().getMaxMemoryForBootstrapBaseFile()
+ -
writeConfig.getViewStorageConfig().getMaxMemoryForPendingCompaction() -
writeConfig.getViewStorageConfig().getMaxMemoryForPendingLogCompaction()
+ -
writeConfig.getViewStorageConfig().getMaxMemoryForPendingClusteringFileGroups()
+ -
writeConfig.getViewStorageConfig().getMaxMemoryForReplacedFileGroups(),
+ writeConfig.getViewStorageConfig().getMaxMemoryForFileGroupMap());
+ }
+
private HoodieWriteConfig createWriteConfig(Map<String, String> configs) {
final Properties properties = new Properties();
configs.forEach(properties::setProperty);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/SerializableConfiguration.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/SerializableConfiguration.java
new file mode 100644
index 00000000000..23a22e01822
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/SerializableConfiguration.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.config;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+/**
+ * A wrapped configuration which can be serialized.
+ */
+public class SerializableConfiguration implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient Configuration configuration;
+
+ public SerializableConfiguration(Configuration configuration) {
+ this.configuration = new Configuration(configuration);
+ }
+
+ public SerializableConfiguration(SerializableConfiguration configuration) {
+ this.configuration = configuration.newCopy();
+ }
+
+ public Configuration newCopy() {
+ return new Configuration(configuration);
+ }
+
+ public Configuration get() {
+ return configuration;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ configuration.write(out);
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException {
+ configuration = new Configuration(false);
+ configuration.readFields(in);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder str = new StringBuilder();
+ configuration.iterator().forEachRemaining(e ->
str.append(String.format("%s => %s \n", e.getKey(), e.getValue())));
+ return configuration.toString();
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
index 038c1d569df..19741cd8ad3 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
@@ -89,13 +89,13 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
public static final ConfigProperty<Double> SPILLABLE_COMPACTION_MEM_FRACTION
= ConfigProperty
.key("hoodie.filesystem.view.spillable.compaction.mem.fraction")
- .defaultValue(0.8)
+ .defaultValue(0.1)
.markAdvanced()
.withDocumentation("Fraction of the file system view memory, to be used
for holding compaction related metadata.");
public static final ConfigProperty<Double>
SPILLABLE_LOG_COMPACTION_MEM_FRACTION = ConfigProperty
.key("hoodie.filesystem.view.spillable.log.compaction.mem.fraction")
- .defaultValue(0.8)
+ .defaultValue(0.02)
.markAdvanced()
.sinceVersion("0.13.0")
.withDocumentation("Fraction of the file system view memory, to be used
for holding log compaction related metadata.");
@@ -108,13 +108,13 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
public static final ConfigProperty<Double> SPILLABLE_REPLACED_MEM_FRACTION =
ConfigProperty
.key("hoodie.filesystem.view.spillable.replaced.mem.fraction")
- .defaultValue(0.01)
+ .defaultValue(0.05)
.markAdvanced()
.withDocumentation("Fraction of the file system view memory, to be used
for holding replace commit related metadata.");
public static final ConfigProperty<Double> SPILLABLE_CLUSTERING_MEM_FRACTION
= ConfigProperty
.key("hoodie.filesystem.view.spillable.clustering.mem.fraction")
- .defaultValue(0.01)
+ .defaultValue(0.02)
.markAdvanced()
.withDocumentation("Fraction of the file system view memory, to be used
for holding clustering related metadata.");
@@ -223,7 +223,8 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
public long getMaxMemoryForFileGroupMap() {
long totalMemory = getLong(SPILLABLE_MEMORY);
- return totalMemory - getMaxMemoryForPendingCompaction() -
getMaxMemoryForBootstrapBaseFile();
+ return totalMemory - getMaxMemoryForPendingCompaction() -
getMaxMemoryForBootstrapBaseFile() - getMaxMemoryForPendingLogCompaction()
+ - getMaxMemoryForPendingClusteringFileGroups() -
getMaxMemoryForReplacedFileGroups();
}
public long getMaxMemoryForPendingCompaction() {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java
b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java
new file mode 100644
index 00000000000..c13b03df1c4
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.exception;
+
+/**
+ * <p>
+ * Exception thrown for Hoodie failures. The root of the exception hierarchy.
+ * </p>
+ * <p>
+ * Hoodie Write/Read clients will throw this exception if any of its
operations fail. This is a runtime (unchecked)
+ * exception.
+ * </p>
+ */
+public class HoodieException extends RuntimeException {
+
+ public HoodieException() {
+ super();
+ }
+
+ public HoodieException(String message) {
+ super(message);
+ }
+
+ public HoodieException(String message, Throwable t) {
+ super(message, t);
+ }
+
+ public HoodieException(Throwable t) {
+ super(t);
+ }
+
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java
b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java
new file mode 100644
index 00000000000..f0cabd6620c
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.exception;
+
+import java.io.IOException;
+
+/**
+ * <p>
+ * Exception thrown for table IO-related failures.
+ * </p>
+ */
+public class HoodieIOException extends HoodieException {
+
+ private IOException ioException;
+
+ public HoodieIOException(String msg, IOException t) {
+ super(msg, t);
+ this.ioException = t;
+ }
+
+ public HoodieIOException(String msg) {
+ super(msg);
+ }
+
+ public IOException getIOException() {
+ return ioException;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java
b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java
new file mode 100644
index 00000000000..a739af67909
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.exception;
+
+/**
+ * Exception for incompatible schema.
+ */
+public class HoodieIncompatibleSchemaException extends RuntimeException {
+
+ public HoodieIncompatibleSchemaException(String msg, Throwable e) {
+ super(msg, e);
+ }
+
+ public HoodieIncompatibleSchemaException(String msg) {
+ super(msg);
+ }
+}
