This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new f1410bf Fixes HUDI-38: Reduce memory overhead of WriteStatus
f1410bf is described below
commit f1410bfdcde0745625d46b9f3f5e996eb368663f
Author: Vinoth Chandar <[email protected]>
AuthorDate: Tue Mar 26 14:31:19 2019 -0700
Fixes HUDI-38: Reduce memory overhead of WriteStatus
- For implicit indexes (e.g BloomIndex), don't buffer up written records
- By default, only collect 10% of failing records to avoid OOMs
- Improves debuggability via above, since data errors can now show up in
collect()
- Unit tests & fixing subclasses & adjusting tests
---
.../src/main/java/com/uber/hoodie/WriteStatus.java | 29 +++++++++---
.../com/uber/hoodie/config/HoodieMemoryConfig.java | 18 +++++++-
.../com/uber/hoodie/config/HoodieWriteConfig.java | 13 +++---
.../java/com/uber/hoodie/io/HoodieIOHandle.java | 4 +-
.../java/com/uber/hoodie/TestHoodieClientBase.java | 2 +
.../test/java/com/uber/hoodie/TestWriteStatus.java | 53 ++++++++++++++++++++++
.../com/uber/hoodie/common/TestRawTripPayload.java | 4 ++
.../java/com/uber/hoodie/index/TestHbaseIndex.java | 2 +-
.../uber/hoodie/table/TestCopyOnWriteTable.java | 10 ++--
9 files changed, 112 insertions(+), 23 deletions(-)
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java
b/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java
index ac54d89..54b32f0 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java
@@ -25,12 +25,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Random;
/**
* Status of a write operation.
*/
public class WriteStatus implements Serializable {
+ private static final long RANDOM_SEED = 9038412832L;
+
private final HashMap<HoodieKey, Throwable> errors = new HashMap<>();
private final List<HoodieRecord> writtenRecords = new ArrayList<>();
@@ -48,6 +51,16 @@ public class WriteStatus implements Serializable {
private long totalRecords = 0;
private long totalErrorRecords = 0;
+ private final double failureFraction;
+ private final boolean trackSuccessRecords;
+ private final transient Random random;
+
+ public WriteStatus(Boolean trackSuccessRecords, Double failureFraction) {
+ this.trackSuccessRecords = trackSuccessRecords;
+ this.failureFraction = failureFraction;
+ this.random = new Random(RANDOM_SEED);
+ }
+
/**
* Mark write as success, optionally using given parameters for the purpose
of calculating some
* aggregate metrics. This method is not meant to cache passed arguments,
since WriteStatus
@@ -58,9 +71,10 @@ public class WriteStatus implements Serializable {
* @param optionalRecordMetadata optional metadata related to data contained
in {@link
* HoodieRecord} before deflation.
*/
- public void markSuccess(HoodieRecord record,
- Optional<Map<String, String>> optionalRecordMetadata) {
- writtenRecords.add(record);
+ public void markSuccess(HoodieRecord record, Optional<Map<String, String>>
optionalRecordMetadata) {
+ if (trackSuccessRecords) {
+ writtenRecords.add(record);
+ }
totalRecords++;
}
@@ -74,10 +88,11 @@ public class WriteStatus implements Serializable {
* @param optionalRecordMetadata optional metadata related to data contained
in {@link
* HoodieRecord} before deflation.
*/
- public void markFailure(HoodieRecord record, Throwable t,
- Optional<Map<String, String>> optionalRecordMetadata) {
- failedRecords.add(record);
- errors.put(record.getKey(), t);
+ public void markFailure(HoodieRecord record, Throwable t,
Optional<Map<String, String>> optionalRecordMetadata) {
+ if (random.nextDouble() <= failureFraction) {
+ failedRecords.add(record);
+ errors.put(record.getKey(), t);
+ }
totalRecords++;
totalErrorRecords++;
}
diff --git
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java
index 5746e8d..051ac16 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java
@@ -52,6 +52,12 @@ public class HoodieMemoryConfig extends DefaultHoodieConfig {
// Default file path prefix for spillable file
public static final String DEFAULT_SPILLABLE_MAP_BASE_PATH = "/tmp/";
+ // Property to control how what fraction of the failed record, exceptions we
report back to driver.
+ public static final String WRITESTATUS_FAILURE_FRACTION_PROP =
"hoodie.memory.writestatus.failure.fraction";
+ // Default is 10%. If set to 100%, with lot of failures, this can cause
memory pressure, cause OOMs and
+ // mask actual data errors.
+ public static final double DEFAULT_WRITESTATUS_FAILURE_FRACTION = 0.1;
+
private HoodieMemoryConfig(Properties props) {
super(props);
}
@@ -97,6 +103,11 @@ public class HoodieMemoryConfig extends DefaultHoodieConfig
{
return this;
}
+ public Builder withWriteStatusFailureFraction(double failureFraction) {
+ props.setProperty(WRITESTATUS_FAILURE_FRACTION_PROP,
String.valueOf(failureFraction));
+ return this;
+ }
+
/**
* Dynamic calculation of max memory to use for for spillable map.
user.available.memory = spark.executor.memory *
* (1 - spark.memory.fraction) spillable.available.memory =
user.available.memory * hoodie.memory.fraction. Anytime
@@ -118,8 +129,8 @@ public class HoodieMemoryConfig extends DefaultHoodieConfig
{
if (SparkEnv.get() != null) {
// 1 GB is the default conf used by Spark, look at SparkContext.scala
long executorMemoryInBytes =
Utils.memoryStringToMb(SparkEnv.get().conf().get(SPARK_EXECUTOR_MEMORY_PROP,
- DEFAULT_SPARK_EXECUTOR_MEMORY_MB)) * 1024
- * 1024L;
+ DEFAULT_SPARK_EXECUTOR_MEMORY_MB)) * 1024
+ * 1024L;
// 0.6 is the default value used by Spark,
// look at {@link
//
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/SparkConf.scala#L507}
@@ -159,6 +170,9 @@ public class HoodieMemoryConfig extends DefaultHoodieConfig
{
setDefaultOnCondition(props,
!props.containsKey(SPILLABLE_MAP_BASE_PATH_PROP),
SPILLABLE_MAP_BASE_PATH_PROP, DEFAULT_SPILLABLE_MAP_BASE_PATH);
+ setDefaultOnCondition(props,
+ !props.containsKey(WRITESTATUS_FAILURE_FRACTION_PROP),
+ WRITESTATUS_FAILURE_FRACTION_PROP,
String.valueOf(DEFAULT_WRITESTATUS_FAILURE_FRACTION));
return config;
}
}
diff --git
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
index 67b38d5..7156623 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
@@ -406,21 +406,21 @@ public class HoodieWriteConfig extends
DefaultHoodieConfig {
}
public Long getMaxMemoryPerCompaction() {
- return Long
- .valueOf(
-
props.getProperty(HoodieMemoryConfig.MAX_MEMORY_FOR_COMPACTION_PROP));
+ return
Long.valueOf(props.getProperty(HoodieMemoryConfig.MAX_MEMORY_FOR_COMPACTION_PROP));
}
public int getMaxDFSStreamBufferSize() {
- return Integer
- .valueOf(
-
props.getProperty(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE_PROP));
+ return
Integer.valueOf(props.getProperty(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE_PROP));
}
public String getSpillableMapBasePath() {
return props.getProperty(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH_PROP);
}
+ public double getWriteStatusFailureFraction() {
+ return
Double.valueOf(props.getProperty(HoodieMemoryConfig.WRITESTATUS_FAILURE_FRACTION_PROP));
+ }
+
public static class Builder {
private final Properties props = new Properties();
@@ -428,7 +428,6 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private boolean isStorageConfigSet = false;
private boolean isCompactionConfigSet = false;
private boolean isMetricsConfigSet = false;
- private boolean isAutoCommit = true;
private boolean isMemoryConfigSet = false;
public Builder fromFile(File propertiesFile) throws IOException {
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java
b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java
index b4a03d1..a1f8f63 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java
@@ -61,7 +61,9 @@ public abstract class HoodieIOHandle<T extends
HoodieRecordPayload> {
this.originalSchema = new Schema.Parser().parse(config.getSchema());
this.writerSchema = createHoodieWriteSchema(originalSchema);
this.timer = new HoodieTimer().startTimer();
- this.writeStatus =
ReflectionUtils.loadClass(config.getWriteStatusClassName());
+ this.writeStatus = (WriteStatus)
ReflectionUtils.loadClass(config.getWriteStatusClassName(),
+ !hoodieTable.getIndex().isImplicitWithStorage(),
+ config.getWriteStatusFailureFraction());
}
/**
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
index 282601d..2df095e 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import com.uber.hoodie.common.HoodieCleanStat;
import com.uber.hoodie.common.HoodieClientTestUtils;
import com.uber.hoodie.common.HoodieTestDataGenerator;
+import com.uber.hoodie.common.TestRawTripPayload;
import com.uber.hoodie.common.model.HoodiePartitionMetadata;
import com.uber.hoodie.common.model.HoodieRecord;
import com.uber.hoodie.common.model.HoodieTableType;
@@ -144,6 +145,7 @@ public class TestHoodieClientBase implements Serializable {
return
HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
.withParallelism(2, 2)
.withBulkInsertParallelism(2).withFinalizeWriteParallelism(2)
+
.withWriteStatusClass(TestRawTripPayload.MetadataMergeWriteStatus.class)
.withConsistencyCheckEnabled(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024
* 1024).build())
.withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024
* 1024).build())
diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java
new file mode 100644
index 0000000..08035ab
--- /dev/null
+++ b/hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019 Uber Technologies, Inc. ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.uber.hoodie;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.uber.hoodie.common.model.HoodieRecord;
+import java.io.IOException;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestWriteStatus {
+ @Test
+ public void testFailureFraction() throws IOException {
+ WriteStatus status = new WriteStatus(true, 0.1);
+ Throwable t = new Exception("some error in writing");
+ for (int i = 0; i < 1000; i++) {
+ status.markFailure(Mockito.mock(HoodieRecord.class), t, null);
+ }
+ assertTrue(status.getFailedRecords().size() > 0);
+ assertTrue(status.getFailedRecords().size() < 150); //150 instead of 100,
to prevent flaky test
+ assertTrue(status.hasErrors());
+ }
+
+ @Test
+ public void testSuccessRecordTracking() {
+ WriteStatus status = new WriteStatus(false, 1.0);
+ Throwable t = new Exception("some error in writing");
+ for (int i = 0; i < 1000; i++) {
+ status.markSuccess(Mockito.mock(HoodieRecord.class), null);
+ status.markFailure(Mockito.mock(HoodieRecord.class), t, null);
+ }
+ assertEquals(1000, status.getFailedRecords().size());
+ assertTrue(status.hasErrors());
+ assertTrue(status.getWrittenRecords().isEmpty());
+ assertEquals(2000, status.getTotalRecords());
+ }
+}
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java
b/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java
index b73cba7..e0a1916 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java
@@ -144,6 +144,10 @@ public class TestRawTripPayload implements
HoodieRecordPayload<TestRawTripPayloa
private Map<String, String> mergedMetadataMap = new HashMap<>();
+ public MetadataMergeWriteStatus(Boolean trackSuccessRecords, Double
failureFraction) {
+ super(trackSuccessRecords, failureFraction);
+ }
+
public static Map<String, String>
mergeMetadataForWriteStatuses(List<WriteStatus> writeStatuses) {
Map<String, String> allWriteStatusMergedMetadataMap = new HashMap<>();
for (WriteStatus writeStatus : writeStatuses) {
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java
b/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java
index f924669..16e67c4 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java
@@ -337,7 +337,7 @@ public class TestHbaseIndex {
}
private WriteStatus getSampleWriteStatus(final int numInserts, final int
numUpdateWrites) {
- final WriteStatus writeStatus = new WriteStatus();
+ final WriteStatus writeStatus = new WriteStatus(false, 0.1);
HoodieWriteStat hoodieWriteStat = new HoodieWriteStat();
hoodieWriteStat.setNumInserts(numInserts);
hoodieWriteStat.setNumUpdateWrites(numUpdateWrites);
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java
b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java
index 3db10a4..714f0e5 100644
---
a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java
+++
b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java
@@ -342,10 +342,10 @@ public class TestCopyOnWriteTable {
assertEquals(2, returnedStatuses.size());
assertEquals("2016/01/31", returnedStatuses.get(0).getPartitionPath());
assertEquals(0, returnedStatuses.get(0).getFailedRecords().size());
- assertEquals(10, returnedStatuses.get(0).getWrittenRecords().size());
+ assertEquals(10, returnedStatuses.get(0).getTotalRecords());
assertEquals("2016/02/01", returnedStatuses.get(1).getPartitionPath());
assertEquals(0, returnedStatuses.get(0).getFailedRecords().size());
- assertEquals(1, returnedStatuses.get(1).getWrittenRecords().size());
+ assertEquals(1, returnedStatuses.get(1).getTotalRecords());
// Case 2:
// 1 record for partition 1, 5 record for partition 2, 1 records for
partition 3.
@@ -358,13 +358,13 @@ public class TestCopyOnWriteTable {
assertEquals(3, returnedStatuses.size());
assertEquals("2016/01/31", returnedStatuses.get(0).getPartitionPath());
- assertEquals(1, returnedStatuses.get(0).getWrittenRecords().size());
+ assertEquals(1, returnedStatuses.get(0).getTotalRecords());
assertEquals("2016/02/01", returnedStatuses.get(1).getPartitionPath());
- assertEquals(5, returnedStatuses.get(1).getWrittenRecords().size());
+ assertEquals(5, returnedStatuses.get(1).getTotalRecords());
assertEquals("2016/02/02", returnedStatuses.get(2).getPartitionPath());
- assertEquals(1, returnedStatuses.get(2).getWrittenRecords().size());
+ assertEquals(1, returnedStatuses.get(2).getTotalRecords());
}