This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9f5d16529d [HUDI-4980] Calculate avg record size using commit only
(#6864)
9f5d16529d is described below
commit 9f5d16529d3e347097518bc18caefad5fbe27ac5
Author: Shiyan Xu <[email protected]>
AuthorDate: Wed Oct 5 18:59:19 2022 +0800
[HUDI-4980] Calculate avg record size using commit only (#6864)
Calculate average record size for Spark upsert partitioner
based on commit instants only. Previously it's based on
commit and replacecommit, of which the latter may be
created by clustering which has inaccurately smaller
average record sizes, which could result in OOM
due to size underestimation.
---
.../hudi/table/action/commit/UpsertPartitioner.java | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
index c2f5a43066..134cfd8d2c 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
@@ -36,6 +37,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
+
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
@@ -54,6 +56,8 @@ import java.util.stream.Collectors;
import scala.Tuple2;
+import static
org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+
/**
* Packs incoming records to be upserted, into buckets (1 bucket = 1 RDD
partition).
*/
@@ -158,13 +162,17 @@ public class UpsertPartitioner<T extends
HoodieRecordPayload<T>> extends SparkHo
private void assignInserts(WorkloadProfile profile, HoodieEngineContext
context) {
// for new inserts, compute buckets depending on how many records we have
for each partition
Set<String> partitionPaths = profile.getPartitionPaths();
- long averageRecordSize =
-
averageBytesPerRecord(table.getMetaClient().getActiveTimeline().getCommitTimeline().filterCompletedInstants(),
- config);
+ /*
+ * NOTE: we only use commit instants to calculate average record size
because replacecommit can be
+ * created by clustering, which has smaller average record size, which
affects assigning inserts and
+ * may result in OOM by making spark underestimate the actual input record
sizes.
+ */
+ long averageRecordSize =
averageBytesPerRecord(table.getMetaClient().getActiveTimeline()
+
.getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION)).filterCompletedInstants(),
config);
LOG.info("AvgRecordSize => " + averageRecordSize);
Map<String, List<SmallFile>> partitionSmallFilesMap =
- getSmallFilesForPartitions(new ArrayList<String>(partitionPaths),
context);
+ getSmallFilesForPartitions(new ArrayList<>(partitionPaths), context);
Map<String, Set<String>> partitionPathToPendingClusteringFileGroupsId =
getPartitionPathToPendingClusteringFileGroupsId();