This is an automated email from the ASF dual-hosted git repository. diwu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push: new 61852e8 [Improvement]Put generateSerializedResult in try catch to avoid insufficient memory caused by excessive data volume (#202) 61852e8 is described below commit 61852e8e26780a9d40dcdb946a391f19c354a9c2 Author: LXW <1264174...@qq.com> AuthorDate: Wed May 15 09:56:35 2024 +0800 [Improvement]Put generateSerializedResult in try catch to avoid insufficient memory caused by excessive data volume (#202) --- .../src/main/java/org/apache/doris/spark/util/ListUtils.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java b/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java index fbfab9a..c1fe5a5 100644 --- a/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java +++ b/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java @@ -44,13 +44,12 @@ public class ListUtils { * recursively splits large collections to normal collection and serializes the collection * @param batch * @param result - * @throws JsonProcessingException */ - public static void divideAndSerialize(List<Map<Object, Object>> batch, List<String> result, String lineDelimiter) - throws JsonProcessingException { - String serializedResult = generateSerializedResult(batch, lineDelimiter); + public static void divideAndSerialize(List<Map<Object, Object>> batch, List<String> result, String lineDelimiter) { + // if an error occurred in the batch call to getBytes ,average divide the batch try { + String serializedResult = generateSerializedResult(batch, lineDelimiter); //the "Requested array size exceeds VM limit" exception occurs when the collection is large serializedResult.getBytes("UTF-8"); result.add(serializedResult); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org