This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 61852e8 [Improvement]Put generateSerializedResult in try catch to
avoid insufficient memory caused by excessive data volume (#202)
61852e8 is described below
commit 61852e8e26780a9d40dcdb946a391f19c354a9c2
Author: LXW <[email protected]>
AuthorDate: Wed May 15 09:56:35 2024 +0800
[Improvement]Put generateSerializedResult in try catch to avoid
insufficient memory caused by excessive data volume (#202)
---
.../src/main/java/org/apache/doris/spark/util/ListUtils.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git
a/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java
b/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java
index fbfab9a..c1fe5a5 100644
---
a/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java
+++
b/spark-doris-connector/src/main/java/org/apache/doris/spark/util/ListUtils.java
@@ -44,13 +44,12 @@ public class ListUtils {
* recursively splits large collections to normal collection and
serializes the collection
* @param batch
* @param result
- * @throws JsonProcessingException
*/
- public static void divideAndSerialize(List<Map<Object, Object>> batch,
List<String> result, String lineDelimiter)
- throws JsonProcessingException {
- String serializedResult = generateSerializedResult(batch,
lineDelimiter);
+ public static void divideAndSerialize(List<Map<Object, Object>> batch,
List<String> result, String lineDelimiter) {
+
// if an error occurred in the batch call to getBytes ,average divide
the batch
try {
+ String serializedResult = generateSerializedResult(batch,
lineDelimiter);
//the "Requested array size exceeds VM limit" exception occurs
when the collection is large
serializedResult.getBytes("UTF-8");
result.add(serializedResult);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
