This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 3efe6ff [improve] improve batch iterator has next check (#164)
3efe6ff is described below
commit 3efe6ffca0bf033aadcb558b49c2db35f9de6480
Author: gnehil <[email protected]>
AuthorDate: Mon Nov 27 11:12:22 2023 +0800
[improve] improve batch iterator has next check (#164)
---
.../scala/org/apache/doris/spark/writer/DorisWriter.scala | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/writer/DorisWriter.scala
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/writer/DorisWriter.scala
index 0485671..53187f4 100644
---
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/writer/DorisWriter.scala
+++
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/writer/DorisWriter.scala
@@ -150,7 +150,17 @@ class DorisWriter(settings: SparkSettings,
preCommittedTxnAcc: CollectionAccumul
private var isReset = false
- override def hasNext: Boolean = recordCount < batchSize && iterator.hasNext
+ override def hasNext: Boolean = {
+ if (recordCount < batchSize) {
+ if (isReset) {
+ recordCount < buffer.size
+ } else {
+ iterator.hasNext
+ }
+ } else {
+ false
+ }
+ }
override def next(): T = {
recordCount += 1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
