This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0d76c16d14d [fix](routine load) reduce routine load task consume log
(#42058) (#42268)
0d76c16d14d is described below
commit 0d76c16d14d6731d11858ea6a7a4cb8010a59f79
Author: hui lai <[email protected]>
AuthorDate: Wed Oct 23 23:30:15 2024 +0800
[fix](routine load) reduce routine load task consume log (#42058) (#42268)
pick (#42058)
There is too much routine load task log, five million logs were
generated in 10 minutes.
```
grep 'consumer meet partition eof' be.INFO.log.20240930-164533 | wc -l
5369624
```
---
be/src/runtime/routine_load/data_consumer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/be/src/runtime/routine_load/data_consumer.cpp
b/be/src/runtime/routine_load/data_consumer.cpp
index 17ceee5d4c8..c6de4849d32 100644
--- a/be/src/runtime/routine_load/data_consumer.cpp
+++ b/be/src/runtime/routine_load/data_consumer.cpp
@@ -261,12 +261,13 @@ Status
KafkaDataConsumer::group_consume(BlockingQueue<RdKafka::Message*>* queue,
}
[[fallthrough]];
case RdKafka::ERR__PARTITION_EOF: {
- LOG(INFO) << "consumer meet partition eof: " << _id
- << " partition offset: " << msg->offset();
+ VLOG_NOTICE << "consumer meet partition eof: " << _id
+ << " partition offset: " << msg->offset();
_consuming_partition_ids.erase(msg->partition());
if (!queue->blocking_put(msg.get())) {
done = true;
} else if (_consuming_partition_ids.size() <= 0) {
+ LOG(INFO) << "all partitions meet eof: " << _id;
msg.release();
done = true;
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
