This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 17919f8aef [MINOR][VL] Bound Kafka consume waits by the remaining poll
timeout (#13449)
17919f8aef is described below
commit 17919f8aefbda14dab15c04c68e57111eab644ed
Author: Reema <[email protected]>
AuthorDate: Thu Nov 12 16:44:47 2026 +0300
[MINOR][VL] Bound Kafka consume waits by the remaining poll timeout (#13449)
---
cpp/velox/operators/reader/KafkaDataSource.cc | 47 +++--
cpp/velox/operators/reader/KafkaDataSource.h | 7 +
cpp/velox/tests/CMakeLists.txt | 7 +-
gluten-ut/native/KafkaPollTimeoutTest.cc | 283 ++++++++++++++++++++++++++
4 files changed, 331 insertions(+), 13 deletions(-)
diff --git a/cpp/velox/operators/reader/KafkaDataSource.cc
b/cpp/velox/operators/reader/KafkaDataSource.cc
index 20c1ae575d..2aedf56504 100644
--- a/cpp/velox/operators/reader/KafkaDataSource.cc
+++ b/cpp/velox/operators/reader/KafkaDataSource.cc
@@ -194,14 +194,42 @@ std::optional<RowVectorPtr>
KafkaDataSource::next(uint64_t size, ContinueFuture&
return nullptr;
}
+std::chrono::steady_clock::time_point KafkaDataSource::pollTime() const {
+ return std::chrono::steady_clock::now();
+}
+
+RdKafka::Message* KafkaDataSource::consume(int timeoutMs) {
+ return consumer_->consume(timeoutMs);
+}
+
std::vector<std::unique_ptr<RdKafka::Message>> KafkaDataSource::poll(uint64_t
maxRows) {
std::vector<std::unique_ptr<RdKafka::Message>> messages;
const auto pollTimeoutMs = split_->pollTimeoutMs > 0 ? split_->pollTimeoutMs
: kDefaultPollTimeoutMs;
- const auto deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(pollTimeoutMs);
+ const auto deadline = pollTime() + std::chrono::milliseconds(pollTimeoutMs);
+ const auto failOnTimeout = [&]() {
+ VELOX_FAIL(
+ "Timed out after {} ms fetching offset {} of {}-{}",
+ pollTimeoutMs,
+ nextOffset_,
+ split_->topic,
+ split_->partition);
+ };
+ // Bound record-consumption waits by the time left in this poll. Metadata
and recovery
+ // operations retain their separate timeout allowances.
+ const auto remainingTimeoutMs = [this, &deadline]() {
+ return static_cast<int>(std::clamp<int64_t>(
+ std::chrono::duration_cast<std::chrono::milliseconds>(deadline -
pollTime()).count(),
+ 0,
+ std::numeric_limits<int>::max()));
+ };
while (!splitDone_ && messages.size() < maxRows) {
+ const auto timeoutMs = remainingTimeoutMs();
+ if (timeoutMs == 0) {
+ break;
+ }
// Only wait for more records while the batch is empty.
- std::unique_ptr<RdKafka::Message> msg(consumer_->consume(messages.empty()
? kConsumeTimeoutMs : 0));
+ std::unique_ptr<RdKafka::Message> msg(consume(messages.empty() ?
std::min(kConsumeTimeoutMs, timeoutMs) : 0));
switch (msg->err()) {
case RdKafka::ERR_NO_ERROR: {
const auto offset = msg->offset();
@@ -283,18 +311,13 @@ std::vector<std::unique_ptr<RdKafka::Message>>
KafkaDataSource::poll(uint64_t ma
break;
}
- if (!splitDone_ && std::chrono::steady_clock::now() >= deadline) {
- if (!messages.empty()) {
- return messages;
- }
- VELOX_FAIL(
- "Timed out after {} ms fetching offset {} of {}-{}",
- pollTimeoutMs,
- nextOffset_,
- split_->topic,
- split_->partition);
+ if (!splitDone_ && pollTime() >= deadline) {
+ break;
}
}
+ if (!splitDone_ && messages.empty()) {
+ failOnTimeout();
+ }
return messages;
}
diff --git a/cpp/velox/operators/reader/KafkaDataSource.h
b/cpp/velox/operators/reader/KafkaDataSource.h
index 59484b5e2a..22dde133ce 100644
--- a/cpp/velox/operators/reader/KafkaDataSource.h
+++ b/cpp/velox/operators/reader/KafkaDataSource.h
@@ -17,6 +17,7 @@
#pragma once
+#include <chrono>
#include <memory>
#include <string>
#include <vector>
@@ -64,6 +65,12 @@ class KafkaDataSource : public
facebook::velox::connector::DataSource {
private:
friend class KafkaConnectorTest;
+ friend class KafkaPollTimeoutArgumentTest;
+
+ // Keep time and record consumption replaceable for deterministic timeout
tests.
+ virtual std::chrono::steady_clock::time_point pollTime() const;
+
+ virtual RdKafka::Message* consume(int timeoutMs);
// Polls up to 'maxRows' messages of the current split. Sets 'splitDone_'
once the end offset is
// reached.
diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt
index a45420d395..d1685a8b34 100644
--- a/cpp/velox/tests/CMakeLists.txt
+++ b/cpp/velox/tests/CMakeLists.txt
@@ -182,5 +182,10 @@ if(ENABLE_ENHANCED_FEATURES)
endif()
if(ENABLE_KAFKA)
- add_velox_test(velox_kafka_connector_test SOURCES KafkaConnectorTest.cc)
+ add_velox_test(
+ velox_kafka_connector_test
+ SOURCES
+ KafkaConnectorTest.cc
+
${CMAKE_CURRENT_SOURCE_DIR}/../../../gluten-ut/native/KafkaPollTimeoutTest.cc
+ )
endif()
diff --git a/gluten-ut/native/KafkaPollTimeoutTest.cc
b/gluten-ut/native/KafkaPollTimeoutTest.cc
new file mode 100644
index 0000000000..45c3324319
--- /dev/null
+++ b/gluten-ut/native/KafkaPollTimeoutTest.cc
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <chrono>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include <librdkafka/rdkafka.h>
+#include <librdkafka/rdkafka_mock.h>
+#include <librdkafka/rdkafkacpp.h>
+
+#include "operators/reader/KafkaConnector.h"
+#include "operators/reader/KafkaDataSource.h"
+#include "velox/common/base/tests/GTestUtils.h"
+#include "velox/vector/tests/utils/VectorTestBase.h"
+
+using namespace facebook::velox;
+
+namespace gluten {
+namespace {
+
+constexpr char kConnectorId[] = "kafka-poll-timeout-test";
+constexpr char kTopic[] = "kafka-poll-timeout";
+constexpr int64_t kPollTimeoutMs = 10;
+
+} // namespace
+
+class KafkaPollTimeoutTest : public ::testing::Test,
+ public test::VectorTestBase {
+protected:
+ static void SetUpTestCase() {
+
memory::MemoryManager::testingSetInstance(memory::MemoryManager::Options{});
+ }
+
+ void SetUp() override {
+ auto *conf = rd_kafka_conf_new();
+ char error[512];
+ ASSERT_EQ(rd_kafka_conf_set(conf, "test.mock.num.brokers", "1", error,
+ sizeof(error)),
+ RD_KAFKA_CONF_OK);
+ producer_ = rd_kafka_new(RD_KAFKA_PRODUCER, conf, error, sizeof(error));
+ ASSERT_NE(producer_, nullptr) << error;
+ cluster_ = rd_kafka_handle_mock_cluster(producer_);
+ ASSERT_NE(cluster_, nullptr);
+ ASSERT_EQ(rd_kafka_mock_topic_create(cluster_, kTopic, 1, 1),
+ RD_KAFKA_RESP_ERR_NO_ERROR);
+ // Keep broker responses outside both the configured timeout and the old
100
+ // ms consume wait.
+ ASSERT_EQ(rd_kafka_mock_broker_set_rtt(cluster_, 1, 1'000),
+ RD_KAFKA_RESP_ERR_NO_ERROR);
+ }
+
+ void TearDown() override {
+ if (producer_ != nullptr) {
+ rd_kafka_destroy(producer_);
+ }
+ }
+
+ std::unique_ptr<KafkaDataSource> source() {
+ auto type = ROW({"offset"}, {BIGINT()});
+ connector::ColumnHandleMap columns{
+ {"offset", std::make_shared<KafkaColumnHandle>("offset", BIGINT())}};
+ return std::make_unique<KafkaDataSource>(type, columns, pool());
+ }
+
+ std::shared_ptr<KafkaConnectorSplit> split(int64_t endOffset) {
+ std::unordered_map<std::string, std::string> params{
+ {"bootstrap.servers", rd_kafka_mock_cluster_bootstraps(cluster_)}};
+ return std::make_shared<KafkaConnectorSplit>(kConnectorId, kTopic, 0, 0,
+ endOffset, kPollTimeoutMs,
+ true, std::move(params));
+ }
+
+ rd_kafka_t *producer_{nullptr};
+ rd_kafka_mock_cluster_t *cluster_{nullptr};
+};
+
+TEST_F(KafkaPollTimeoutTest, shortPollTimeoutReportsTimeout) {
+ auto reader = source();
+ reader->addSplit(split(1));
+ ContinueFuture future = ContinueFuture::makeEmpty();
+ VELOX_ASSERT_THROW(reader->next(1, future),
+ "Timed out after 10 ms fetching offset");
+ EXPECT_EQ(reader->getCompletedRows(), 0);
+ EXPECT_EQ(reader->getCompletedBytes(), 0);
+}
+
+TEST_F(KafkaPollTimeoutTest, emptyRangeDoesNotWaitForBroker) {
+ auto reader = source();
+ reader->addSplit(split(0));
+ ContinueFuture future = ContinueFuture::makeEmpty();
+ EXPECT_EQ(reader->next(1, future).value(), nullptr);
+ EXPECT_EQ(reader->getCompletedRows(), 0);
+ EXPECT_EQ(reader->getCompletedBytes(), 0);
+}
+
+namespace {
+
+// Supply only the message fields used by poll(), without involving a broker or
+// vector allocation.
+class PollMessage final : public RdKafka::Message {
+public:
+ PollMessage(RdKafka::ErrorCode error, int64_t offset)
+ : error_(error), offset_(offset) {}
+
+ RdKafka::ErrorCode err() const override { return error_; }
+
+ std::string errstr() const override { return RdKafka::err2str(error_); }
+
+ int64_t offset() const override { return offset_; }
+
+ size_t len() const override { return 3; }
+
+ size_t key_len() const override { return 2; }
+
+ RdKafka::Topic *topic() const override { VELOX_UNREACHABLE(); }
+
+ std::string topic_name() const override { VELOX_UNREACHABLE(); }
+
+ int32_t partition() const override { VELOX_UNREACHABLE(); }
+
+ void *payload() const override { VELOX_UNREACHABLE(); }
+
+ const std::string *key() const override { VELOX_UNREACHABLE(); }
+
+ const void *key_pointer() const override { VELOX_UNREACHABLE(); }
+
+ RdKafka::MessageTimestamp timestamp() const override { VELOX_UNREACHABLE(); }
+
+ void *msg_opaque() const override { VELOX_UNREACHABLE(); }
+
+ int64_t latency() const override { VELOX_UNREACHABLE(); }
+
+ rd_kafka_message_t *c_ptr() override { VELOX_UNREACHABLE(); }
+
+ Status status() const override { VELOX_UNREACHABLE(); }
+
+ RdKafka::Headers *headers() override { VELOX_UNREACHABLE(); }
+
+ RdKafka::Headers *headers(RdKafka::ErrorCode * /*error*/) override {
+ VELOX_UNREACHABLE();
+ }
+
+ int32_t broker_id() const override { VELOX_UNREACHABLE(); }
+
+ int32_t leader_epoch() const override { VELOX_UNREACHABLE(); }
+
+ RdKafka::Error *offset_store() override { VELOX_UNREACHABLE(); }
+
+private:
+ const RdKafka::ErrorCode error_;
+ const int64_t offset_;
+};
+
+class ScriptedKafkaDataSource final : public KafkaDataSource {
+public:
+ using KafkaDataSource::KafkaDataSource;
+
+ struct Event {
+ RdKafka::ErrorCode error;
+ int64_t offset;
+ std::chrono::steady_clock::duration elapsed;
+ };
+
+ std::deque<Event> events;
+ std::vector<int> consumeTimeouts;
+
+private:
+ std::chrono::steady_clock::time_point pollTime() const override {
+ return now_;
+ }
+
+ RdKafka::Message *consume(int timeoutMs) override {
+ consumeTimeouts.push_back(timeoutMs);
+ VELOX_CHECK(!events.empty(), "Unexpected consume call");
+ const auto event = events.front();
+ events.pop_front();
+ now_ += event.elapsed;
+ return new PollMessage(event.error, event.offset);
+ }
+
+ std::chrono::steady_clock::time_point now_{};
+};
+
+} // namespace
+
+// Exercise the production poll loop with a controlled clock and recorded I/O
+// arguments. These tests neither sleep nor use wall-clock tolerances.
+class KafkaPollTimeoutArgumentTest : public ::testing::Test,
+ public test::VectorTestBase {
+protected:
+ static void SetUpTestCase() {
+
memory::MemoryManager::testingSetInstance(memory::MemoryManager::Options{});
+ }
+
+ std::unique_ptr<ScriptedKafkaDataSource> source(int64_t timeoutMs,
+ int64_t endOffset = 3) {
+ auto type = ROW({"offset"}, {BIGINT()});
+ connector::ColumnHandleMap columns{
+ {"offset", std::make_shared<KafkaColumnHandle>("offset", BIGINT())}};
+ auto reader =
+ std::make_unique<ScriptedKafkaDataSource>(type, columns, pool());
+ reader->split_ = std::make_shared<KafkaConnectorSplit>(
+ kConnectorId, kTopic, 0, 0, endOffset, timeoutMs, false,
+ std::unordered_map<std::string, std::string>{});
+ return reader;
+ }
+
+ std::vector<std::unique_ptr<RdKafka::Message>> poll(KafkaDataSource &reader,
+ uint64_t maxRows = 1) {
+ return reader.poll(maxRows);
+ }
+
+ bool splitDone(const KafkaDataSource &reader) { return reader.splitDone_; }
+
+ int64_t nextOffset(const KafkaDataSource &reader) {
+ return reader.nextOffset_;
+ }
+};
+
+TEST_F(KafkaPollTimeoutArgumentTest, consumeUsesExactShortTimeout) {
+ auto reader = source(10);
+ reader->events = {
+ {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(10)}};
+ VELOX_ASSERT_THROW(poll(*reader), "Timed out after 10 ms fetching offset");
+ EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({10}));
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, successiveConsumesShareDeadline) {
+ auto reader = source(250);
+ reader->events = {
+ {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(100)},
+ {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(100)},
+ {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(50)}};
+ VELOX_ASSERT_THROW(poll(*reader), "Timed out after 250 ms fetching offset");
+ EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100, 100, 50}));
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, collectedRecordsUseNonBlockingConsume) {
+ auto reader = source(300);
+ reader->events = {
+ {RdKafka::ERR_NO_ERROR, 0, std::chrono::milliseconds(25)},
+ {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(0)}};
+ const auto messages = poll(*reader, 2);
+ ASSERT_EQ(messages.size(), 1);
+ EXPECT_EQ(messages.front()->offset(), 0);
+ EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100, 0}));
+ EXPECT_EQ(reader->getCompletedBytes(), 5);
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, expiredBudgetReturnsCollectedRecord) {
+ auto reader = source(300);
+ reader->events = {{RdKafka::ERR_NO_ERROR, 0,
std::chrono::milliseconds(301)}};
+ const auto messages = poll(*reader, 2);
+ ASSERT_EQ(messages.size(), 1);
+ EXPECT_EQ(messages.front()->offset(), 0);
+ EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100}));
+ EXPECT_EQ(reader->getCompletedBytes(), 5);
+ EXPECT_EQ(nextOffset(*reader), 1);
+ EXPECT_FALSE(splitDone(*reader));
+}
+
+} // namespace gluten
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]