numinnex commented on code in PR #4400:
URL: https://github.com/apache/iggy/pull/4400#discussion_r4194767994
##########
gateways/kafka/src/bridge/iggy_bridge/topics.rs:
##########
@@ -257,6 +257,31 @@ impl IggyBridge {
}
}
+ /// Deletes the Iggy topic backing `kafka_topic`, resolved through the
configured
+ /// [`TopicMapping`](crate::bridge::topic_map::TopicMapping).
+ ///
+ /// Deliberately leaves the backing Iggy stream alone, even when this
delete empties it out:
+ /// `TopicMapping::resolve` sends every *unmapped* Kafka topic to the same
`default_stream`
+ /// (`ensure_stream_and_topic`'s own doc comment flags the same sharing
risk for create), so a
+ /// stream this call happens to empty may still be the live home for a
different Kafka topic a
+ /// moment later - this call has no way to tell the two cases apart, and
real Kafka itself has
+ /// no analogous container object for a `DeleteTopics` to clean up.
+ ///
+ /// # Errors
+ ///
+ /// Returns [`BridgeError::InvalidKafkaTopicName`] if `kafka_topic` fails
Kafka's own
+ /// topic-naming rules. Returns [`BridgeError::Timeout`] if a call takes
longer than
+ /// `REQUEST_TIMEOUT`. Returns [`BridgeError::Iggy`] with
`StreamIdNotFound` (1009) or
+ /// `TopicIdNotFound` (2010) if the mapped stream or topic doesn't exist,
or for
+ /// connectivity/auth failures.
+ pub async fn delete_kafka_topic(&self, kafka_topic: &str) -> Result<(),
BridgeError> {
Review Comment:
this irreversible delete runs as the bridge's Iggy user and the Kafka
principal is discarded, so with SASL on any authenticated principal can delete
any topic in the default/override streams, even one DescribeAcls shows without
DELETE. The README warning only covers SASL off, which implies SASL on protects
it.
With SASL on, gate on `principal.permissions.manage_topics` (auth.rs:512),
answer 29, and fail closed when permissions aren't known. A kill switch
answering 73 `TOPIC_DELETION_DISABLED` would be worth having too.
##########
gateways/kafka/src/protocol/handlers/delete_topics.rs:
##########
@@ -0,0 +1,307 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `DeleteTopics` (API key 20).
+//!
+//! Deletes the Iggy topic a Kafka topic name resolves to. Never deletes the
backing Iggy
+//! stream - see [`IggyBridge::delete_kafka_topic`]'s own doc comment for why:
the shared
+//! `default_stream` can hold other Kafka topics' data, and this bridge has no
way to tell
+//! whether the stream it just emptied is actually abandoned or just
temporarily topic-less.
+//!
+//! `DeletableTopicResult.error_message` is only encoded on the wire at v5+
(`kafka_protocol`'s
+//! own gate) - a client negotiating v1-v4 gets `error_code` only, with any
text this module sets
+//! silently dropped. Unlike `CreateTopics`, whose equivalent field is encoded
at every version it
+//! supports.
+
+use bytes::Bytes;
+use iggy::prelude::IggyError;
+use kafka_protocol::messages::delete_topics_response::DeletableTopicResult;
+use kafka_protocol::messages::{DeleteTopicsRequest, DeleteTopicsResponse,
TopicName};
+use kafka_protocol::protocol::StrBytes;
+
+use tokio::time::Instant;
+
+use crate::bridge::{BridgeError, IggyBridge};
+use crate::error::Result;
+use crate::protocol::api::{
+ API_KEY_DELETE_TOPICS, ApiVersionRange, ERROR_INVALID_REQUEST, ERROR_NONE,
+ ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, ERROR_REQUEST_TIMED_OUT,
GatewayState,
+ HandleOutcome,
+};
+use crate::protocol::bounds_guard::validate_delete_topics_shape;
+use crate::protocol::handlers::{
+ clamp_request_timeout, decode_guarded, encode_message,
handle_versioned_request,
+ is_supported_version, respond_or_close, unsupported_version_response,
+};
+
+/// This bridge never advertises v6 (the `topics: Vec<DeleteTopicState>`,
topic-id-based shape).
+///
+/// It has no concept of a Kafka topic id, only the Kafka-side name
`TopicMapping` resolves. v1
+/// is the crate's own floor (`kafka_protocol` does not implement v0 for this
message).
+pub const RANGE: ApiVersionRange = ApiVersionRange {
+ api_key: API_KEY_DELETE_TOPICS,
+ min_version: 1,
+ max_version: 5,
+};
+
+/// Cap on topic names one `DeleteTopics` request may address through the
bridge.
+///
+/// Same rationale as `create_topics::MAX_BRIDGE_BACKED_TOPICS`, but counted
differently: every
+/// name here costs its own bridge round trip against the single lockstep
`IggyClient` every
+/// Kafka connection on this gateway shares, duplicates included, so this
charges the raw count
+/// rather than the distinct one `CreateTopics` charges.
+const MAX_BRIDGE_BACKED_TOPICS: usize = 100;
+
+pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) ->
HandleOutcome {
+ let Some(bridge) = &state.bridge else {
+ return handle_versioned_request(
+ API_KEY_DELETE_TOPICS,
+ api_version,
+ body,
+ |v, b| {
+ decode_guarded::<DeleteTopicsRequest>(v, b, |v, b| {
+ validate_delete_topics_shape(v, b, state.max_frame_size)
+ })
+ },
+ encode_response,
+ encode_error_response,
+ "DeleteTopics",
+ );
+ };
+
+ if !is_supported_version(API_KEY_DELETE_TOPICS, api_version) {
+ return unsupported_version_response(API_KEY_DELETE_TOPICS,
api_version, |version| {
+ encode_error_response(version, ERROR_INVALID_REQUEST)
+ });
+ }
+
+ let req = match decode_guarded::<DeleteTopicsRequest>(api_version, body,
|v, b| {
+ validate_delete_topics_shape(v, b, state.max_frame_size)
+ }) {
+ Ok(req) => req,
+ Err(error) => {
+ // debug!, not warn!: attacker-controlled, not operator-actionable.
+ tracing::debug!(%error, "Failed to decode DeleteTopics request");
+ return respond_or_close(
+ encode_error_response(api_version, ERROR_INVALID_REQUEST),
+ "DeleteTopics",
+ );
+ }
+ };
+
+ // RANGE caps at v5, so req.topics (the v6 topic-id shape) is always empty
- topic_names is
+ // the only populated field at any version this bridge advertises.
+ if req.topic_names.len() > MAX_BRIDGE_BACKED_TOPICS {
+ tracing::warn!(
+ requested_topics = req.topic_names.len(),
+ max = MAX_BRIDGE_BACKED_TOPICS,
+ "DeleteTopics request addresses too many topics; rejecting"
+ );
+ let message = StrBytes::from(format!(
+ "this gateway addresses at most {MAX_BRIDGE_BACKED_TOPICS} topic
names per DeleteTopics request"
+ ));
+ let results = req
+ .topic_names
+ .iter()
+ .map(|name| {
+ DeletableTopicResult::default()
+ .with_name(Some(name.clone()))
+ .with_error_code(ERROR_POLICY_VIOLATION)
+ .with_error_message(Some(message.clone()))
+ })
+ .collect();
+ let resp = DeleteTopicsResponse::default().with_responses(results);
+ return respond_or_close(encode_message(&resp, api_version, 256),
"DeleteTopics");
+ }
+
+ let deadline = Instant::now() + clamp_request_timeout(req.timeout_ms);
+ let results = delete_all_topics(bridge, &req.topic_names, deadline).await;
+ let resp = DeleteTopicsResponse::default().with_responses(results);
+ respond_or_close(encode_message(&resp, api_version, 256), "DeleteTopics")
+}
+
+/// Deletes every requested topic, independently of the others.
+///
+/// `deadline` bounds each topic's own bridge work individually
(`timeout_at`), not the whole
+/// loop - same reasoning as `create_topics::create_all_topics`: a single
`timeout` around the
+/// entire call would discard every already-resolved result the moment one
topic's call ran
+/// long, answering a retriable code even for topics that had already deleted
cleanly.
+///
+/// A name repeated within the same request is not specially rejected the way
`CreateTopics`
+/// rejects a duplicate: deleting the same topic twice has no race to protect
against the way
+/// creating it twice does - the first occurrence deletes it, the second then
answers
+/// `UNKNOWN_TOPIC_OR_PARTITION` because by the time it runs, the topic
genuinely doesn't exist
+/// any more. That's the correct answer for a duplicate, not a special case.
+async fn delete_all_topics(
+ bridge: &IggyBridge,
+ topic_names: &[TopicName],
+ deadline: Instant,
+) -> Vec<DeletableTopicResult> {
+ let mut results = Vec::with_capacity(topic_names.len());
+ for name in topic_names {
+ let result = match tokio::time::timeout_at(deadline,
delete_one_topic(bridge, name)).await {
Review Comment:
`timeout_at` polls the inner future before it checks the delay, and that
first poll already reaches the SDK's spawned write
(core/sdk/src/tcp/tcp_client.rs:1517). So every name past the deadline still
sends a real delete while the client is told `REQUEST_TIMED_OUT`. A late read
also drops the shared bridge stream (tcp_client.rs:1655), so every Kafka
connection's next Produce/Metadata gets `NotConnected` and a reconnect.
Check `Instant::now() >= deadline` before dispatching and answer 7 with no
call, like list_offsets.rs:196, and add a unit test. The comments at :139 and
:161 and SCOPE.md:252 need the same fix, and create_topics has the same shape.
##########
gateways/kafka/src/protocol/bounds_guard.rs:
##########
@@ -703,6 +703,44 @@ pub fn validate_create_topics_shape(
Ok(())
}
+/// Mirrors the field order `DeleteTopicsRequest::decode` walks, for the
`topic_names`-based
+/// shape (v1-v5) only.
+///
+/// This bridge's `RANGE` never advertises v6, the `topics:
Vec<DeleteTopicState>`
+/// (topic-id-based) shape, so that branch is never reached and isn't walked
here.
+///
+/// # Errors
+///
+/// Returns an error when a declared array/string/bytes length cannot fit in
the bytes remaining
+/// in the frame, or the body is truncated or malformed in a way that cannot
be walked.
+pub fn validate_delete_topics_shape(
+ version: i16,
+ body: &Bytes,
+ max_frame_size: usize,
+) -> Result<()> {
+ let mut c = ShapeCursor::new(body.clone(), max_frame_size);
+ let flexible = version >= 4;
Review Comment:
nothing pins v1-v4. Changing this to `version >= 5` survives all 548
no-server tests and would reject every v4 request (Java 2.4-2.6). Changing the
bridge-off stub at delete_topics.rs:265 from 41 to NONE, which makes a client
believe the delete happened, also survives. version_firewall_tests.rs:498 sends
only an empty request.
Add a no-server handler test with one name at v1/v3/v4/v5 that asserts a
per-topic 41.
##########
gateways/kafka/src/protocol/handlers/delete_topics.rs:
##########
@@ -0,0 +1,307 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `DeleteTopics` (API key 20).
+//!
+//! Deletes the Iggy topic a Kafka topic name resolves to. Never deletes the
backing Iggy
+//! stream - see [`IggyBridge::delete_kafka_topic`]'s own doc comment for why:
the shared
+//! `default_stream` can hold other Kafka topics' data, and this bridge has no
way to tell
+//! whether the stream it just emptied is actually abandoned or just
temporarily topic-less.
+//!
+//! `DeletableTopicResult.error_message` is only encoded on the wire at v5+
(`kafka_protocol`'s
+//! own gate) - a client negotiating v1-v4 gets `error_code` only, with any
text this module sets
+//! silently dropped. Unlike `CreateTopics`, whose equivalent field is encoded
at every version it
+//! supports.
+
+use bytes::Bytes;
+use iggy::prelude::IggyError;
+use kafka_protocol::messages::delete_topics_response::DeletableTopicResult;
+use kafka_protocol::messages::{DeleteTopicsRequest, DeleteTopicsResponse,
TopicName};
+use kafka_protocol::protocol::StrBytes;
+
+use tokio::time::Instant;
+
+use crate::bridge::{BridgeError, IggyBridge};
+use crate::error::Result;
+use crate::protocol::api::{
+ API_KEY_DELETE_TOPICS, ApiVersionRange, ERROR_INVALID_REQUEST, ERROR_NONE,
+ ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, ERROR_REQUEST_TIMED_OUT,
GatewayState,
+ HandleOutcome,
+};
+use crate::protocol::bounds_guard::validate_delete_topics_shape;
+use crate::protocol::handlers::{
+ clamp_request_timeout, decode_guarded, encode_message,
handle_versioned_request,
+ is_supported_version, respond_or_close, unsupported_version_response,
+};
+
+/// This bridge never advertises v6 (the `topics: Vec<DeleteTopicState>`,
topic-id-based shape).
+///
+/// It has no concept of a Kafka topic id, only the Kafka-side name
`TopicMapping` resolves. v1
+/// is the crate's own floor (`kafka_protocol` does not implement v0 for this
message).
+pub const RANGE: ApiVersionRange = ApiVersionRange {
+ api_key: API_KEY_DELETE_TOPICS,
+ min_version: 1,
+ max_version: 5,
+};
+
+/// Cap on topic names one `DeleteTopics` request may address through the
bridge.
+///
+/// Same rationale as `create_topics::MAX_BRIDGE_BACKED_TOPICS`, but counted
differently: every
+/// name here costs its own bridge round trip against the single lockstep
`IggyClient` every
+/// Kafka connection on this gateway shares, duplicates included, so this
charges the raw count
+/// rather than the distinct one `CreateTopics` charges.
+const MAX_BRIDGE_BACKED_TOPICS: usize = 100;
+
+pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) ->
HandleOutcome {
+ let Some(bridge) = &state.bridge else {
+ return handle_versioned_request(
+ API_KEY_DELETE_TOPICS,
+ api_version,
+ body,
+ |v, b| {
+ decode_guarded::<DeleteTopicsRequest>(v, b, |v, b| {
+ validate_delete_topics_shape(v, b, state.max_frame_size)
+ })
+ },
+ encode_response,
+ encode_error_response,
+ "DeleteTopics",
+ );
+ };
+
+ if !is_supported_version(API_KEY_DELETE_TOPICS, api_version) {
+ return unsupported_version_response(API_KEY_DELETE_TOPICS,
api_version, |version| {
+ encode_error_response(version, ERROR_INVALID_REQUEST)
+ });
+ }
+
+ let req = match decode_guarded::<DeleteTopicsRequest>(api_version, body,
|v, b| {
+ validate_delete_topics_shape(v, b, state.max_frame_size)
+ }) {
+ Ok(req) => req,
+ Err(error) => {
+ // debug!, not warn!: attacker-controlled, not operator-actionable.
+ tracing::debug!(%error, "Failed to decode DeleteTopics request");
+ return respond_or_close(
+ encode_error_response(api_version, ERROR_INVALID_REQUEST),
+ "DeleteTopics",
+ );
+ }
+ };
+
+ // RANGE caps at v5, so req.topics (the v6 topic-id shape) is always empty
- topic_names is
+ // the only populated field at any version this bridge advertises.
+ if req.topic_names.len() > MAX_BRIDGE_BACKED_TOPICS {
+ tracing::warn!(
+ requested_topics = req.topic_names.len(),
+ max = MAX_BRIDGE_BACKED_TOPICS,
+ "DeleteTopics request addresses too many topics; rejecting"
+ );
+ let message = StrBytes::from(format!(
+ "this gateway addresses at most {MAX_BRIDGE_BACKED_TOPICS} topic
names per DeleteTopics request"
+ ));
+ let results = req
+ .topic_names
+ .iter()
+ .map(|name| {
+ DeletableTopicResult::default()
+ .with_name(Some(name.clone()))
+ .with_error_code(ERROR_POLICY_VIOLATION)
+ .with_error_message(Some(message.clone()))
+ })
+ .collect();
+ let resp = DeleteTopicsResponse::default().with_responses(results);
+ return respond_or_close(encode_message(&resp, api_version, 256),
"DeleteTopics");
+ }
+
+ let deadline = Instant::now() + clamp_request_timeout(req.timeout_ms);
+ let results = delete_all_topics(bridge, &req.topic_names, deadline).await;
+ let resp = DeleteTopicsResponse::default().with_responses(results);
+ respond_or_close(encode_message(&resp, api_version, 256), "DeleteTopics")
+}
+
+/// Deletes every requested topic, independently of the others.
+///
+/// `deadline` bounds each topic's own bridge work individually
(`timeout_at`), not the whole
+/// loop - same reasoning as `create_topics::create_all_topics`: a single
`timeout` around the
+/// entire call would discard every already-resolved result the moment one
topic's call ran
+/// long, answering a retriable code even for topics that had already deleted
cleanly.
+///
+/// A name repeated within the same request is not specially rejected the way
`CreateTopics`
Review Comment:
KRaft `ControllerApis.deleteTopics` answers 42 "Duplicate topic name." for
every occurrence and deletes nothing. Here the first occurrence deletes the
topic, so the divergence goes the destructive way, and this crate's own
CreateTopics rejects duplicates (`find_duplicate_names`).
Pre-scan the request and answer 42 for each occurrence with no bridge call,
then update the real-bridge test (:204-228) and SCOPE.md:246.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
