This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch rework_topic_commands
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/rework_topic_commands by this
push:
new 8245d0d3c add limit to the max key value pair entries
8245d0d3c is described below
commit 8245d0d3c442efb2d4eda4fbedbded4f0e125bf0
Author: Grzegorz Koszyk <[email protected]>
AuthorDate: Fri Aug 14 07:36:59 2026 +0200
add limit to the max key value pair entries
---
core/binary_protocol/src/lib.rs | 4 +--
core/binary_protocol/src/primitives/options.rs | 46 +++++++++++++++-----------
core/common/src/types/options/mod.rs | 26 +++++++++++++++
foreign/node/src/wire/options.utils.ts | 34 +++++++++++--------
4 files changed, 74 insertions(+), 36 deletions(-)
diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs
index 72540d792..9d478f996 100644
--- a/core/binary_protocol/src/lib.rs
+++ b/core/binary_protocol/src/lib.rs
@@ -89,9 +89,7 @@ pub use message_view::{
pub use primitives::ack_level::AckLevel;
pub use primitives::consumer::{KIND_CONSUMER_GROUP, WireConsumer};
pub use primitives::identifier::{MAX_WIRE_NAME_LENGTH, WireIdentifier,
WireName};
-pub use primitives::options::{
- MAX_OPTION_KEY_LENGTH, MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions,
validate_options,
-};
+pub use primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions,
validate_options};
pub use primitives::partition_assignment::CreatedPartitionAssignment;
pub use primitives::partitioning::{MAX_MESSAGES_KEY_LENGTH, WirePartitioning};
pub use primitives::permissions::{
diff --git a/core/binary_protocol/src/primitives/options.rs
b/core/binary_protocol/src/primitives/options.rs
index e14724776..9b76423bd 100644
--- a/core/binary_protocol/src/primitives/options.rs
+++ b/core/binary_protocol/src/primitives/options.rs
@@ -36,13 +36,21 @@ use crate::primitives::user_headers::{
};
/// Maximum number of key-value entries in one options block.
-pub const MAX_OPTIONS: u32 = 64;
-
-/// Maximum byte length of an option key.
-pub const MAX_OPTION_KEY_LENGTH: usize = 64;
+pub const MAX_OPTIONS: u32 = 1024;
/// Maximum total byte length of an encoded options block.
-pub const MAX_OPTIONS_BYTES: usize = 4096;
+///
+/// Mirrors `iggy_common::MAX_USER_HEADERS_SIZE`: options ride the user-headers
+/// codec, so they inherit its budget rather than inventing a second one. The
+/// constant is duplicated because `iggy_common` depends on this crate, not the
+/// other way round; `options_block_budget_matches_user_headers` pins the two
+/// together from the side that can see both.
+///
+/// Sized so [`MAX_OPTIONS`] is actually reachable: the cheapest entry is 12
+/// bytes (kind + length + one byte, for key and value each), so 1024 entries
+/// need at least 12 KiB. A smaller budget would silently cap the entry count
+/// below `MAX_OPTIONS` and make that limit a lie.
+pub const MAX_OPTIONS_BYTES: usize = 100 * 1000;
/// Wire kind code for UTF-8 string fields, matching `HeaderKind::String`
/// in `iggy_common`.
@@ -54,7 +62,7 @@ const STRING_KIND: WireHeaderKind = WireHeaderKind(2);
///
/// - Total block size within [`MAX_OPTIONS_BYTES`]
/// - At most [`MAX_OPTIONS`] entries
-/// - Every key is a UTF-8 string of at most [`MAX_OPTION_KEY_LENGTH`] bytes
+/// - Every key is a UTF-8 string (length already bounded by the codec)
/// - No duplicate keys
///
/// Value kind codes are deliberately NOT restricted to the currently defined
@@ -92,12 +100,6 @@ pub fn validate_options(buf: &[u8]) -> Result<u32,
WireError> {
entry.key_kind.0
))));
}
- if entry.key.len() > MAX_OPTION_KEY_LENGTH {
- return Err(WireError::Validation(Cow::Owned(format!(
- "option key is {} bytes, exceeds maximum
{MAX_OPTION_KEY_LENGTH}",
- entry.key.len()
- ))));
- }
if std::str::from_utf8(entry.key).is_err() {
return Err(WireError::Validation(Cow::Borrowed(
"option key is not valid UTF-8",
@@ -321,16 +323,19 @@ mod tests {
}
#[test]
- fn key_over_length_limit_is_rejected() {
- let key = [b'k'; MAX_OPTION_KEY_LENGTH + 1];
- let buf = encode(&[(STRING, &key, STRING, b"value")]);
+ fn key_length_is_bounded_by_the_user_headers_codec() {
+ // Options add no key-length rule of their own: every field is already
+ // bounded to 1..=255 by `validate_user_headers`, so the codec's limit
+ // is the option key's limit.
+ let over = [b'k'; 256];
+ let buf = encode(&[(STRING, &over, STRING, b"value")]);
assert!(matches!(
validate_options(&buf),
Err(WireError::Validation(_))
));
- let max_key = [b'k'; MAX_OPTION_KEY_LENGTH];
- let buf = encode(&[(STRING, &max_key, STRING, b"value")]);
+ let at_limit = [b'k'; 255];
+ let buf = encode(&[(STRING, &at_limit, STRING, b"value")]);
assert_eq!(validate_options(&buf).unwrap(), 1);
}
@@ -353,8 +358,11 @@ mod tests {
#[test]
fn block_over_byte_limit_is_rejected() {
- let value = [b'v'; 250];
- let keys: Vec<String> = (0..16).map(|i| format!("key_{i}")).collect();
+ // 200 maximum-size entries are ~104 KB, over the byte budget while
+ // still well under `MAX_OPTIONS`, so this proves the byte cap trips
+ // independently of the entry count.
+ let value = [b'v'; 255];
+ let keys: Vec<String> = (0..200).map(|i|
format!("{i:0>255}")).collect();
let entries: Vec<(u8, &[u8], u8, &[u8])> = keys
.iter()
.map(|key| (STRING, key.as_bytes(), STRING, value.as_slice()))
diff --git a/core/common/src/types/options/mod.rs
b/core/common/src/types/options/mod.rs
index 584aa94d8..d84c3489b 100644
--- a/core/common/src/types/options/mod.rs
+++ b/core/common/src/types/options/mod.rs
@@ -1125,4 +1125,30 @@ mod tests {
);
assert!(parsed.max_topic_size.is_some());
}
+
+ #[test]
+ fn options_block_budget_matches_user_headers() {
+ // `MAX_OPTIONS_BYTES` duplicates `MAX_USER_HEADERS_SIZE` because
+ // `iggy_binary_protocol` cannot import this crate. This is the only
+ // place both are visible, so it is where they get tied together.
+ assert_eq!(
+ iggy_binary_protocol::MAX_OPTIONS_BYTES,
+ crate::MAX_USER_HEADERS_SIZE as usize,
+ "options inherit the user-headers byte budget; update both"
+ );
+ }
+
+ #[test]
+ fn max_options_is_reachable_within_the_byte_budget() {
+ // The cheapest entry is 12 bytes: kind + u32 length + one byte, for
+ // key and value each. If the budget ever drops below this product the
+ // entry cap becomes unreachable and `MAX_OPTIONS` stops being the
+ // limit that actually binds.
+ const MIN_ENTRY_BYTES: usize = 2 * (1 + 4 + 1);
+ assert!(
+ iggy_binary_protocol::MAX_OPTIONS as usize * MIN_ENTRY_BYTES
+ <= iggy_binary_protocol::MAX_OPTIONS_BYTES,
+ "MAX_OPTIONS entries must fit in MAX_OPTIONS_BYTES"
+ );
+ }
}
diff --git a/foreign/node/src/wire/options.utils.ts
b/foreign/node/src/wire/options.utils.ts
index 65c8c22be..cb38b2589 100644
--- a/foreign/node/src/wire/options.utils.ts
+++ b/foreign/node/src/wire/options.utils.ts
@@ -26,13 +26,22 @@ import {
} from './message/header.utils.js';
/** Maximum number of key-value entries in one options block. */
-export const MAX_OPTIONS = 64;
+export const MAX_OPTIONS = 1024;
-/** Maximum byte length of an option key. */
-export const MAX_OPTION_KEY_LENGTH = 64;
+/**
+ * Maximum total byte length of an encoded options block.
+ *
+ * Mirrors the Rust `MAX_OPTIONS_BYTES`, which in turn mirrors the user-headers
+ * budget: options ride that codec and inherit its limit.
+ */
+export const MAX_OPTIONS_BYTES = 100 * 1000;
-/** Maximum total byte length of an encoded options block. */
-export const MAX_OPTIONS_BYTES = 4096;
+/**
+ * Key and value length bound, inherited from the header-field codec rather
+ * than being an options-specific rule (`serializeHeaders` enforces the same
+ * range on the way out).
+ */
+const MAX_HEADER_FIELD_LENGTH = 255;
/** A resource option entry: UTF-8 string key with a typed value. */
export type OptionEntry = {
@@ -66,13 +75,10 @@ export const serializeOptions = (options: OptionEntry[]):
Buffer => {
throw new Error(
`Options block has ${options.length} entries, exceeds maximum
${MAX_OPTIONS}`);
- const block = serializeHeaders(options.map(({ key, value }) => {
- const keyLength = Buffer.byteLength(key);
- if (keyLength < 1 || keyLength > MAX_OPTION_KEY_LENGTH)
- throw new Error(
- `Option key should be between 1 and ${MAX_OPTION_KEY_LENGTH} bytes`);
- return { key: HeaderKeyFactory.String(key), value };
- }));
+ // No key-length check here: `serializeHeaders` already bounds every field
+ // to 1..=255, so an options-specific cap would only duplicate it.
+ const block = serializeHeaders(options.map(({ key, value }) =>
+ ({ key: HeaderKeyFactory.String(key), value })));
if (block.length > MAX_OPTIONS_BYTES)
throw new Error(
@@ -99,10 +105,10 @@ export const deserializeOptions = (
if (keyKind !== HeaderKind.String)
throw new Error(`Option key kind ${keyKind} is not a string`);
const keyLength = p.readUInt32LE(pos + 1);
- if (keyLength < 1 || keyLength > MAX_OPTION_KEY_LENGTH)
+ if (keyLength < 1 || keyLength > MAX_HEADER_FIELD_LENGTH)
throw new Error(
`Invalid option key length: ${keyLength}, ` +
- `must be between 1 and ${MAX_OPTION_KEY_LENGTH}`);
+ `must be between 1 and ${MAX_HEADER_FIELD_LENGTH}`);
if (pos + 5 + keyLength > end)
throw new Error('Option key overruns the block');
const key = p.subarray(pos + 5, pos + 5 + keyLength).toString();
