This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a change to branch journal_handle
in repository https://gitbox.apache.org/repos/asf/iggy.git
from aa20852c0 feat(metadata): expose metadata generics and create journal
handle
add 4fb1fc7ba fix(metadata): fix license header in module file (#2608)
add de6995d59 refactor(server): replace slab ECS with LeftRight-based
global metadata (#2555)
add 73de5c1c1 fix(sdk): prevent client from getting stuck after disconnect
(#2610)
add e46f294b7 refactor(bench): remove TestServer startup capability (#2611)
add 13a1bf339 feat(connectors,mcp): implement logging with telemetry
(#2612)
add b26246252 deps(connectors): Update iceberg to 0.8.0 (#2605)
add 486cdbcf8 fix(sdk): improve high-level SDK direct and background
producers (#2621)
add 5b1098db3 test(java): add unit tests for serialization utilities
(#2607)
add 19cb83f14 Merge branch 'master' into journal_handle
No new revisions were added by this update.
Summary of changes:
Cargo.lock | 823 +++++-----
Cargo.toml | 51 +-
DEPENDENCIES.md | 161 +-
core/ai/mcp/Cargo.toml | 11 +-
core/ai/mcp/README.md | 18 +
core/ai/mcp/config.toml | 12 +
core/ai/mcp/src/configs.rs | 108 +-
core/ai/mcp/src/log.rs | 180 +++
core/ai/mcp/src/main.rs | 20 +-
core/bench/Cargo.toml | 6 +-
core/bench/dashboard/server/Cargo.toml | 2 +-
.../bench/src/actors/consumer/client/high_level.rs | 2 +-
core/bench/src/actors/consumer/client/low_level.rs | 2 +-
.../actors/consumer/typed_benchmark_consumer.rs | 2 +-
.../bench/src/actors/producer/client/high_level.rs | 2 +-
core/bench/src/actors/producer/client/low_level.rs | 2 +-
.../actors/producer/typed_benchmark_producer.rs | 2 +-
.../typed_banchmark_producing_consumer.rs | 2 +-
core/bench/src/analytics/report_builder.rs | 2 +-
core/bench/src/args/common.rs | 45 +-
core/bench/src/args/defaults.rs | 4 -
core/bench/src/args/examples.rs | 3 -
.../src/benchmarks/balanced_consumer_group.rs | 2 +-
core/bench/src/benchmarks/balanced_producer.rs | 2 +-
.../balanced_producer_and_consumer_group.rs | 2 +-
core/bench/src/benchmarks/benchmark.rs | 2 +-
core/bench/src/benchmarks/common.rs | 2 +-
.../benchmarks/end_to_end_producing_consumer.rs | 2 +-
.../end_to_end_producing_consumer_group.rs | 2 +-
core/bench/src/benchmarks/pinned_consumer.rs | 2 +-
core/bench/src/benchmarks/pinned_producer.rs | 2 +-
.../src/benchmarks/pinned_producer_and_consumer.rs | 2 +-
core/bench/src/runner.rs | 11 +-
core/bench/src/utils/client_factory.rs | 163 +-
core/bench/src/utils/mod.rs | 4 +-
core/bench/src/utils/server_starter.rs | 188 ---
core/binary_protocol/Cargo.toml | 2 +-
core/common/Cargo.toml | 4 +-
core/connectors/runtime/Cargo.toml | 9 +-
core/connectors/runtime/README.md | 18 +
core/connectors/runtime/config.toml | 12 +
core/connectors/runtime/src/configs/runtime.rs | 106 +-
core/connectors/runtime/src/log.rs | 181 +++
core/connectors/runtime/src/main.rs | 21 +-
core/connectors/runtime/src/sink.rs | 16 +-
core/connectors/runtime/src/source.rs | 10 +-
core/connectors/sdk/Cargo.toml | 2 +-
core/connectors/sdk/src/lib.rs | 2 +
core/connectors/sdk/src/log.rs | 96 ++
core/connectors/sdk/src/sink.rs | 17 +-
core/connectors/sdk/src/source.rs | 8 +-
.../connectors/sinks/elasticsearch_sink/Cargo.toml | 2 +-
core/connectors/sinks/iceberg_sink/Cargo.toml | 2 +-
core/connectors/sinks/iceberg_sink/src/catalog.rs | 37 +-
.../sinks/iceberg_sink/src/router/mod.rs | 41 +-
core/connectors/sinks/postgres_sink/Cargo.toml | 2 +-
core/connectors/sinks/quickwit_sink/Cargo.toml | 2 +-
core/connectors/sinks/stdout_sink/Cargo.toml | 2 +-
.../sources/elasticsearch_source/Cargo.toml | 2 +-
core/connectors/sources/postgres_source/Cargo.toml | 2 +-
core/connectors/sources/random_source/Cargo.toml | 2 +-
core/integration/Cargo.toml | 4 +-
core/integration/src/test_mcp_server.rs | 1 +
.../data_integrity/verify_after_server_restart.rs | 205 ++-
core/integration/tests/mcp/mod.rs | 5 +-
core/integration/tests/sdk/producer/background.rs | 143 ++
.../tests/server/concurrent_addition.rs | 15 +-
.../tests/server/scenarios/concurrent_scenario.rs | 596 +++++---
...h_multiple_clients_polling_messages_scenario.rs | 60 +-
...with_single_client_polling_messages_scenario.rs | 39 +-
.../scenarios/segment_rotation_race_scenario.rs | 2 +-
core/metadata/Cargo.toml | 2 +-
core/metadata/src/stm/mod.rs | 1 +
core/sdk/Cargo.toml | 3 +-
core/sdk/src/clients/consumer.rs | 71 +-
core/sdk/src/clients/producer.rs | 44 +-
core/sdk/src/clients/producer_config.rs | 44 +-
core/sdk/src/clients/producer_dispatcher.rs | 9 +-
core/sdk/src/clients/producer_sharding.rs | 61 +-
core/sdk/src/prelude.rs | 1 +
core/sdk/src/tcp/tcp_client.rs | 2 +
core/server/Cargo.toml | 34 +-
.../cluster/get_cluster_metadata_handler.rs | 1 -
.../create_consumer_group_handler.rs | 123 +-
.../delete_consumer_group_handler.rs | 95 +-
.../consumer_groups/get_consumer_group_handler.rs | 30 +-
.../consumer_groups/get_consumer_groups_handler.rs | 41 +-
.../consumer_groups/join_consumer_group_handler.rs | 74 +-
.../leave_consumer_group_handler.rs | 74 +-
.../delete_consumer_offset_handler.rs | 9 +-
.../get_consumer_offset_handler.rs | 6 +-
.../store_consumer_offset_handler.rs | 9 +-
.../messages/flush_unsaved_buffer_handler.rs | 1 -
.../handlers/messages/poll_messages_handler.rs | 1 -
.../handlers/messages/send_messages_handler.rs | 59 +-
.../partitions/create_partitions_handler.rs | 127 +-
.../partitions/delete_partitions_handler.rs | 126 +-
.../create_personal_access_token_handler.rs | 109 +-
.../delete_personal_access_token_handler.rs | 85 +-
.../login_with_personal_access_token_handler.rs | 1 -
.../handlers/segments/delete_segments_handler.rs | 49 +-
.../handlers/streams/create_stream_handler.rs | 44 +-
.../handlers/streams/delete_stream_handler.rs | 29 +-
.../binary/handlers/streams/get_stream_handler.rs | 31 +-
.../binary/handlers/streams/get_streams_handler.rs | 13 +-
.../handlers/streams/purge_stream_handler.rs | 88 +-
.../handlers/streams/update_stream_handler.rs | 82 +-
.../binary/handlers/system/get_client_handler.rs | 5 +-
.../binary/handlers/system/get_clients_handler.rs | 5 +-
.../binary/handlers/system/get_stats_handler.rs | 5 +-
.../src/binary/handlers/system/ping_handler.rs | 1 -
.../binary/handlers/topics/create_topic_handler.rs | 72 +-
.../binary/handlers/topics/delete_topic_handler.rs | 51 +-
.../binary/handlers/topics/get_topic_handler.rs | 32 +-
.../binary/handlers/topics/get_topics_handler.rs | 21 +-
.../binary/handlers/topics/purge_topic_handler.rs | 105 +-
.../binary/handlers/topics/update_topic_handler.rs | 64 +-
.../handlers/users/change_password_handler.rs | 124 +-
.../binary/handlers/users/create_user_handler.rs | 40 +-
.../binary/handlers/users/delete_user_handler.rs | 26 +-
.../src/binary/handlers/users/get_user_handler.rs | 5 +-
.../src/binary/handlers/users/get_users_handler.rs | 5 +-
.../binary/handlers/users/login_user_handler.rs | 1 -
.../binary/handlers/users/logout_user_handler.rs | 1 -
.../handlers/users/update_permissions_handler.rs | 99 +-
.../binary/handlers/users/update_user_handler.rs | 111 +-
core/server/src/binary/macros.rs | 2 +-
core/server/src/binary/mapper.rs | 152 +-
core/server/src/bootstrap.rs | 441 +++---
core/server/src/configs/sharding.rs | 11 -
core/server/src/configs/system.rs | 20 +-
core/server/src/http/consumer_groups.rs | 182 +--
core/server/src/http/consumer_offsets.rs | 42 +-
core/server/src/http/http_shard_wrapper.rs | 97 +-
core/server/src/http/mapper.rs | 383 ++---
core/server/src/http/messages.rs | 3 +-
core/server/src/http/partitions.rs | 26 +-
core/server/src/http/personal_access_tokens.rs | 37 +-
core/server/src/http/streams.rs | 151 +-
core/server/src/http/system.rs | 15 +-
core/server/src/http/topics.rs | 200 +--
core/server/src/http/users.rs | 124 +-
core/server/src/lib.rs | 1 -
core/server/src/main.rs | 108 +-
core/server/src/metadata/absorb.rs | 101 +-
core/server/src/metadata/inner.rs | 14 +-
core/server/src/metadata/mod.rs | 6 +-
core/server/src/metadata/ops.rs | 3 +-
core/server/src/metadata/partition.rs | 11 +-
core/server/src/metadata/reader.rs | 699 ++++++++-
core/server/src/metadata/writer.rs | 19 +-
core/server/src/quic/listener.rs | 2 +-
core/server/src/shard/builder.rs | 47 +-
core/server/src/shard/communication.rs | 17 -
core/server/src/shard/handlers.rs | 903 ++++++++---
core/server/src/shard/mod.rs | 125 +-
core/server/src/shard/namespace.rs | 53 -
core/server/src/shard/system/clients.rs | 52 +-
core/server/src/shard/system/consumer_groups.rs | 245 ++-
core/server/src/shard/system/consumer_offsets.rs | 348 +++--
core/server/src/shard/system/messages.rs | 570 +++++--
core/server/src/shard/system/partitions.rs | 473 +++---
.../src/shard/system/personal_access_tokens.rs | 153 +-
core/server/src/shard/system/segments.rs | 224 ++-
core/server/src/shard/system/stats.rs | 61 +-
core/server/src/shard/system/streams.rs | 320 +++-
core/server/src/shard/system/topics.rs | 474 ++++--
core/server/src/shard/system/users.rs | 252 ++-
core/server/src/shard/system/utils.rs | 164 +-
.../src/shard/tasks/periodic/heartbeat_verifier.rs | 2 +-
.../src/shard/tasks/periodic/message_cleaner.rs | 177 +--
.../src/shard/tasks/periodic/message_saver.rs | 80 +-
.../periodic/personal_access_token_cleaner.rs | 40 +-
core/server/src/shard/transmission/event.rs | 115 +-
core/server/src/shard/transmission/frame.rs | 28 +-
core/server/src/shard/transmission/message.rs | 88 +-
core/server/src/slab/consumer_groups.rs | 200 ---
core/server/src/slab/helpers.rs | 70 -
core/server/src/slab/mod.rs | 43 -
core/server/src/slab/partitions.rs | 281 ----
core/server/src/slab/streams.rs | 1612 --------------------
core/server/src/slab/topics.rs | 307 ----
core/server/src/slab/traits_ext.rs | 117 --
core/server/src/slab/users.rs | 207 ---
core/server/src/streaming/mod.rs | 1 -
.../partitions/consumer_group_offsets.rs} | 33 +-
.../src/streaming/partitions/consumer_offset.rs | 20 +-
.../src/streaming/partitions/consumer_offsets.rs} | 32 +-
core/server/src/streaming/partitions/helpers.rs | 512 +------
.../src/streaming/partitions/local_partition.rs | 102 ++
.../src/streaming/partitions/local_partitions.rs | 213 +++
core/server/src/streaming/partitions/mod.rs | 6 +-
core/server/src/streaming/partitions/ops.rs | 700 +++++++++
core/server/src/streaming/partitions/partition.rs | 454 ------
.../src/streaming/segments/indexes/indexes_mut.rs | 5 -
.../streaming/segments/messages/messages_reader.rs | 43 -
.../streaming/segments/messages/messages_writer.rs | 3 -
core/server/src/streaming/segments/segment.rs | 4 -
.../segments/types/message_header_view_mut.rs | 15 -
.../streaming/segments/types/messages_batch_mut.rs | 28 +-
.../streaming/segments/types/messages_batch_set.rs | 3 +-
core/server/src/streaming/stats/mod.rs | 15 +
core/server/src/streaming/streams/helpers.rs | 107 --
core/server/src/streaming/streams/mod.rs | 2 -
core/server/src/streaming/streams/storage.rs | 27 +-
core/server/src/streaming/streams/stream.rs | 225 ---
core/server/src/streaming/topics/consumer_group.rs | 243 ---
core/server/src/streaming/topics/helpers.rs | 305 +---
core/server/src/streaming/topics/mod.rs | 2 -
core/server/src/streaming/topics/storage.rs | 43 +-
core/server/src/streaming/topics/topic.rs | 379 -----
core/server/src/streaming/traits.rs | 45 -
core/server/src/tcp/tcp_listener.rs | 2 +-
core/server/src/tcp/tcp_tls_listener.rs | 2 +-
core/server/src/websocket/websocket_listener.rs | 2 +-
.../server/src/websocket/websocket_tls_listener.rs | 2 +-
.../org/apache/iggy/serde/BytesSerializer.java | 5 +-
.../client/async/tcp/IggyFrameDecoderTest.java | 455 ++++++
.../client/blocking/tcp/BytesSerializerTest.java | 555 +++++++
.../apache/iggy/serde/BytesDeserializerTest.java | 671 ++++++++
justfile | 4 +-
scripts/check-backwards-compat.sh | 6 +-
222 files changed, 11256 insertions(+), 10579 deletions(-)
create mode 100644 core/ai/mcp/src/log.rs
delete mode 100644 core/bench/src/utils/server_starter.rs
create mode 100644 core/connectors/runtime/src/log.rs
create mode 100644 core/connectors/sdk/src/log.rs
delete mode 100644 core/server/src/shard/namespace.rs
delete mode 100644 core/server/src/slab/consumer_groups.rs
delete mode 100644 core/server/src/slab/helpers.rs
delete mode 100644 core/server/src/slab/mod.rs
delete mode 100644 core/server/src/slab/partitions.rs
delete mode 100644 core/server/src/slab/streams.rs
delete mode 100644 core/server/src/slab/topics.rs
delete mode 100644 core/server/src/slab/traits_ext.rs
delete mode 100644 core/server/src/slab/users.rs
copy core/{common/src/sharding/shard_id.rs =>
server/src/streaming/partitions/consumer_group_offsets.rs} (51%)
copy core/{common/src/sharding/shard_id.rs =>
server/src/streaming/partitions/consumer_offsets.rs} (55%)
create mode 100644 core/server/src/streaming/partitions/local_partition.rs
create mode 100644 core/server/src/streaming/partitions/local_partitions.rs
create mode 100644 core/server/src/streaming/partitions/ops.rs
delete mode 100644 core/server/src/streaming/partitions/partition.rs
delete mode 100644 core/server/src/streaming/streams/helpers.rs
delete mode 100644 core/server/src/streaming/streams/stream.rs
delete mode 100644 core/server/src/streaming/topics/consumer_group.rs
delete mode 100644 core/server/src/streaming/topics/topic.rs
delete mode 100644 core/server/src/streaming/traits.rs
create mode 100644
foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java
