This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a change to branch experiment
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 1627ce940 advance v2
add ccb620f6e fix(java): use Netty EventLoop for concurrent async
operations (#2558)
add e4460cb3e feat(server): require auth for stats endpoint (#2559)
add 70a829436 perf(ci): use opt-level=2, optimize sccache usage (#2566)
add 8eebb4108 feat(connectors,mcp): Implement reading iggy token from file
(#2565)
add 1be2aa3c0 fix(rust): fix log in with PAT by publishing missing
diagnostic event (#2569)
add 3648c2b72 chore(ci): improve sccache cache effectiveness (#2568)
add ab7902eb3 perf(ci): reduce dev profile opt-level from 2 to 1 (#2573)
add a53a4ed3d fix(server): maintain message availability during async disk
I/O (#2575)
add ac6b30bb8 fix(integration): add watchdog to detect test server crashes
(#2576)
add f5f9bb51c fix(server): prevent message unavailability during
background persistence (#2578)
add 5b864bedc refactor(server): consolidate permissions into metadata
crate and fix inheritance bugs (#2564)
add 9319220e4 feat(connectors): extend Postgres sink & source connectors,
add integration tests (#2579)
add b13456c1e fix(server): chunk vectored writes to avoid exceeding
IOV_MAX limit (#2581)
add 979e70b76 Merge branch 'master' into experiment
No new revisions were added by this update.
Summary of changes:
.github/actions/csharp-dotnet/pre-merge/action.yml | 1 +
.github/actions/go/pre-merge/action.yml | 1 +
.github/actions/java-gradle/pre-merge/action.yml | 1 +
.github/actions/node-npm/pre-merge/action.yml | 1 +
.../actions/python-maturin/pre-merge/action.yml | 1 +
.github/actions/rust/post-merge/action.yml | 3 +-
.github/actions/rust/pre-merge/action.yml | 31 +-
.github/actions/utils/docker-buildx/action.yml | 10 +
.../actions/utils/setup-rust-with-cache/action.yml | 102 +-
.github/workflows/_build_python_wheels.yml | 1 +
.github/workflows/_build_rust_artifacts.yml | 2 +
.github/workflows/_publish_rust_crates.yml | 2 +-
.github/workflows/_test_bdd.yml | 1 +
.github/workflows/_test_examples.yml | 1 +
.github/workflows/post-merge.yml | 1 +
.github/workflows/publish.yml | 3 +-
Cargo.lock | 859 +++----
Cargo.toml | 40 +-
DEPENDENCIES.md | 196 +-
.../tests/tcp_test/offset_feature_deserialize.go | 5 +-
core/ai/mcp/Cargo.toml | 8 +-
core/ai/mcp/src/error.rs | 6 +
core/ai/mcp/src/stream.rs | 155 +-
core/bench/dashboard/server/Cargo.toml | 4 +-
core/binary_protocol/Cargo.toml | 2 +-
.../client/binary_personal_access_tokens/mod.rs | 8 +-
core/cli/Cargo.toml | 2 +-
core/common/src/alloc/buffer.rs | 23 +-
core/common/src/configs/mod.rs | 113 +-
core/common/src/types/message/messages_batch.rs | 10 +
core/common/src/types/message/polled_messages.rs | 8 +-
core/connectors/runtime/Cargo.toml | 6 +-
core/connectors/runtime/src/configs/connectors.rs | 10 +
.../src/configs/connectors/local_provider.rs | 75 +-
core/connectors/runtime/src/error.rs | 9 +
core/connectors/runtime/src/main.rs | 2 +
core/connectors/runtime/src/sink.rs | 36 +-
core/connectors/runtime/src/source.rs | 26 +-
core/connectors/runtime/src/stream.rs | 165 +-
core/connectors/sdk/Cargo.toml | 4 +-
core/connectors/sdk/src/lib.rs | 2 +
core/connectors/sinks/README.md | 2 +
core/connectors/sinks/postgres_sink/Cargo.toml | 2 +
core/connectors/sinks/postgres_sink/README.md | 298 ++-
.../sinks/postgres_sink/config.toml} | 12 +-
core/connectors/sinks/postgres_sink/src/lib.rs | 741 +++++-
core/connectors/sources/README.md | 2 +
core/connectors/sources/postgres_source/Cargo.toml | 1 +
core/connectors/sources/postgres_source/README.md | 399 +++-
.../sources/postgres_source/config.toml} | 27 +-
core/connectors/sources/postgres_source/src/lib.rs | 1299 ++++++----
core/integration/Cargo.toml | 9 +-
core/integration/src/test_server.rs | 107 +-
core/integration/tests/connectors/mod.rs | 158 +-
core/integration/tests/connectors/postgres/mod.rs | 511 +++-
.../tests/connectors/postgres/postgres_sink.rs | 141 +-
.../tests/connectors/postgres/postgres_source.rs | 209 ++
.../connectors/postgres/{config.toml => sink.toml} | 2 +-
.../{random/config.toml => postgres/source.toml} | 2 +-
.../tests/connectors/random/random_source.rs | 5 +-
core/integration/tests/mcp/mod.rs | 1 +
core/integration/tests/server/general.rs | 6 +-
core/integration/tests/server/mod.rs | 98 +-
.../server/scenarios/authentication_scenario.rs | 465 ++++
core/integration/tests/server/scenarios/mod.rs | 4 +
.../tests/server/scenarios/permissions_scenario.rs | 2511 ++++++++++++++++++++
.../scenarios/read_during_persistence_scenario.rs | 198 ++
.../scenarios/single_message_per_batch_scenario.rs | 178 ++
core/integration/tests/server/specific.rs | 24 +-
core/metadata/src/lib.rs | 2 +-
core/metadata/src/permissioner/mod.rs | 8 +-
.../permissioner_rules/consumer_groups.rs | 4 -
.../permissioner_rules/consumer_offsets.rs | 4 -
.../permissioner/permissioner_rules/messages.rs | 84 +-
.../permissioner/permissioner_rules/partitions.rs | 4 -
.../permissioner/permissioner_rules/segments.rs | 4 -
.../src/permissioner/permissioner_rules/streams.rs | 4 -
.../src/permissioner/permissioner_rules/system.rs | 4 -
.../src/permissioner/permissioner_rules/topics.rs | 72 +-
.../src/permissioner/permissioner_rules/users.rs | 4 -
core/sdk/Cargo.toml | 2 +-
core/sdk/src/http/http_client.rs | 1 -
core/server/Cargo.toml | 9 +-
core/server/server.http | 33 +-
core/server/src/binary/command.rs | 2 +-
.../cluster/get_cluster_metadata_handler.rs | 7 +-
.../create_consumer_group_handler.rs | 12 +-
.../delete_consumer_group_handler.rs | 27 +-
.../consumer_groups/get_consumer_group_handler.rs | 29 +-
.../consumer_groups/get_consumer_groups_handler.rs | 15 +-
.../consumer_groups/join_consumer_group_handler.rs | 11 +-
.../leave_consumer_group_handler.rs | 14 +-
.../delete_consumer_offset_handler.rs | 9 +-
.../get_consumer_offset_handler.rs | 17 +-
.../store_consumer_offset_handler.rs | 9 +-
.../messages/flush_unsaved_buffer_handler.rs | 1 +
.../handlers/messages/poll_messages_handler.rs | 7 +-
.../handlers/messages/send_messages_handler.rs | 21 +-
.../partitions/create_partitions_handler.rs | 15 +-
.../partitions/delete_partitions_handler.rs | 15 +-
.../create_personal_access_token_handler.rs | 6 +-
.../delete_personal_access_token_handler.rs | 6 +-
.../get_personal_access_tokens_handler.rs | 8 +-
.../handlers/segments/delete_segments_handler.rs | 22 +-
.../handlers/streams/create_stream_handler.rs | 7 +-
.../handlers/streams/delete_stream_handler.rs | 16 +-
.../binary/handlers/streams/get_stream_handler.rs | 23 +-
.../handlers/streams/purge_stream_handler.rs | 9 +-
.../handlers/streams/update_stream_handler.rs | 14 +-
.../binary/handlers/system/get_client_handler.rs | 12 +-
.../binary/handlers/system/get_clients_handler.rs | 11 +-
.../src/binary/handlers/system/get_me_handler.rs | 12 +-
.../{get_snapshot.rs => get_snapshot_handler.rs} | 3 +-
.../binary/handlers/system/get_stats_handler.rs | 5 +
core/server/src/binary/handlers/system/mod.rs | 2 +-
.../binary/handlers/topics/create_topic_handler.rs | 8 +-
.../binary/handlers/topics/delete_topic_handler.rs | 8 +-
.../binary/handlers/topics/get_topic_handler.rs | 30 +-
.../binary/handlers/topics/get_topics_handler.rs | 9 +-
.../binary/handlers/topics/purge_topic_handler.rs | 8 +-
.../binary/handlers/topics/update_topic_handler.rs | 7 +-
.../handlers/users/change_password_handler.rs | 27 +-
.../binary/handlers/users/create_user_handler.rs | 7 +-
.../binary/handlers/users/delete_user_handler.rs | 7 +-
.../src/binary/handlers/users/get_user_handler.rs | 11 +-
.../src/binary/handlers/users/get_users_handler.rs | 11 +-
.../binary/handlers/users/logout_user_handler.rs | 1 +
.../handlers/users/update_permissions_handler.rs | 20 +-
.../binary/handlers/users/update_user_handler.rs | 24 +-
core/server/src/binary/macros.rs | 2 +-
core/server/src/http/consumer_groups.rs | 100 +-
core/server/src/http/consumer_offsets.rs | 46 +-
core/server/src/http/http_shard_wrapper.rs | 79 +-
core/server/src/http/jwt/middleware.rs | 1 -
core/server/src/http/messages.rs | 6 +-
core/server/src/http/partitions.rs | 27 +-
core/server/src/http/personal_access_tokens.rs | 14 +-
core/server/src/http/streams.rs | 84 +-
core/server/src/http/system.rs | 80 +-
core/server/src/http/topics.rs | 98 +-
core/server/src/http/users.rs | 98 +-
core/server/src/shard/handlers.rs | 71 +-
core/server/src/shard/mod.rs | 10 +-
core/server/src/shard/system/clients.rs | 37 +-
core/server/src/shard/system/cluster.rs | 11 +-
core/server/src/shard/system/consumer_groups.rs | 118 +-
core/server/src/shard/system/consumer_offsets.rs | 70 +-
core/server/src/shard/system/messages.rs | 12 +-
core/server/src/shard/system/partitions.rs | 45 -
.../src/shard/system/personal_access_tokens.rs | 16 +-
core/server/src/shard/system/snapshot/mod.rs | 4 -
core/server/src/shard/system/streams.rs | 83 +-
core/server/src/shard/system/topics.rs | 72 -
core/server/src/shard/system/users.rs | 110 +-
core/server/src/shard/system/utils.rs | 116 +-
core/server/src/slab/streams.rs | 168 +-
core/server/src/streaming/partitions/helpers.rs | 36 +-
core/server/src/streaming/partitions/in_flight.rs | 100 +
core/server/src/streaming/partitions/log.rs | 23 +-
core/server/src/streaming/partitions/mod.rs | 1 +
.../src/streaming/segments/indexes/indexes_mut.rs | 13 +-
.../streaming/segments/messages/messages_writer.rs | 38 +-
core/server/src/streaming/segments/messages/mod.rs | 91 +-
.../streaming/segments/types/messages_batch_mut.rs | 16 +-
.../streaming/segments/types/messages_batch_set.rs | 29 +-
core/server/src/streaming/users/mod.rs | 2 -
core/server/src/streaming/users/permissioner.rs | 101 -
.../users/permissioner_rules/consumer_groups.rs | 76 -
.../users/permissioner_rules/consumer_offsets.rs | 49 -
.../streaming/users/permissioner_rules/messages.rs | 135 --
.../src/streaming/users/permissioner_rules/mod.rs | 27 -
.../users/permissioner_rules/partitions.rs | 40 -
.../streaming/users/permissioner_rules/segments.rs | 30 -
.../streaming/users/permissioner_rules/streams.rs | 86 -
.../streaming/users/permissioner_rules/system.rs | 44 -
.../streaming/users/permissioner_rules/topics.rs | 153 --
.../streaming/users/permissioner_rules/users.rs | 70 -
examples/rust/src/multi-tenant/consumer/main.rs | 18 +-
examples/rust/src/multi-tenant/producer/main.rs | 19 +-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 90 +-
.../client/async/AsyncClientIntegrationTest.java | 8 +-
181 files changed, 9191 insertions(+), 3814 deletions(-)
copy
core/{integration/tests/connectors/postgres/connectors_config/postgres.toml =>
connectors/sinks/postgres_sink/config.toml} (81%)
rename
core/{integration/tests/connectors/postgres/connectors_config/postgres.toml =>
connectors/sources/postgres_source/config.toml} (70%)
create mode 100644
core/integration/tests/connectors/postgres/postgres_source.rs
rename core/integration/tests/connectors/postgres/{config.toml => sink.toml}
(93%)
copy core/integration/tests/connectors/{random/config.toml =>
postgres/source.toml} (93%)
create mode 100644
core/integration/tests/server/scenarios/authentication_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/permissions_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/read_during_persistence_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs
rename core/server/src/binary/handlers/system/{get_snapshot.rs =>
get_snapshot_handler.rs} (95%)
create mode 100644 core/server/src/streaming/partitions/in_flight.rs
delete mode 100644 core/server/src/streaming/users/permissioner.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/consumer_groups.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/consumer_offsets.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/messages.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/mod.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/partitions.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/segments.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/streams.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/system.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/topics.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/users.rs
