This is an automated email from the ASF dual-hosted git repository.
ryerraguntla pushed a change to branch feat/gateways-3548-describe-list-groups
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 54291df84 Fixing review comments
add f03ee4614 ci(repo): upgrade Cargo-Rail planning to v0.30.1 (#3999)
add a2895fdc6 feat(cpp): add code coverage (#4118)
add f6dfbdbb3 feat(server): store offsets of groups managed outside Iggy
(#4392)
add 56e109a58 feat(consensus): persist retry receipts and shared sessions
(#4387)
add 07d2a2136 chore(deps): Bump the github-actions group with 3 updates
(#4426)
add 651ccea4e chore(deps): Bump postcss-selector-parser from 7.1.4 to
7.1.6 in /web (#4425)
add 50a0db603 chore(deps): Bump mako from 1.4.1 to 1.4.2 in /bdd/python
(#4423)
add 0ce152402 chore(deps): Bump joi from 17.13.7 to 17.13.8 in /web (#4422)
add a263a813d fix(ci): skip unaffected license checks (#4428)
add 4843593b1 chore(deps): Bump source-map-js from 1.2.1 to 1.2.2 in /web
(#4424)
add 2c1f70b53 chore(repo): bump Rust toolchain to 1.99.0 (#4430)
add 5da16eba2 chore(deps): Bump the java group across 3 directories with 6
updates (#4427)
add 43fb44364 feat(node): wire up snapshot command in system client (#4389)
add fd207baac Merge branch 'master' into
feat/gateways-3548-describe-list-groups
No new revisions were added by this update.
Summary of changes:
.config/rail.toml | 36 +-
.devcontainer/Dockerfile | 2 +-
.github/actions/cpp-bazel/pre-merge/action.yml | 16 +-
.github/actions/rust/pre-merge/action.yml | 258 +-
.github/config/edge-image-variants.json | 117 +
.github/config/publish.yml | 28 +-
.github/workflows/_common.yml | 39 +-
.github/workflows/_detect.yml | 27 +
.github/workflows/_test.yml | 18 +-
.github/workflows/coverage-baseline.yml | 32 +-
.github/workflows/post-merge.yml | 61 +-
.github/workflows/pre-merge.yml | 5 +
.github/workflows/publish.yml | 2 +-
Cargo.lock | 5 +-
Cargo.toml | 1 -
Dockerfile | 2 +-
bdd/java/Dockerfile | 2 +-
bdd/java/build.gradle.kts | 6 +-
bdd/java/gradle/wrapper/gradle-wrapper.properties | 2 +-
bdd/java/gradlew | 4 +-
bdd/php/Dockerfile | 2 +-
bdd/python/Dockerfile | 2 +-
bdd/python/uv.lock | 6 +-
bdd/rust/Dockerfile | 2 +-
codecov.yml | 7 +
core/ai/mcp/Dockerfile | 2 +-
core/bench/dashboard/frontend/src/api/mod.rs | 9 +-
.../frontend/src/components/app_content.rs | 28 +-
.../frontend/src/components/chart/trend_chart.rs | 4 +-
.../frontend/src/components/layout/sidebar.rs | 4 +-
.../dashboard/frontend/src/state/benchmark.rs | 22 +-
core/bench/dashboard/server/Dockerfile | 2 +-
core/bench/report/Cargo.toml | 1 -
core/bench/report/src/types/hardware.rs | 2 +-
core/bench/src/utils/client_factory.rs | 20 +-
core/binary_protocol/src/codes.rs | 7 +-
core/binary_protocol/src/consensus/command.rs | 8 +-
.../src/consensus/consumer_session.rs | 32 +-
core/binary_protocol/src/consensus/header.rs | 108 +-
core/binary_protocol/src/consensus/operation.rs | 24 +-
core/binary_protocol/src/dispatch.rs | 8 +-
core/binary_protocol/src/framing.rs | 6 +-
core/binary_protocol/src/lib.rs | 4 +-
core/binary_protocol/src/primitives/consumer.rs | 29 +-
.../src/requests/system/attach_consumer_session.rs | 87 -
.../src/requests/system/bind_session.rs | 155 +
core/binary_protocol/src/requests/system/mod.rs | 6 +-
.../src/requests/system/retire_sessions.rs | 117 +
.../src/requests/users/login_register.rs | 60 +-
.../src/requests/users/login_register_with_pat.rs | 17 +-
.../src/responses/messages/poll_routing.rs | 8 +-
.../src/responses/messages/send_messages.rs | 14 +-
core/binary_protocol/src/version.rs | 145 +-
core/cli/src/args/consumer_offset.rs | 6 +-
.../binary_consumer_offsets/get_consumer_offset.rs | 1 +
core/common/src/error/iggy_error.rs | 4 +
core/common/src/lib.rs | 1 +
core/common/src/traits/binary_impls/mod.rs | 2 +-
.../traits/binary_impls/personal_access_tokens.rs | 25 +-
core/common/src/traits/binary_impls/users.rs | 25 +-
core/common/src/traits/binary_transport.rs | 46 +
core/common/src/traits/consumer_offset_client.rs | 6 +
core/common/src/traits/message_client.rs | 1 +
core/common/src/traits/mod.rs | 1 +
core/common/src/types/consumer/consumer_kind.rs | 78 +-
core/common/src/types/streaming_stats.rs | 4 +-
core/configs/src/server_config/metadata.rs | 12 +-
core/configs/src/server_config/partition.rs | 11 +-
core/connectors/runtime/Dockerfile | 2 +-
core/consensus/Cargo.toml | 1 +
core/consensus/src/client_table.rs | 3844 ++++++++-----------
core/consensus/src/impls.rs | 221 +-
core/consensus/src/lib.rs | 23 +-
core/consensus/src/metadata_helpers.rs | 398 +-
core/consensus/src/observability.rs | 2 +
core/consensus/src/plane_helpers.rs | 15 +
core/integration/src/harness/disk.rs | 1 +
core/integration/tests/cli/consumer_offset/mod.rs | 1 +
.../test_consumer_offset_external_group_command.rs | 149 +
.../test_consumer_offset_get_command.rs | 6 +-
.../test_consumer_offset_set_command.rs | 6 +-
.../tests/cluster/client_table_adversarial.rs | 112 +-
.../tests/cluster/client_table_restart.rs | 308 +-
core/integration/tests/cluster/crash_durability.rs | 56 +
.../tests/cluster/crash_recovery_corruption.rs | 30 +-
.../tests/cluster/fast_primary_rejoin.rs | 1 +
core/integration/tests/cluster/partition_dedup.rs | 548 ++-
.../tests/cluster/partition_primary_routing.rs | 121 +-
.../tests/cluster/partition_state_transfer.rs | 54 +-
.../tests/connectors/runtime/http_state.rs | 2 +-
.../verify_consumer_group_partition_assignment.rs | 136 +-
core/integration/tests/sdk/producer/mod.rs | 52 +
core/integration/tests/sdk/protocol_version.rs | 25 +
core/integration/tests/server/cg.rs | 10 +-
.../tests/server/consumer_offset_quota_vsr.rs | 4 +
.../tests/server/descriptor_exhaustion_vsr.rs | 71 +-
core/integration/tests/server/http_rbac.rs | 91 +
core/integration/tests/server/http_vsr.rs | 176 +-
core/integration/tests/server/raw_tcp.rs | 4 +
.../server/scenarios/authentication_scenario.rs | 38 +-
...umer_group_auto_commit_reconnection_scenario.rs | 2 +
...er_group_new_messages_after_restart_scenario.rs | 1 +
.../scenarios/external_group_offsets_scenario.rs | 142 +
core/integration/tests/server/scenarios/mod.rs | 1 +
.../tests/server/scenarios/permissions_scenario.rs | 11 +-
.../scenarios/reconnect_after_restart_scenario.rs | 29 +-
.../stale_client_consumer_group_scenario.rs | 4 +-
core/journal/src/partition_journal.rs | 90 +
core/journal/src/partition_journal/segments.rs | 120 +-
core/journal/src/prepare_journal.rs | 2 +-
core/message_bus/src/connection_cap.rs | 2 +-
core/message_bus/src/replica/auth.rs | 25 +-
core/message_bus/src/replica/handshake.rs | 230 +-
core/message_bus/tests/common/mod.rs | 2 +
core/message_bus/tests/head_of_line.rs | 47 +-
core/message_bus/tests/replica_auth_handshake.rs | 68 +
core/metadata/src/applied_frontier.rs | 37 +-
core/metadata/src/impls/metadata.rs | 1982 +++++++---
core/metadata/src/impls/recovery.rs | 471 ++-
core/metadata/src/stm/authz.rs | 2 +
core/metadata/src/stm/mux.rs | 2 +-
core/metadata/src/stm/result.rs | 10 +-
core/metadata/src/stm/snapshot.rs | 105 +-
core/metadata/src/stm/stream.rs | 21 +-
core/metadata/src/stm/user.rs | 33 +-
core/partitions/src/consumer_offset_capacity.rs | 3 +
core/partitions/src/iggy_partition.rs | 4022 +++++++++++++++-----
core/partitions/src/iggy_partitions.rs | 44 +-
core/partitions/src/lib.rs | 10 +-
core/partitions/src/offset_recovery.rs | 23 +-
core/partitions/src/partition_storage.rs | 288 +-
core/partitions/src/persistence.rs | 62 +-
core/partitions/src/segment_recovery.rs | 25 +-
core/partitions/src/state_transfer.rs | 846 +++-
core/partitions/src/types.rs | 18 +-
core/partitions/tests/consumer_offset_history.rs | 11 +-
core/sdk/Cargo.toml | 2 +
core/sdk/README.md | 13 +-
core/sdk/src/clients/client.rs | 9 +-
core/sdk/src/clients/producer.rs | 93 +-
core/sdk/src/clients/producer_builder.rs | 25 +-
core/sdk/src/clients/producer_dispatcher.rs | 1 +
core/sdk/src/http/consumer_offsets.rs | 72 +-
core/sdk/src/http/messages.rs | 1 +
core/sdk/src/lib.rs | 3 +-
core/sdk/src/poll_routing.rs | 166 +-
core/sdk/src/quic/quic_client.rs | 647 +++-
core/sdk/src/session.rs | 248 +-
.../stream_builder/build/build_iggy_consumer.rs | 3 +
.../stream_builder/build/build_iggy_producer.rs | 1 +
.../stream_builder/config/config_iggy_consumer.rs | 41 +-
.../stream_builder/config/config_iggy_producer.rs | 20 +-
core/sdk/src/stream_builder/iggy_stream.rs | 35 +-
.../sdk/src/stream_builder/iggy_stream_consumer.rs | 28 +-
core/sdk/src/tcp/tcp_client.rs | 747 +++-
core/sdk/src/vsr.rs | 240 +-
core/sdk/src/websocket/websocket_client.rs | 986 +++--
core/server/Dockerfile | 2 +-
core/server/README.md | 121 +-
core/server/config.toml | 51 +-
core/server/src/boot/listeners.rs | 2 +
core/server/src/boot/mod.rs | 168 +-
core/server/src/boot/threads.rs | 2 +-
core/server/src/consumer_group.rs | 38 +-
core/server/src/consumer_group/lease.rs | 207 +-
core/server/src/consumer_group/liveness.rs | 968 ++++-
core/server/src/dispatch/authz.rs | 10 +-
core/server/src/dispatch/host.rs | 41 +-
core/server/src/dispatch/login_error.rs | 10 +-
core/server/src/dispatch/mod.rs | 390 +-
core/server/src/dispatch/partition.rs | 293 +-
core/server/src/dispatch/reads.rs | 82 +-
core/server/src/dispatch/session_ops.rs | 313 +-
core/server/src/dispatch/submit.rs | 29 +-
core/server/src/dispatch/test_support.rs | 2 +
core/server/src/http.rs | 2 +
core/server/src/http/admission.rs | 11 +-
core/server/src/http/extractor.rs | 1 +
core/server/src/http/handlers.rs | 76 +-
core/server/src/http/session.rs | 246 +-
core/server/src/http/state.rs | 155 +-
core/server/src/http/submit.rs | 186 +-
core/server/src/http/wire.rs | 2 +-
core/server/src/namespace.rs | 18 +-
core/server/src/partition_helpers.rs | 724 +++-
core/server/src/partition_reconciler.rs | 182 +-
core/server/src/server_error.rs | 23 +
core/server/src/session_manager.rs | 659 ++--
core/server/src/sysinfo_probe.rs | 2 +-
core/server/src/wire.rs | 11 +-
core/server_common/src/consensus_message.rs | 22 +-
core/server_common/src/iobuf.rs | 2 +-
core/server_common/src/poll.rs | 2 +-
core/shard/Cargo.toml | 3 +
core/shard/src/lib.rs | 217 +-
core/shard/src/metrics.rs | 58 +-
core/shard/src/poll.rs | 53 +-
core/shard/src/poll/completion.rs | 2 +-
core/shard/src/poll/completion_tests.rs | 237 +-
core/shard/src/poll/test_support.rs | 30 +-
core/shard/src/router.rs | 43 +-
core/simulator/README.md | 5 +-
core/simulator/src/bin/workload-fuzz.rs | 13 -
core/simulator/src/client.rs | 57 +-
.../simulator/src/consumer_offset_history_tests.rs | 29 +-
core/simulator/src/executor/time.rs | 4 +-
core/simulator/src/lib.rs | 304 +-
core/simulator/src/packet.rs | 3 +-
core/simulator/src/replica.rs | 3 +-
core/simulator/src/storage/purge.rs | 226 +-
core/simulator/src/storage/tests.rs | 261 +-
core/simulator/src/workload/ops/mod.rs | 25 +-
.../src/workload/ops/store_consumer_offset.rs | 2 +-
examples/java/build.gradle.kts | 2 +-
.../java/gradle/wrapper/gradle-wrapper.properties | 2 +-
examples/java/gradlew | 4 +-
.../iggy/examples/MultiTenantAccessTest.java | 4 +-
examples/rust/src/new-sdk/consumer/main.rs | 1 +
foreign/cpp/BUILD.bazel | 50 +-
foreign/cpp/MODULE.bazel | 19 +-
foreign/cpp/MODULE.bazel.lock | 2 +-
.../Fixtures/IggyServerFixture.cs | 4 +-
.../Iggy_SDK.Tests.Integration/HeartbeatTests.cs | 5 +-
.../TcpMessageStream.PollRouting.cs | 32 +-
.../Implementations/TcpMessageStream.Vsr.cs | 64 +-
.../IggyClient/Implementations/TcpMessageStream.cs | 18 +-
foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 2 +-
foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 50 +-
foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs | 52 +-
foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs | 22 +-
foreign/csharp/Iggy_SDK/Vsr/VsrError.cs | 3 +
foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 20 +-
.../VsrTests/ConsensusSessionTests.cs | 68 +-
.../VsrTests/EndpointFailoverTests.cs | 257 +-
.../Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs | 38 +-
foreign/csharp/Iggy_SDK_Tests/VsrTests/MockNode.cs | 7 +-
.../Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs | 78 +-
.../Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs | 4 +-
.../VsrTests/VsrProtocolDriftTests.cs | 33 +-
foreign/csharp/README.md | 41 +-
foreign/go/README.md | 15 +-
foreign/go/client/tcp/tcp_connect_test.go | 47 +-
foreign/go/client/tcp/tcp_core.go | 30 +-
foreign/go/client/tcp/tcp_core_test.go | 34 +-
foreign/go/client/tcp/tcp_failover_test.go | 99 +-
foreign/go/client/tcp/tcp_poll_routing.go | 33 +-
foreign/go/client/tcp/tcp_poll_routing_test.go | 110 +-
foreign/go/client/tcp/tcp_session_management.go | 77 +-
foreign/go/client/tcp/tcp_testing_test.go | 5 +
foreign/go/errors/errors.yaml | 11 +
foreign/go/errors/errors_gen.go | 35 +
foreign/go/internal/command/code.go | 2 +-
foreign/go/internal/vsr/operation.go | 8 +-
foreign/go/internal/vsr/protocol_parity_test.go | 19 +-
foreign/go/internal/vsr/register.go | 42 +-
foreign/go/internal/vsr/register_test.go | 40 +-
foreign/go/internal/vsr/session.go | 13 +-
foreign/java/README.md | 14 +
foreign/java/gradle/libs.versions.toml | 6 +-
.../java/gradle/wrapper/gradle-wrapper.properties | 2 +-
foreign/java/gradlew | 4 +-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 111 +-
.../apache/iggy/client/async/tcp/PollRouter.java | 51 +-
.../client/async/tcp/vsr/ConsensusSession.java | 79 +-
.../iggy/client/async/tcp/vsr/VsrLoginCodec.java | 43 +-
.../iggy/client/async/tcp/vsr/VsrOperation.java | 4 +
.../client/async/tcp/vsr/VsrRequestEncoder.java | 25 +-
.../client/async/tcp/vsr/VsrResponseHandler.java | 2 +-
.../org/apache/iggy/exception/IggyErrorCode.java | 2 +
.../java/org/apache/iggy/serde/CommandCode.java | 2 +-
.../test/java/org/apache/iggy/TestConstants.java | 9 +
.../client/async/AsyncClientIntegrationTest.java | 25 +-
.../client/async/AsyncConnectionPoolAuthTest.java | 7 +-
.../AsyncIggyTcpClientEndpointFailoverTest.java | 2 +-
.../tcp/AsyncIggyTcpClientEventLoopGroupTest.java | 2 +-
.../AsyncIggyTcpClientTransientFailoverTest.java | 194 +-
.../tcp/AsyncTcpConnectionConcurrencyTest.java | 2 +-
.../client/async/tcp/ClientRoutingStateTest.java | 5 +-
.../client/async/tcp/vsr/VsrLoginCodecTest.java | 58 +-
.../async/tcp/vsr/VsrRequestEncoderTest.java | 25 +-
.../async/tcp/vsr/VsrResponseHandlerTest.java | 7 +-
.../iggy/client/blocking/IntegrationTest.java | 4 +-
.../client/blocking/tcp/MessagesTcpClientTest.java | 9 +-
.../apache/iggy/exception/IggyErrorCodeTest.java | 1 +
foreign/node/README.md | 19 +-
foreign/node/scripts/check-vsr-protocol.mjs | 30 +-
foreign/node/src/client/client.socket.test.ts | 241 +-
foreign/node/src/client/client.socket.ts | 78 +-
foreign/node/src/e2e/tcp.system.e2e.ts | 7 +
foreign/node/src/wire/command-set.ts | 2 +
foreign/node/src/wire/command.code.ts | 4 +-
foreign/node/src/wire/error.code.ts | 2 +
foreign/node/src/wire/system/index.ts | 1 +
.../node/src/wire/system/snapshot.command.test.ts | 107 +
foreign/node/src/wire/system/snapshot.command.ts | 103 +
foreign/node/src/wire/vsr/index.ts | 33 +-
foreign/node/src/wire/vsr/operation.test.ts | 7 +-
foreign/node/src/wire/vsr/operation.ts | 11 +-
foreign/node/src/wire/vsr/register.test.ts | 41 +-
foreign/node/src/wire/vsr/register.ts | 40 +-
foreign/node/src/wire/vsr/session.test.ts | 15 +-
foreign/node/src/wire/vsr/session.ts | 17 +-
foreign/node/src/wire/vsr/vsr.test.ts | 10 +-
foreign/php/Dockerfile.test | 2 +-
foreign/python/README.md | 30 +-
foreign/python/apache_iggy.pyi | 18 +-
foreign/python/src/client.rs | 11 +-
foreign/python/src/send_message.rs | 13 +-
foreign/python/tests/test_producer.py | 76 +-
.../swift/Sources/Iggy/Errors/IggyErrorCode.swift | 4 +
foreign/swift/Tests/IggyTests/Fixtures/golden.json | 13 +-
foreign/swift/Tools/golden-vectors/src/main.rs | 16 +
gateways/kafka/Dockerfile | 2 +-
gateways/kafka/src/protocol/acl.rs | 4 +-
gateways/kafka/src/protocol/handlers/metadata.rs | 2 +-
.../kafka/tests/bridge_iggy_integration_tests.rs | 2 +-
gateways/kafka/tests/consumer_group_tests.rs | 10 +-
gateways/kafka/tests/fetch_real_bridge_tests.rs | 8 +-
gateways/kafka/tests/metadata_real_bridge_tests.rs | 2 +-
gateways/kafka/tests/produce_real_bridge_tests.rs | 2 +-
gateways/kafka/tests/server_e2e_tests.rs | 2 +-
justfile | 6 +-
rust-toolchain.toml | 2 +-
scripts/ci/coverage-baseline-affected.sh | 6 +-
scripts/ci/cpp-coverage.sh | 84 +
scripts/ci/edge-affected-images.sh | 150 -
scripts/ci/edge-image-catalog.sh | 88 +
scripts/ci/sync-rustc-version.sh | 21 +
web/package-lock.json | 18 +-
329 files changed, 21666 insertions(+), 9033 deletions(-)
create mode 100644 .github/config/edge-image-variants.json
delete mode 100644
core/binary_protocol/src/requests/system/attach_consumer_session.rs
create mode 100644 core/binary_protocol/src/requests/system/bind_session.rs
create mode 100644 core/binary_protocol/src/requests/system/retire_sessions.rs
create mode 100644
core/integration/tests/cli/consumer_offset/test_consumer_offset_external_group_command.rs
create mode 100644
core/integration/tests/server/scenarios/external_group_offsets_scenario.rs
create mode 100644 foreign/node/src/wire/system/snapshot.command.test.ts
create mode 100644 foreign/node/src/wire/system/snapshot.command.ts
create mode 100755 scripts/ci/cpp-coverage.sh
delete mode 100755 scripts/ci/edge-affected-images.sh
create mode 100755 scripts/ci/edge-image-catalog.sh
