This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a change to branch fix_late_reply
in repository https://gitbox.apache.org/repos/asf/iggy.git
from d489ea131 merge master
add 69429e5e2 feat(connectors): store source state on an HTTP state server
(#3940)
add f8311d651 feat(cpp): add functions related to user management, and
initial high-level client (#3733)
add 2bd9d0bce fix(shard): tell an unknown consensus operation from a
corrupt header (#3931)
add a2a8f0491 chore(deps): Bump the csharp group with 12 updates (#3961)
add 228f47e0c refactor(csharp): cleanup tcp connection after vsr
implementation (#3858)
add 7207c640a ci(csharp): separate baseline coverage reports per framework
(#3960)
add 4e5ec2586 fix(consensus): keep a client's dedup fence across capacity
eviction (#3934)
add 2cbf6c15e chore(repo): bump Rust toolchain to 1.98.0 (#3962)
add 1a9874859 fix(connectors): Harden Elasticsearch sink test readiness
(#3729)
add 4c475dfcf fix(ci): bump cargo-zigbuild so arm64 images link under Rust
1.98 (#3963)
add 37d7eee35 refactor(sdk): introduce NonZeroIggyDuration for retries and
heartbeats (#3891)
add 2e420398b feat(connectors): add Redshift sink connector (#3654)
add 1e2d2220f fix(sdk): error callback should await shard flush on
producer shutdown (#3953)
add 5068a960a chore(deps): Bump the github-actions group across 1
directory with 2 updates (#3968)
add d7c2b6180 fix(python): read consumer metadata without the consumer
lock (#3888)
add 03f5b3034 fix(server): restore the mimalloc global allocator (#3971)
add ed59b955d chore(deps): Bump gradle from 9.6.1-jdk17 to 9.7.0-jdk17 in
/bdd/java in the docker group across 1 directory (#3972)
add 37aec0877 feat(sdk): fail over to a surviving node when the current
one dies (#3944)
add 5364c3d18 merge master
No new revisions were added by this update.
Summary of changes:
.devcontainer/Dockerfile | 4 +-
.github/actions/cpp-bazel/pre-merge/action.yml | 25 +-
.github/config/components.yml | 2 +-
.github/workflows/_common.yml | 4 +-
.github/workflows/_test_examples.yml | 2 +-
.github/workflows/coverage-baseline.yml | 34 +-
.github/workflows/post-merge.yml | 4 +-
Cargo.lock | 362 ++++-
Cargo.toml | 6 +
Dockerfile | 2 +-
README.md | 2 +-
.../features/step_definitions/background_steps.cpp | 5 +-
.../features/step_definitions/messaging_steps.cpp | 18 +-
bdd/java/Dockerfile | 2 +-
bdd/php/Dockerfile | 2 +-
bdd/python/Dockerfile | 2 +-
bdd/rust/Dockerfile | 2 +-
bdd/rust/tests/helpers/cluster.rs | 19 +
bdd/rust/tests/steps/leader_redirection.rs | 14 +-
core/ai/mcp/Dockerfile | 4 +-
.../frontend/src/components/chart/trend_chart.rs | 2 +-
core/bench/dashboard/server/Dockerfile | 2 +-
.../actors/consumer/typed_benchmark_consumer.rs | 12 +-
core/binary_protocol/src/consensus/error.rs | 9 +-
core/binary_protocol/src/consensus/header.rs | 56 +
core/binary_protocol/src/consensus/mod.rs | 2 +-
core/binary_protocol/src/lib.rs | 3 +-
core/cli/src/commands/binary_client/get_clients.rs | 4 +-
.../get_personal_access_tokens.rs | 2 +-
core/common/src/lib.rs | 2 +-
.../traits/binary_impls/personal_access_tokens.rs | 7 +-
core/common/src/traits/binary_impls/system.rs | 6 +-
core/common/src/traits/binary_impls/users.rs | 12 +-
core/common/src/traits/binary_transport.rs | 18 +-
core/common/src/traits/system_client.rs | 6 +-
.../configuration/auth_config/connection_string.rs | 8 +-
.../auth_config/connection_string_options.rs | 4 +-
.../http_config/http_client_config.rs | 9 +-
.../http_config/http_client_config_builder.rs | 22 +-
.../http_config/http_connection_string_options.rs | 34 +-
.../quic_config/quic_client_config.rs | 6 +-
.../quic_config/quic_client_config_builder.rs | 9 +-
.../quic_config/quic_client_reconnection_config.rs | 9 +-
.../quic_config/quic_connection_string_options.rs | 17 +-
.../configuration/tcp_config/tcp_client_config.rs | 6 +-
.../tcp_config/tcp_client_config_builder.rs | 7 +-
.../tcp_config/tcp_client_reconnection_config.rs | 40 +-
.../tcp_config/tcp_connection_string_options.rs | 17 +-
.../websocket_config/websocket_client_config.rs | 6 +-
.../websocket_client_config_builder.rs | 9 +-
.../websocket_client_reconnection_config.rs | 6 +-
.../websocket_connection_string_options.rs | 17 +-
core/common/src/utils/duration.rs | 395 ++++-
core/connectors/README.md | 1 +
core/connectors/runtime/Cargo.toml | 4 +
core/connectors/runtime/Dockerfile | 4 +-
core/connectors/runtime/README.md | 61 +
core/connectors/runtime/config.toml | 23 +-
core/connectors/runtime/example_config/config.toml | 22 +-
core/connectors/runtime/src/api/auth.rs | 7 +-
core/connectors/runtime/src/api/source.rs | 1 -
core/connectors/runtime/src/configs/runtime.rs | 241 ++-
core/connectors/runtime/src/context.rs | 7 +-
core/connectors/runtime/src/error.rs | 10 +
core/connectors/runtime/src/main.rs | 8 +-
core/connectors/runtime/src/manager/source.rs | 22 +-
core/connectors/runtime/src/source.rs | 134 +-
core/connectors/runtime/src/state.rs | 146 ++
core/connectors/runtime/src/state/http.rs | 1539 ++++++++++++++++++
core/connectors/runtime/src/stream.rs | 102 +-
core/connectors/sdk/src/lib.rs | 15 +
core/connectors/sinks/README.md | 1 +
core/connectors/sinks/elasticsearch_sink/README.md | 2 +-
.../connectors/sinks/elasticsearch_sink/src/lib.rs | 20 +-
.../sinks/{s3_sink => redshift_sink}/Cargo.toml | 23 +-
core/connectors/sinks/redshift_sink/README.md | 216 +++
.../{postgres_sink => redshift_sink}/config.toml | 21 +-
core/connectors/sinks/redshift_sink/src/config.rs | 159 ++
core/connectors/sinks/redshift_sink/src/lib.rs | 1491 +++++++++++++++++
core/consensus/Cargo.toml | 4 +-
core/consensus/src/client_table.rs | 773 ++++++++-
core/consensus/src/lib.rs | 2 +-
core/consensus/src/metadata_helpers.rs | 16 +-
core/integration/Cargo.toml | 6 +
.../src/harness/handle/connectors_runtime.rs | 56 +-
.../connectors/fixtures/elasticsearch/container.rs | 385 ++++-
.../connectors/fixtures/elasticsearch/sink.rs | 103 +-
.../connectors/fixtures/elasticsearch/source.rs | 2 -
core/integration/tests/connectors/fixtures/mod.rs | 5 +
.../connectors/fixtures/redshift/container.rs | 276 ++++
.../fixtures/{clickhouse => redshift}/mod.rs | 5 +-
.../fixtures/redshift/redshift_mock/ddl.rs | 77 +
.../fixtures/redshift/redshift_mock/dml.rs | 209 +++
.../fixtures/redshift/redshift_mock/dql.rs | 92 ++
.../fixtures/redshift/redshift_mock/handler.rs | 256 +++
.../fixtures/redshift/redshift_mock/load.rs | 314 ++++
.../fixtures/redshift/redshift_mock}/mod.rs | 8 +-
.../fixtures/redshift/redshift_mock/parser.rs | 556 +++++++
.../fixtures/redshift/redshift_mock/util.rs | 152 ++
.../tests/connectors/fixtures/redshift/sink.rs | 480 ++++++
core/integration/tests/connectors/mod.rs | 1 +
.../tests/connectors/{http => redshift}/mod.rs | 2 +-
.../tests/connectors/redshift/redshift_sink.rs | 418 +++++
.../tests/connectors/{delta => redshift}/sink.toml | 2 +-
.../tests/connectors/runtime/http_state.rs | 417 +++++
.../{benchmark_json.toml => http_state.toml} | 16 +-
.../random_source.toml | 11 +-
core/integration/tests/connectors/runtime/mod.rs | 1 +
.../verify_consumer_group_partition_assignment.rs | 4 +-
core/integration/tests/sdk/disconnect_relogin.rs | 11 +-
...umer_group_auto_commit_reconnection_scenario.rs | 2 +-
.../consumer_group_offset_cleanup_scenario.rs | 2 +-
.../consumer_timestamp_polling_scenario.rs | 2 +-
.../server/scenarios/purge_delete_scenario.rs | 4 +-
.../scenarios/reconnect_after_restart_scenario.rs | 4 +-
.../stale_client_consumer_group_scenario.rs | 8 +-
core/message_bus/Cargo.toml | 4 +-
core/message_bus/src/replica/handshake.rs | 2 +-
core/metadata/Cargo.toml | 4 +-
core/metadata/src/impls/metadata.rs | 57 +-
core/metadata/src/impls/recovery.rs | 11 +-
core/metadata/src/stm/snapshot.rs | 87 +-
core/partitions/Cargo.toml | 4 +-
core/sdk/README.md | 2 +-
core/sdk/src/client_provider.rs | 40 +-
.../src/client_wrappers/binary_system_client.rs | 6 +-
.../src/clients/binary_personal_access_tokens.rs | 10 +
core/sdk/src/clients/binary_system.rs | 6 +-
core/sdk/src/clients/binary_users.rs | 10 +
core/sdk/src/clients/client_builder.rs | 20 +-
core/sdk/src/clients/consumer.rs | 448 ++++--
core/sdk/src/clients/consumer_builder.rs | 18 +-
core/sdk/src/clients/mod.rs | 33 +
core/sdk/src/clients/producer.rs | 8 +-
core/sdk/src/clients/producer_builder.rs | 9 +-
core/sdk/src/clients/producer_dispatcher.rs | 110 +-
core/sdk/src/http/http_client.rs | 17 +-
core/sdk/src/http/system.rs | 4 +-
core/sdk/src/leader_aware.rs | 205 ++-
core/sdk/src/prelude.rs | 9 +-
core/sdk/src/quic/quic_client.rs | 42 +-
.../stream_builder/config/config_iggy_consumer.rs | 55 +-
.../stream_builder/config/config_iggy_producer.rs | 20 +-
core/sdk/src/tcp/tcp_client.rs | 1680 ++++++++++++++++----
core/sdk/src/vsr.rs | 2 +-
core/sdk/src/websocket/websocket_client.rs | 35 +-
core/server/Cargo.toml | 4 +-
core/server/Dockerfile | 4 +-
core/server/src/dispatch.rs | 18 +-
core/server/src/lib.rs | 8 +
core/server/src/main.rs | 4 +
core/server/src/partition_reconciler.rs | 71 +-
core/server/src/segment_recovery.rs | 2 +-
core/server_common/src/consensus_message.rs | 192 ++-
core/shard/src/metrics.rs | 2 +-
core/shard/src/router.rs | 53 +-
core/simulator/Cargo.toml | 4 +-
.../Iggy_SDK.Examples.Basic.Consumer.csproj | 8 +-
.../Iggy_SDK.Examples.Basic.Producer.csproj | 10 +-
...ggy_SDK.Examples.GettingStarted.Consumer.csproj | 4 +-
...ggy_SDK.Examples.GettingStarted.Producer.csproj | 4 +-
.../Iggy_SDK.Examples.Shared.csproj | 2 +-
...gy_SDK.Examples.MessageEnvelope.Consumer.csproj | 10 +-
...gy_SDK.Examples.MessageEnvelope.Producer.csproj | 10 +-
...ggy_SDK.Examples.MessageHeaders.Consumer.csproj | 4 +-
...ggy_SDK.Examples.MessageHeaders.Producer.csproj | 4 +-
.../Iggy_SDK.Examples.NewSdk.Consumer.csproj | 10 +-
.../Iggy_SDK.Examples.NewSdk.Producer.csproj | 10 +-
.../Iggy_SDK.Examples.TcpTls.Consumer.csproj | 4 +-
.../Iggy_SDK.Examples.TcpTls.Producer.csproj | 4 +-
.../stream-builder/stream-consumer-config/main.rs | 4 +-
.../stream-builder/stream-producer-config/main.rs | 2 +-
foreign/cpp/.bazelrc | 5 +-
foreign/cpp/include/iggy.hpp | 917 ++++++++---
foreign/cpp/src/client.cpp | 200 +++
foreign/cpp/src/client.rs | 197 ++-
foreign/cpp/src/lib.rs | 89 +-
foreign/cpp/src/type_conversion.rs | 146 +-
foreign/cpp/tests/e2e/client.cpp | 1606 ++++++++++++++++++-
foreign/cpp/tests/e2e/test_helpers.hpp | 202 ++-
foreign/cpp/tests/e2e/topic.cpp | 10 +-
foreign/cpp/tests/unit/unit_tests.cpp | 169 +-
foreign/csharp/Directory.Packages.props | 14 +-
.../Iggy_SDK.Tests.Integration/HeartbeatTests.cs | 31 +-
.../Iggy_SDK.Tests.Integration/OptionsTests.cs | 189 +++
.../Iggy_SDK/ConnectionStream/IConnectionStream.cs | 26 -
.../ConnectionStream/TcpConnectionStream.cs | 53 -
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 3 +-
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 99 +-
.../Implementations/TcpMessageStream.Vsr.cs | 504 ++----
.../IggyClient/Implementations/TcpMessageStream.cs | 647 +++-----
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 2 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 4 +-
.../Utils/ArrayPoolHelper.cs} | 34 +-
foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs | 23 -
.../Iggy_SDK/Utils/TcpMessageStreamHelpers.cs | 19 -
foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 19 +
.../{Command.cs => ISessionGenerationProvider.cs} | 14 +-
foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs | 372 +++++
.../ConsumerSessionGenerationTests.cs | 245 +++
.../ConsumerTests/RentedConsumerTests.cs | 4 +-
.../MessageBatchGoldenVectorTests.cs | 4 +-
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 11 +-
.../UtilityTests/SlicedMemoryOwnerTests.cs | 85 -
.../Utils/Errors/ErrorModelFactory.cs | 33 -
.../VsrTests/ConsensusSessionTests.cs | 20 +
.../Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs | 15 +
.../VsrTests/EndpointFailoverTests.cs | 188 ++-
foreign/csharp/README.md | 27 +-
foreign/go/client/tcp/tcp_connect_test.go | 15 +-
foreign/go/client/tcp/tcp_core.go | 350 +++-
foreign/go/client/tcp/tcp_core_review_test.go | 13 +-
foreign/go/client/tcp/tcp_failover_test.go | 604 ++++++-
foreign/go/client/tcp/tcp_session_management.go | 90 +-
.../iggy/client/async/tcp/AsyncIggyTcpClient.java | 273 +++-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 61 +-
.../iggy/client/async/tcp/LeaderAwareness.java | 29 +-
.../iggy/client/async/tcp/ReconnectPlan.java | 20 +-
.../client/async/tcp/vsr/VsrResponseHandler.java | 11 +-
.../AsyncIggyTcpClientEndpointFailoverTest.java | 264 ++-
.../AsyncIggyTcpClientTransientFailoverTest.java | 197 ++-
.../tcp/AsyncTcpConnectionConcurrencyTest.java | 4 +-
.../iggy/client/async/tcp/LeaderAwarenessTest.java | 73 +
.../iggy/client/async/tcp/ReconnectPlanTest.java | 28 -
.../async/tcp/vsr/VsrResponseHandlerTest.java | 10 +-
foreign/node/src/client/client.connection.test.ts | 334 ++++
foreign/node/src/client/client.connection.ts | 186 ++-
foreign/node/src/client/client.socket.test.ts | 390 ++++-
foreign/node/src/client/client.socket.ts | 360 ++++-
foreign/node/src/client/client.type.ts | 26 +-
foreign/php/Dockerfile.test | 2 +-
foreign/php/src/client.rs | 15 +-
foreign/php/src/consumer.rs | 25 +-
foreign/php/tests/IggySdkTest.php | 58 +-
foreign/python/apache_iggy.pyi | 34 +-
foreign/python/pyproject.toml | 4 +
foreign/python/src/client.rs | 8 +
foreign/python/src/config.rs | 40 +-
foreign/python/src/consumer.rs | 126 +-
foreign/python/src/duration.rs | 16 +-
foreign/python/tests/test_client_config.py | 77 +-
foreign/python/tests/test_consumer_group.py | 118 ++
rust-toolchain.toml | 2 +-
243 files changed, 20572 insertions(+), 3168 deletions(-)
create mode 100644 core/connectors/runtime/src/state/http.rs
copy core/connectors/sinks/{s3_sink => redshift_sink}/Cargo.toml (71%)
create mode 100644 core/connectors/sinks/redshift_sink/README.md
copy core/connectors/sinks/{postgres_sink => redshift_sink}/config.toml (70%)
create mode 100644 core/connectors/sinks/redshift_sink/src/config.rs
create mode 100644 core/connectors/sinks/redshift_sink/src/lib.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/container.rs
copy core/integration/tests/connectors/fixtures/{clickhouse =>
redshift}/mod.rs (80%)
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/ddl.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/dml.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/dql.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/handler.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/load.rs
copy core/{bench/dashboard/frontend/src/components/theme =>
integration/tests/connectors/fixtures/redshift/redshift_mock}/mod.rs (88%)
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/parser.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/util.rs
create mode 100644 core/integration/tests/connectors/fixtures/redshift/sink.rs
copy core/integration/tests/connectors/{http => redshift}/mod.rs (97%)
create mode 100644 core/integration/tests/connectors/redshift/redshift_sink.rs
copy core/integration/tests/connectors/{delta => redshift}/sink.toml (94%)
create mode 100644 core/integration/tests/connectors/runtime/http_state.rs
copy core/integration/tests/connectors/runtime/{benchmark_json.toml =>
http_state.toml} (73%)
copy core/integration/tests/connectors/runtime/{benchmark_config =>
http_state_config}/random_source.toml (89%)
create mode 100644 foreign/cpp/src/client.cpp
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK/ConnectionStream/IConnectionStream.cs
delete mode 100644
foreign/csharp/Iggy_SDK/ConnectionStream/TcpConnectionStream.cs
copy
foreign/csharp/{Iggy_SDK.Tests.Integration/Attributes/SkipHttpAttribute.cs =>
Iggy_SDK/Utils/ArrayPoolHelper.cs} (53%)
delete mode 100644 foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs
copy foreign/csharp/Iggy_SDK/Vsr/{Command.cs => ISessionGenerationProvider.cs}
(58%)
create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs
create mode 100644
foreign/csharp/Iggy_SDK_Tests/ConsumerTests/ConsumerSessionGenerationTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK_Tests/Utils/Errors/ErrorModelFactory.cs
