This is an automated email from the ASF dual-hosted git repository.
krishvishal pushed a change to branch sim-workload-faults
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 358d4aadc fix: address review comments
add f8311d651 feat(cpp): add functions related to user management, and
initial high-level client (#3733)
add 2bd9d0bce fix(shard): tell an unknown consensus operation from a
corrupt header (#3931)
add a2a8f0491 chore(deps): Bump the csharp group with 12 updates (#3961)
add 228f47e0c refactor(csharp): cleanup tcp connection after vsr
implementation (#3858)
add 7207c640a ci(csharp): separate baseline coverage reports per framework
(#3960)
add 4e5ec2586 fix(consensus): keep a client's dedup fence across capacity
eviction (#3934)
add 2cbf6c15e chore(repo): bump Rust toolchain to 1.98.0 (#3962)
add 1a9874859 fix(connectors): Harden Elasticsearch sink test readiness
(#3729)
add 4c475dfcf fix(ci): bump cargo-zigbuild so arm64 images link under Rust
1.98 (#3963)
add 37d7eee35 refactor(sdk): introduce NonZeroIggyDuration for retries and
heartbeats (#3891)
add 2e420398b feat(connectors): add Redshift sink connector (#3654)
add 579434c14 Merge branch 'master' into sim-workload-faults
No new revisions were added by this update.
Summary of changes:
.devcontainer/Dockerfile | 4 +-
.github/actions/cpp-bazel/pre-merge/action.yml | 25 +-
.github/config/components.yml | 2 +-
.github/workflows/coverage-baseline.yml | 28 +-
.github/workflows/post-merge.yml | 2 +-
Cargo.lock | 358 ++++-
Cargo.toml | 5 +
Dockerfile | 2 +-
README.md | 2 +-
.../features/step_definitions/background_steps.cpp | 5 +-
.../features/step_definitions/messaging_steps.cpp | 18 +-
bdd/php/Dockerfile | 2 +-
bdd/python/Dockerfile | 2 +-
bdd/rust/Dockerfile | 2 +-
core/ai/mcp/Dockerfile | 4 +-
.../frontend/src/components/chart/trend_chart.rs | 2 +-
core/bench/dashboard/server/Dockerfile | 2 +-
core/binary_protocol/src/consensus/error.rs | 11 +
core/binary_protocol/src/consensus/header.rs | 70 +
core/binary_protocol/src/consensus/mod.rs | 2 +-
core/binary_protocol/src/consensus/operation.rs | 11 +
core/binary_protocol/src/lib.rs | 3 +-
core/cli/src/commands/binary_client/get_clients.rs | 4 +-
.../get_personal_access_tokens.rs | 2 +-
core/common/src/lib.rs | 2 +-
core/common/src/traits/binary_impls/system.rs | 6 +-
core/common/src/traits/binary_transport.rs | 4 +-
core/common/src/traits/system_client.rs | 6 +-
.../configuration/auth_config/connection_string.rs | 8 +-
.../auth_config/connection_string_options.rs | 4 +-
.../http_config/http_client_config.rs | 9 +-
.../http_config/http_client_config_builder.rs | 22 +-
.../http_config/http_connection_string_options.rs | 34 +-
.../quic_config/quic_client_config.rs | 6 +-
.../quic_config/quic_client_config_builder.rs | 9 +-
.../quic_config/quic_client_reconnection_config.rs | 9 +-
.../quic_config/quic_connection_string_options.rs | 17 +-
.../configuration/tcp_config/tcp_client_config.rs | 6 +-
.../tcp_config/tcp_client_config_builder.rs | 7 +-
.../tcp_config/tcp_client_reconnection_config.rs | 7 +-
.../tcp_config/tcp_connection_string_options.rs | 17 +-
.../websocket_config/websocket_client_config.rs | 6 +-
.../websocket_client_config_builder.rs | 9 +-
.../websocket_client_reconnection_config.rs | 6 +-
.../websocket_connection_string_options.rs | 17 +-
core/common/src/utils/duration.rs | 395 ++++-
core/connectors/README.md | 1 +
core/connectors/runtime/Dockerfile | 4 +-
core/connectors/sinks/README.md | 1 +
core/connectors/sinks/elasticsearch_sink/README.md | 2 +-
.../connectors/sinks/elasticsearch_sink/src/lib.rs | 20 +-
.../sinks/{s3_sink => redshift_sink}/Cargo.toml | 23 +-
core/connectors/sinks/redshift_sink/README.md | 216 +++
.../{postgres_sink => redshift_sink}/config.toml | 21 +-
core/connectors/sinks/redshift_sink/src/config.rs | 159 ++
core/connectors/sinks/redshift_sink/src/lib.rs | 1491 ++++++++++++++++++
core/consensus/Cargo.toml | 4 +-
core/consensus/src/client_table.rs | 1064 ++++++++++++-
core/consensus/src/lib.rs | 2 +-
core/consensus/src/metadata_helpers.rs | 16 +-
core/integration/Cargo.toml | 6 +
.../src/harness/handle/connectors_runtime.rs | 56 +-
.../tests/cluster/client_table_adversarial.rs | 40 +-
.../connectors/fixtures/elasticsearch/container.rs | 385 ++++-
.../connectors/fixtures/elasticsearch/sink.rs | 103 +-
.../connectors/fixtures/elasticsearch/source.rs | 2 -
core/integration/tests/connectors/fixtures/mod.rs | 5 +
.../connectors/fixtures/redshift/container.rs | 276 ++++
.../fixtures/{clickhouse => redshift}/mod.rs | 5 +-
.../fixtures/redshift/redshift_mock/ddl.rs | 77 +
.../fixtures/redshift/redshift_mock/dml.rs | 209 +++
.../fixtures/redshift/redshift_mock/dql.rs | 92 ++
.../fixtures/redshift/redshift_mock/handler.rs | 256 ++++
.../fixtures/redshift/redshift_mock/load.rs | 314 ++++
.../fixtures/redshift/redshift_mock}/mod.rs | 8 +-
.../fixtures/redshift/redshift_mock/parser.rs | 556 +++++++
.../fixtures/redshift/redshift_mock/util.rs | 152 ++
.../tests/connectors/fixtures/redshift/sink.rs | 480 ++++++
core/integration/tests/connectors/mod.rs | 1 +
.../tests/connectors/{http => redshift}/mod.rs | 2 +-
.../tests/connectors/redshift/redshift_sink.rs | 418 +++++
.../tests/connectors/{delta => redshift}/sink.toml | 2 +-
.../verify_consumer_group_partition_assignment.rs | 4 +-
...umer_group_auto_commit_reconnection_scenario.rs | 2 +-
.../consumer_group_offset_cleanup_scenario.rs | 2 +-
.../consumer_timestamp_polling_scenario.rs | 2 +-
.../server/scenarios/purge_delete_scenario.rs | 4 +-
.../scenarios/reconnect_after_restart_scenario.rs | 4 +-
.../stale_client_consumer_group_scenario.rs | 8 +-
core/message_bus/Cargo.toml | 4 +-
core/message_bus/src/replica/handshake.rs | 2 +-
core/metadata/Cargo.toml | 4 +-
core/metadata/src/impls/metadata.rs | 57 +-
core/metadata/src/impls/recovery.rs | 11 +-
core/metadata/src/stm/snapshot.rs | 87 +-
core/partitions/Cargo.toml | 4 +-
core/sdk/README.md | 2 +-
core/sdk/src/client_provider.rs | 40 +-
.../src/client_wrappers/binary_system_client.rs | 6 +-
core/sdk/src/clients/binary_system.rs | 6 +-
core/sdk/src/clients/client_builder.rs | 20 +-
core/sdk/src/clients/consumer.rs | 60 +-
core/sdk/src/clients/consumer_builder.rs | 18 +-
core/sdk/src/clients/producer.rs | 8 +-
core/sdk/src/clients/producer_builder.rs | 9 +-
core/sdk/src/http/http_client.rs | 17 +-
core/sdk/src/http/system.rs | 4 +-
core/sdk/src/prelude.rs | 7 +-
core/sdk/src/quic/quic_client.rs | 36 +-
.../stream_builder/config/config_iggy_consumer.rs | 55 +-
.../stream_builder/config/config_iggy_producer.rs | 20 +-
core/sdk/src/tcp/tcp_client.rs | 48 +-
core/sdk/src/websocket/websocket_client.rs | 29 +-
core/server/Cargo.toml | 4 +-
core/server/Dockerfile | 4 +-
core/server/src/dispatch.rs | 18 +-
core/server/src/partition_reconciler.rs | 124 ++
core/server/src/segment_recovery.rs | 2 +-
core/server_common/src/consensus_message.rs | 219 ++-
core/shard/src/metrics.rs | 14 +-
core/shard/src/router.rs | 98 +-
core/simulator/Cargo.toml | 4 +-
.../Iggy_SDK.Examples.Basic.Consumer.csproj | 8 +-
.../Iggy_SDK.Examples.Basic.Producer.csproj | 10 +-
...ggy_SDK.Examples.GettingStarted.Consumer.csproj | 4 +-
...ggy_SDK.Examples.GettingStarted.Producer.csproj | 4 +-
.../Iggy_SDK.Examples.Shared.csproj | 2 +-
...gy_SDK.Examples.MessageEnvelope.Consumer.csproj | 10 +-
...gy_SDK.Examples.MessageEnvelope.Producer.csproj | 10 +-
...ggy_SDK.Examples.MessageHeaders.Consumer.csproj | 4 +-
...ggy_SDK.Examples.MessageHeaders.Producer.csproj | 4 +-
.../Iggy_SDK.Examples.NewSdk.Consumer.csproj | 10 +-
.../Iggy_SDK.Examples.NewSdk.Producer.csproj | 10 +-
.../Iggy_SDK.Examples.TcpTls.Consumer.csproj | 4 +-
.../Iggy_SDK.Examples.TcpTls.Producer.csproj | 4 +-
.../stream-builder/stream-consumer-config/main.rs | 4 +-
.../stream-builder/stream-producer-config/main.rs | 2 +-
foreign/cpp/.bazelrc | 5 +-
foreign/cpp/include/iggy.hpp | 917 ++++++++---
foreign/cpp/src/client.cpp | 200 +++
foreign/cpp/src/client.rs | 197 ++-
foreign/cpp/src/lib.rs | 89 +-
foreign/cpp/src/type_conversion.rs | 146 +-
foreign/cpp/tests/e2e/client.cpp | 1606 +++++++++++++++++++-
foreign/cpp/tests/e2e/test_helpers.hpp | 202 +--
foreign/cpp/tests/e2e/topic.cpp | 10 +-
foreign/cpp/tests/unit/unit_tests.cpp | 169 +-
foreign/csharp/Directory.Packages.props | 14 +-
.../Iggy_SDK.Tests.Integration/OptionsTests.cs | 189 +++
.../Iggy_SDK/ConnectionStream/IConnectionStream.cs | 26 -
.../ConnectionStream/TcpConnectionStream.cs | 53 -
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 3 +-
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 99 +-
.../Implementations/TcpMessageStream.Vsr.cs | 494 ++----
.../IggyClient/Implementations/TcpMessageStream.cs | 535 ++-----
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 2 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 4 +-
.../Utils/ArrayPoolHelper.cs} | 34 +-
foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs | 23 -
.../Iggy_SDK/Utils/TcpMessageStreamHelpers.cs | 19 -
foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 19 +
.../{Command.cs => ISessionGenerationProvider.cs} | 14 +-
foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs | 372 +++++
.../ConsumerSessionGenerationTests.cs | 245 +++
.../ConsumerTests/RentedConsumerTests.cs | 4 +-
.../MessageBatchGoldenVectorTests.cs | 4 +-
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 11 +-
.../UtilityTests/SlicedMemoryOwnerTests.cs | 85 --
.../Utils/Errors/ErrorModelFactory.cs | 33 -
.../VsrTests/ConsensusSessionTests.cs | 20 +
foreign/php/Dockerfile.test | 2 +-
foreign/php/src/client.rs | 15 +-
foreign/php/src/consumer.rs | 25 +-
foreign/php/tests/IggySdkTest.php | 58 +-
foreign/python/apache_iggy.pyi | 3 +-
foreign/python/src/config.rs | 21 +-
foreign/python/src/consumer.rs | 6 +-
foreign/python/src/duration.rs | 16 +-
foreign/python/tests/test_client_config.py | 77 +-
rust-toolchain.toml | 2 +-
180 files changed, 12781 insertions(+), 2087 deletions(-)
copy core/connectors/sinks/{s3_sink => redshift_sink}/Cargo.toml (71%)
create mode 100644 core/connectors/sinks/redshift_sink/README.md
copy core/connectors/sinks/{postgres_sink => redshift_sink}/config.toml (70%)
create mode 100644 core/connectors/sinks/redshift_sink/src/config.rs
create mode 100644 core/connectors/sinks/redshift_sink/src/lib.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/container.rs
copy core/integration/tests/connectors/fixtures/{clickhouse =>
redshift}/mod.rs (80%)
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/ddl.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/dml.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/dql.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/handler.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/load.rs
copy core/{bench/dashboard/frontend/src/components/theme =>
integration/tests/connectors/fixtures/redshift/redshift_mock}/mod.rs (88%)
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/parser.rs
create mode 100644
core/integration/tests/connectors/fixtures/redshift/redshift_mock/util.rs
create mode 100644 core/integration/tests/connectors/fixtures/redshift/sink.rs
copy core/integration/tests/connectors/{http => redshift}/mod.rs (97%)
create mode 100644 core/integration/tests/connectors/redshift/redshift_sink.rs
copy core/integration/tests/connectors/{delta => redshift}/sink.toml (94%)
create mode 100644 foreign/cpp/src/client.cpp
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK/ConnectionStream/IConnectionStream.cs
delete mode 100644
foreign/csharp/Iggy_SDK/ConnectionStream/TcpConnectionStream.cs
copy
foreign/csharp/{Iggy_SDK.Tests.Integration/Attributes/SkipHttpAttribute.cs =>
Iggy_SDK/Utils/ArrayPoolHelper.cs} (53%)
delete mode 100644 foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs
copy foreign/csharp/Iggy_SDK/Vsr/{Command.cs => ISessionGenerationProvider.cs}
(58%)
create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs
create mode 100644
foreign/csharp/Iggy_SDK_Tests/ConsumerTests/ConsumerSessionGenerationTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs
delete mode 100644
foreign/csharp/Iggy_SDK_Tests/Utils/Errors/ErrorModelFactory.cs
