This is an automated email from the ASF dual-hosted git repository.
spetz pushed a change to branch fix/consumer-recovery
in repository https://gitbox.apache.org/repos/asf/iggy.git
from e71bdb88d fix
add e8912af1d feat(partitions)!: add idempotent producers to Iggy (#4691)
add 3078e0b98 merge
No new revisions were added by this update.
Summary of changes:
Cargo.lock | 8 +-
Cargo.toml | 8 +-
bdd/python/uv.lock | 2 +-
.../bench/src/actors/producer/client/high_level.rs | 14 +-
core/binary_protocol/Cargo.toml | 2 +-
core/binary_protocol/src/batch.rs | 254 ++++++++-
.../src/requests/messages/send_messages.rs | 20 +-
.../src/responses/messages/fetch_messages.rs | 6 +-
core/cli/Cargo.toml | 2 +-
core/common/Cargo.toml | 2 +-
core/common/src/error/iggy_error.rs | 23 +
core/common/src/lib.rs | 5 +-
core/common/src/traits/binary_impls/messages.rs | 107 +++-
core/common/src/traits/binary_impls/mod.rs | 3 +-
core/common/src/traits/binary_transport.rs | 6 +
core/common/src/traits/message_client.rs | 39 +-
core/common/src/traits/mod.rs | 3 +-
core/connectors/runtime/README.md | 41 ++
core/connectors/runtime/src/configs/connectors.rs | 17 +-
core/connectors/runtime/src/exactly_once.rs | 613 +++++++++++++++++++++
core/connectors/runtime/src/main.rs | 3 +
core/connectors/runtime/src/manager/source.rs | 7 +-
core/connectors/runtime/src/source.rs | 127 ++++-
core/connectors/sdk/README.md | 4 +
core/connectors/sources/README.md | 5 +
.../tests/cluster/idempotent_producer.rs | 509 +++++++++++++++++
core/integration/tests/cluster/mod.rs | 1 +
.../tests/connectors/runtime/http_state.rs | 275 ++++++++-
...ttp_state.toml => http_state_exactly_once.toml} | 2 +-
.../random_source.toml | 12 +-
core/kafka_wire/src/api.rs | 14 +-
core/kafka_wire/src/records.rs | 42 +-
core/partitions/src/iggy_partition.rs | 225 +++++++-
core/partitions/src/lib.rs | 2 +
core/partitions/src/producer_state.rs | 535 ++++++++++++++++++
core/partitions/src/state_transfer.rs | 279 +++++++++-
core/sdk/Cargo.toml | 2 +-
core/sdk/README.md | 52 ++
core/sdk/src/binary/mod.rs | 4 +
.../src/client_wrappers/binary_message_client.rs | 29 +-
core/sdk/src/client_wrappers/client_wrapper.rs | 40 +-
core/sdk/src/clients/binary_message.rs | 28 +-
core/sdk/src/clients/mod.rs | 1 +
core/sdk/src/clients/producer.rs | 573 ++++++++++++++++++-
core/sdk/src/clients/producer_builder.rs | 30 +
core/sdk/src/clients/producer_idempotence.rs | 308 +++++++++++
core/sdk/src/prelude.rs | 18 +-
core/sdk/src/session_owner.rs | 25 +-
core/sdk/src/session_owner/producer_ids.rs | 135 +++++
core/server/src/dispatch/partition.rs | 19 +
core/server/src/namespace.rs | 10 +
core/server/src/partition_helpers.rs | 4 +-
core/simulator/src/client.rs | 34 +-
core/simulator/src/idempotent_producer_tests.rs | 231 ++++++++
core/simulator/src/lib.rs | 4 +
core/simulator/src/stale_prepare_commit_tests.rs | 6 +-
examples/python/uv.lock | 2 +-
foreign/python/Cargo.toml | 4 +-
foreign/python/pyproject.toml | 2 +-
foreign/python/uv.lock | 2 +-
.../swift/Sources/Iggy/Errors/IggyErrorCode.swift | 10 +
foreign/swift/Sources/Iggy/Iggy.swift | 2 +-
foreign/swift/Sources/Iggy/Protocol/Batch.swift | 61 +-
foreign/swift/Tests/IggyTests/Fixtures/golden.json | 14 +-
.../Tests/IggyTests/ResponseVectorTests.swift | 7 +
foreign/swift/Tools/golden-vectors/src/main.rs | 20 +-
gateways/kafka/README.md | 52 +-
gateways/kafka/docs/BRIDGE_MAPPING.md | 17 +-
gateways/kafka/docs/IDEMPOTENCE.md | 230 +++-----
gateways/kafka/docs/SCOPE.md | 5 +-
gateways/kafka/src/bridge/config.rs | 43 +-
gateways/kafka/src/bridge/error.rs | 20 +-
gateways/kafka/src/bridge/iggy_bridge/mod.rs | 16 +-
gateways/kafka/src/bridge/iggy_bridge/produce.rs | 76 ++-
.../kafka/src/bridge/iggy_bridge/producer_ids.rs | 170 ++++++
gateways/kafka/src/bridge/mod.rs | 6 +-
gateways/kafka/src/protocol/api.rs | 10 +
.../src/protocol/handlers/init_producer_id.rs | 54 +-
gateways/kafka/src/protocol/handlers/produce.rs | 153 ++++-
.../kafka/tests/bridge_iggy_integration_tests.rs | 12 +-
gateways/kafka/tests/common/gateway_process.rs | 4 +
gateways/kafka/tests/common/iggy_server.rs | 2 +
gateways/kafka/tests/kafka_client_e2e_tests.rs | 238 +++++++-
gateways/kafka/tests/produce_real_bridge_tests.rs | 160 +++++-
84 files changed, 5729 insertions(+), 433 deletions(-)
create mode 100644 core/connectors/runtime/src/exactly_once.rs
create mode 100644 core/integration/tests/cluster/idempotent_producer.rs
copy core/integration/tests/connectors/runtime/{http_state.toml =>
http_state_exactly_once.toml} (93%)
copy core/integration/tests/connectors/runtime/{http_state_config =>
http_state_exactly_once_config}/random_source.toml (80%)
create mode 100644 core/partitions/src/producer_state.rs
create mode 100644 core/sdk/src/clients/producer_idempotence.rs
create mode 100644 core/sdk/src/session_owner/producer_ids.rs
create mode 100644 core/simulator/src/idempotent_producer_tests.rs
create mode 100644 gateways/kafka/src/bridge/iggy_bridge/producer_ids.rs
