This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a change to branch
dependabot/npm_and_yarn/foreign/node/node-febceb97ad
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 0cc7475ea chore(deps): Bump @types/node
add 768ea8093 feat(gateways): SASL/PLAIN authentication for the Kafka
gateway (#4222)
add 1951e654c chore(integration): replace MinIO with Floci (#4290)
add b8bf85fd9 feat(swift): scaffold the Swift SDK package and CI lanes
(#4190)
add 8d0e28d9c feat(gateways): send Kafka records to Iggy from the Produce
handler (#4282)
add 15a49d082 feat(gateways): Implement CreateTopics and Metadata Kafka
Keys backed by the Iggy bridge (#4258)
add 3adf4fe39 docs: add how to report a bug found by an agent (#4309)
add 105ebe52f chore(deps): Bump rules_cc from 0.2.22 to 0.2.25 in
/foreign/cpp in the cpp group across 1 directory (#4310)
add e8cb28790 chore(deps): Bump the go group across 1 directory with 2
updates (#4300)
add d2f970661 fix(server): recover stalled consumer groups after restarts
(#4283)
add 11dd94c62 test(go): fail when a manual group commit on a split primary
drops membership (#4293)
add 8152a6074 feat(gateways): describe Iggy permissions as Kafka ACLs
(#4245)
add 32278f5dd refactor(server): move recovery readers to partitions, add
ShardHost (#4312)
add 91df1da99 feat(gateways): support idempotent Kafka producers, refuse
transactions (#4250)
add b2742e529 ci: resolve the PR triage trigger from the API, not the
artifact (#4257)
add 47731e34d feat(gateways): coordinate Kafka consumer groups (#4263)
add 3c2e7e8b5 chore(deps): Bump the web group across 1 directory with 5
updates (#4296)
add e325d63da Merge branch 'master' into
dependabot/npm_and_yarn/foreign/node/node-febceb97ad
No new revisions were added by this update.
Summary of changes:
.config/nextest.toml | 22 +-
.github/actions/rust/pre-merge/action.yml | 23 +
.github/actions/swift/pre-merge/action.yml | 75 +
.github/actions/utils/setup-swift/action.yml | 56 +
.github/config/components.yml | 14 +
.github/workflows/_detect.yml | 11 +-
.github/workflows/_test.yml | 8 +
.github/workflows/pr-triage-apply.yml | 233 ++-
.github/workflows/pr-triage-collect.yml | 6 +-
.github/workflows/pre-merge.yml | 21 +-
.github/workflows/stale-prs-unmark-on-activity.yml | 101 +-
.gitignore | 8 +
AGENTS.md | 3 +-
CONTRIBUTING.md | 9 +
Cargo.lock | 2 +
bdd/docker-compose.cluster.yml | 1 +
bdd/go/go.mod | 4 +-
bdd/go/go.sum | 8 +-
core/binary_protocol/src/consensus/command.rs | 11 +-
.../src/consensus/consumer_session.rs | 218 ++
core/binary_protocol/src/consensus/header.rs | 6 +
core/binary_protocol/src/consensus/mod.rs | 4 +
core/binary_protocol/src/lib.rs | 22 +-
core/cli/src/args/consumer_group.rs | 6 +-
core/configs/src/common/defaults.rs | 10 +
core/configs/src/common/displays.rs | 6 +-
core/configs/src/common/server.rs | 7 +
core/configs/src/common/system.rs | 3 -
core/configs/src/server_config/server.rs | 76 +-
core/configs/src/server_config/validators.rs | 98 +-
core/connectors/sinks/redshift_sink/README.md | 2 +-
.../test_consumer_group_create_command.rs | 34 +-
.../tests/cluster/client_table_restart.rs | 478 ++++-
.../tests/cluster/crash_recovery_corruption.rs | 2 +-
.../tests/cluster/partition_primary_routing.rs | 29 +-
.../tests/connectors/fixtures/delta/fixture.rs | 158 +-
.../integration/tests/connectors/fixtures/floci.rs | 159 ++
.../tests/connectors/fixtures/iceberg/container.rs | 166 +-
core/integration/tests/connectors/fixtures/mod.rs | 1 +
.../connectors/fixtures/redshift/container.rs | 73 +-
.../tests/connectors/fixtures/redshift/mod.rs | 2 +-
.../tests/connectors/fixtures/redshift/sink.rs | 203 +-
.../tests/connectors/fixtures/s3/fixture.rs | 145 +-
core/integration/tests/connectors/s3/s3_sink.rs | 33 +-
.../stale_client_consumer_group_scenario.rs | 262 ++-
core/metadata/src/impls/metadata.rs | 427 +++-
core/metadata/src/impls/recovery.rs | 53 +-
core/metadata/src/stm/authz.rs | 8 +-
core/metadata/src/stm/consumer_group.rs | 242 ++-
core/metadata/src/stm/mod.rs | 24 +-
core/metadata/src/stm/stream.rs | 116 +-
core/metadata/src/stm/user.rs | 21 +-
core/partitions/src/iggy_index.rs | 6 +-
core/partitions/src/iggy_partition.rs | 1 -
core/partitions/src/lib.rs | 12 +
core/{server => partitions}/src/offset_recovery.rs | 22 +-
core/partitions/src/partition_storage.rs | 725 +++++++
.../{server => partitions}/src/segment_recovery.rs | 703 +++++--
core/partitions/src/types.rs | 48 +
core/server/config.toml | 19 +-
core/server/src/boot/mod.rs | 94 +-
core/server/src/boot/recovery.rs | 43 +-
core/server/src/boot/threads.rs | 12 +-
core/server/src/consumer_group.rs | 14 +-
core/server/src/consumer_group/lease.rs | 160 ++
core/server/src/consumer_group/liveness.rs | 1034 ++++++++++
core/server/src/dispatch/authz.rs | 65 +-
core/server/src/dispatch/host.rs | 213 ++
core/server/src/dispatch/mod.rs | 224 +--
core/server/src/dispatch/partition.rs | 12 +-
core/server/src/dispatch/session_ops.rs | 18 +-
core/server/src/dispatch/submit.rs | 242 +--
core/server/src/dispatch/test_support.rs | 9 +-
core/server/src/http/reads.rs | 125 +-
core/server/src/lib.rs | 14 +-
core/server/src/namespace.rs | 218 +-
core/server/src/partition_helpers.rs | 792 +-------
core/server/src/partition_reconciler.rs | 20 +-
core/server/src/server_error.rs | 320 +--
core/server/src/session_manager.rs | 72 +-
core/server/src/shell.rs | 53 +-
core/server_common/src/consensus_message.rs | 16 +-
core/server_common/src/sharding/mod.rs | 10 +
core/shard/src/builder.rs | 26 +-
core/shard/src/host.rs | 103 +
core/shard/src/lib.rs | 178 +-
core/shard/src/poll/completion_tests.rs | 12 +-
core/shard/src/router.rs | 6 +-
core/simulator/README.md | 1 +
core/simulator/src/client.rs | 18 +-
core/simulator/src/lib.rs | 274 ++-
core/simulator/src/packet.rs | 7 +-
core/simulator/src/replica.rs | 53 +-
core/simulator/src/storage/purge.rs | 17 +-
foreign/cpp/MODULE.bazel | 2 +-
foreign/cpp/MODULE.bazel.lock | 7 +-
foreign/go/client/tcp/tcp_offset_management.go | 9 +
foreign/go/tests/e2e_test.go | 49 +
foreign/swift/.swift-format | 22 +
.../consts.go => foreign/swift/Package.swift | 26 +-
foreign/swift/README.md | 87 +
foreign/swift/Sources/Iggy/Errors/IggyError.swift | 78 +
.../swift/Sources/Iggy/Errors/IggyErrorCode.swift | 515 +++++
.../swift/Sources/Iggy/Iggy.swift | 20 +-
.../Sources/Iggy/Utilities/UInt128Value.swift | 156 ++
foreign/swift/Sources/Iggy/Wire/ByteCodec.swift | 218 ++
foreign/swift/Tests/IggyTests/ByteCodecTests.swift | 120 ++
foreign/swift/Tests/IggyTests/ErrorCodeTests.swift | 55 +
gateways/kafka/Cargo.toml | 11 +-
gateways/kafka/README.md | 196 +-
gateways/kafka/docs/ACL_MAPPING.md | 195 ++
gateways/kafka/docs/AUTHENTICATION.md | 260 +++
gateways/kafka/docs/BRIDGE_MAPPING.md | 70 +-
gateways/kafka/docs/CONSUMER_GROUPS.md | 133 ++
gateways/kafka/docs/IDEMPOTENCE.md | 139 +-
gateways/kafka/docs/MANUAL_TESTING.md | 192 +-
gateways/kafka/docs/SCOPE.md | 223 ++-
gateways/kafka/docs/TEST_SUITE.md | 41 +-
gateways/kafka/docs/kafka_api_keys_reference.md | 45 +-
gateways/kafka/scripts/ci-wire-fixtures.sh | 2 +-
gateways/kafka/src/auth.rs | 805 ++++++++
gateways/kafka/src/bridge/config.rs | 53 +-
gateways/kafka/src/bridge/error.rs | 139 +-
gateways/kafka/src/bridge/iggy_bridge/mod.rs | 107 +-
gateways/kafka/src/bridge/iggy_bridge/offsets.rs | 37 +-
gateways/kafka/src/bridge/iggy_bridge/produce.rs | 183 +-
gateways/kafka/src/bridge/iggy_bridge/topics.rs | 330 +++-
gateways/kafka/src/bridge/mod.rs | 8 +-
gateways/kafka/src/bridge/topic_map.rs | 6 +
gateways/kafka/src/env.rs | 64 +
gateways/kafka/src/group/mod.rs | 493 +++++
gateways/kafka/src/group/state.rs | 2089 ++++++++++++++++++++
gateways/kafka/src/lib.rs | 3 +
gateways/kafka/src/main.rs | 198 +-
gateways/kafka/src/protocol/acl.rs | 600 ++++++
gateways/kafka/src/protocol/api.rs | 427 +++-
gateways/kafka/src/protocol/bounds_guard.rs | 600 +++++-
.../kafka/src/protocol/handlers/api_versions.rs | 34 +-
.../kafka/src/protocol/handlers/create_topics.rs | 711 ++++++-
.../src/protocol/handlers/find_coordinator.rs | 169 ++
gateways/kafka/src/protocol/handlers/heartbeat.rs | 78 +
.../src/protocol/handlers/init_producer_id.rs | 207 ++
gateways/kafka/src/protocol/handlers/join_group.rs | 158 ++
gateways/kafka/src/protocol/handlers/metadata.rs | 544 ++++-
gateways/kafka/src/protocol/handlers/mod.rs | 28 +-
gateways/kafka/src/protocol/handlers/produce.rs | 914 ++++++++-
gateways/kafka/src/protocol/handlers/sync_group.rs | 89 +
gateways/kafka/src/protocol/mod.rs | 2 +
gateways/kafka/src/protocol/sasl.rs | 573 ++++++
gateways/kafka/src/records.rs | 976 ++++++---
gateways/kafka/src/server.rs | 721 ++++++-
gateways/kafka/tests/api_handler_tests.rs | 36 +-
.../kafka/tests/bridge_iggy_integration_tests.rs | 272 ++-
gateways/kafka/tests/common/codec.rs | 11 +
gateways/kafka/tests/common/iggy_server.rs | 3 +-
gateways/kafka/tests/common/scope.rs | 11 +-
gateways/kafka/tests/common/server.rs | 28 +-
gateways/kafka/tests/common/wire.rs | 217 +-
gateways/kafka/tests/consumer_group_tests.rs | 1157 +++++++++++
.../kafka/tests/create_topics_real_bridge_tests.rs | 540 +++++
gateways/kafka/tests/golden_wire_fixtures_tests.rs | 59 +-
gateways/kafka/tests/idempotence_tests.rs | 439 ++++
gateways/kafka/tests/kafka_client_e2e_tests.rs | 725 +++++++
.../kafka/tests/list_offsets_real_bridge_tests.rs | 5 +
gateways/kafka/tests/listener_robustness_tests.rs | 60 +-
gateways/kafka/tests/metadata_real_bridge_tests.rs | 428 ++++
gateways/kafka/tests/produce_real_bridge_tests.rs | 754 +++++++
gateways/kafka/tests/response_negative_tests.rs | 9 +-
gateways/kafka/tests/sasl_tests.rs | 1709 ++++++++++++++++
gateways/kafka/tests/server_e2e_tests.rs | 30 +-
gateways/kafka/tests/version_firewall_tests.rs | 68 +-
gateways/kafka/tools/kafka-tool/src/main.rs | 32 +-
gateways/kafka/tools/kafka-tool/src/response.rs | 18 +-
licenserc.toml | 7 +
web/package-lock.json | 40 +-
web/package.json | 10 +-
176 files changed, 26767 insertions(+), 3839 deletions(-)
create mode 100644 .github/actions/swift/pre-merge/action.yml
create mode 100644 .github/actions/utils/setup-swift/action.yml
create mode 100644 core/binary_protocol/src/consensus/consumer_session.rs
create mode 100644 core/integration/tests/connectors/fixtures/floci.rs
rename core/{server => partitions}/src/offset_recovery.rs (92%)
create mode 100644 core/partitions/src/partition_storage.rs
rename core/{server => partitions}/src/segment_recovery.rs (89%)
create mode 100644 core/server/src/consumer_group/lease.rs
create mode 100644 core/server/src/consumer_group/liveness.rs
create mode 100644 core/server/src/dispatch/host.rs
create mode 100644 core/shard/src/host.rs
create mode 100644 foreign/swift/.swift-format
copy examples/go/common/consts.go => foreign/swift/Package.swift (62%)
create mode 100644 foreign/swift/README.md
create mode 100644 foreign/swift/Sources/Iggy/Errors/IggyError.swift
create mode 100644 foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift
copy bdd/rust/tests/leader_redirection.rs =>
foreign/swift/Sources/Iggy/Iggy.swift (67%)
create mode 100644 foreign/swift/Sources/Iggy/Utilities/UInt128Value.swift
create mode 100644 foreign/swift/Sources/Iggy/Wire/ByteCodec.swift
create mode 100644 foreign/swift/Tests/IggyTests/ByteCodecTests.swift
create mode 100644 foreign/swift/Tests/IggyTests/ErrorCodeTests.swift
create mode 100644 gateways/kafka/docs/ACL_MAPPING.md
create mode 100644 gateways/kafka/docs/AUTHENTICATION.md
create mode 100644 gateways/kafka/docs/CONSUMER_GROUPS.md
create mode 100644 gateways/kafka/src/auth.rs
create mode 100644 gateways/kafka/src/env.rs
create mode 100644 gateways/kafka/src/group/mod.rs
create mode 100644 gateways/kafka/src/group/state.rs
create mode 100644 gateways/kafka/src/protocol/acl.rs
create mode 100644 gateways/kafka/src/protocol/handlers/find_coordinator.rs
create mode 100644 gateways/kafka/src/protocol/handlers/heartbeat.rs
create mode 100644 gateways/kafka/src/protocol/handlers/init_producer_id.rs
create mode 100644 gateways/kafka/src/protocol/handlers/join_group.rs
create mode 100644 gateways/kafka/src/protocol/handlers/sync_group.rs
create mode 100644 gateways/kafka/src/protocol/sasl.rs
create mode 100644 gateways/kafka/tests/consumer_group_tests.rs
create mode 100644 gateways/kafka/tests/create_topics_real_bridge_tests.rs
create mode 100644 gateways/kafka/tests/idempotence_tests.rs
create mode 100644 gateways/kafka/tests/kafka_client_e2e_tests.rs
create mode 100644 gateways/kafka/tests/metadata_real_bridge_tests.rs
create mode 100644 gateways/kafka/tests/produce_real_bridge_tests.rs
create mode 100644 gateways/kafka/tests/sasl_tests.rs
