This is an automated email from the ASF dual-hosted git repository.
mmodzelewski pushed a change to branch java-leader-aware
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 131c51352 Merge branch 'master' into java-leader-aware
add 53a7ff99e feat(server-ng): harden listener, auth, shutdown, and
disk-poll paths (#3774)
add 2afd4b12b feat(consensus): persist metadata VSR state in a durable
superblock (#3767)
add 102484af2 feat(python): expose Permissions and remaining user auth
methods (#3727)
add 1453114da feat(metadata): state transfer for snapshot and client table
(#3765)
add 71c67e432 feat(server-ng): enforce PAT limits and expose Prometheus
metrics (#3773)
add a59a1db01 chore(deps): Bump the node group across 1 directory with 3
updates (#3779)
add 01e721028 chore(deps): Bump github.com/klauspost/compress from 1.19.0
to 1.19.1 in /bdd/go in the go group across 1 directory (#3785)
add 75a2fa286 feat(connectors): retry transient Doris Stream Load failures
in-request (#3574)
add ddead57af fix(ci): make README CLI preflight execute its commands
(#3792)
add e2bbaea1c ci(rust): skip unnecessary setup steps (#3749)
add a684e550f feat(python): add user header and origin timestamp support
(#3613)
add 3ca8695d3 feat(connectors): add Meilisearch sink connector (#3497)
add 54d8a49dd feat(server-ng): close the partition materialisation race at
the owner (#3786)
add 198364f22 chore(deps): Bump the web group across 1 directory with 16
updates (#3807)
add 4199fed86 chore(deps): Bump fast-uri from 3.1.4 to 3.1.5 in
/foreign/node (#3812)
add e4e25648b chore(deps): Bump the csharp group with 1 update (#3810)
add b452d150d fix(node): TLS + VSR connection support (#3813)
add c51b75456 perf(server-ng): cap sealed-segment read handles with
per-partition LRU (#3806)
add 60b878923 perf(server-ng): compute batch checksum in a single produce
pass (#3746)
add 962f064ba Merge branch 'master' into java-leader-aware
No new revisions were added by this update.
Summary of changes:
.github/actions/rust/pre-merge/action.yml | 6 +
.../actions/utils/setup-rust-with-cache/action.yml | 14 +-
.github/workflows/coverage-baseline.yml | 1 +
Cargo.lock | 96 +
Cargo.toml | 6 +
bdd/go/go.mod | 2 +-
bdd/go/go.sum | 4 +-
core/binary_protocol/src/consensus/command.rs | 19 +-
core/binary_protocol/src/consensus/header.rs | 337 ++-
core/binary_protocol/src/consensus/mod.rs | 3 +-
core/binary_protocol/src/lib.rs | 5 +-
core/configs/src/server_ng_config/cluster.rs | 69 +
core/configs/src/server_ng_config/sharding.rs | 74 +
core/configs/src/server_ng_config/validators.rs | 13 +
core/connectors/README.md | 1 +
.../example_config/connectors/doris_sink.toml | 9 +
core/connectors/sinks/README.md | 1 +
core/connectors/sinks/doris_sink/README.md | 31 +-
core/connectors/sinks/doris_sink/config.toml | 9 +
core/connectors/sinks/doris_sink/src/lib.rs | 901 +++++--
.../Cargo.toml | 14 +-
core/connectors/sinks/meilisearch_sink/README.md | 105 +
.../config.toml | 18 +-
core/connectors/sinks/meilisearch_sink/src/lib.rs | 1412 +++++++++++
core/connectors/sinks/s3_sink/Cargo.toml | 6 +
core/consensus/Cargo.toml | 2 +
core/consensus/src/client_table.rs | 979 +++++++-
core/consensus/src/impls.rs | 840 ++++++-
core/consensus/src/le_cursor.rs | 115 +
core/consensus/src/lib.rs | 15 +-
core/consensus/src/metadata_helpers.rs | 27 +-
core/consensus/src/observability.rs | 4 +-
core/consensus/src/plane_helpers.rs | 15 +-
core/consensus/src/state_manifest.rs | 309 +++
core/consensus/src/vsr_state.rs | 224 ++
core/integration/Cargo.toml | 13 +-
core/integration/src/harness/config/resolve.rs | 7 +-
core/integration/src/harness/handle/server.rs | 83 +-
.../src/harness/orchestrator/harness.rs | 26 +
.../cli/message/test_message_flush_command.rs | 5 +
.../tests/cluster/client_table_restart.rs | 14 +-
.../tests/cluster/metadata_checkpoint_restart.rs | 276 +++
.../tests/cluster/metadata_state_transfer.rs | 406 ++++
core/integration/tests/cluster/mod.rs | 3 +
.../cluster/multi_shard_partition_convergence.rs | 223 ++
.../tests/connectors/doris/doris_sink.rs | 9 +-
.../connectors/fixtures/meilisearch/container.rs | 180 ++
.../fixtures/{quickwit => meilisearch}/mod.rs | 4 +-
.../tests/connectors/fixtures/meilisearch/sink.rs | 90 +
core/integration/tests/connectors/fixtures/mod.rs | 2 +
.../connectors/meilisearch/meilisearch_sink.rs | 81 +
.../tests/connectors/{delta => meilisearch}/mod.rs | 2 +-
.../connectors/{delta => meilisearch}/sink.toml | 2 +-
core/integration/tests/connectors/mod.rs | 1 +
core/integration/tests/data_integrity/mod.rs | 11 +-
.../data_integrity/verify_after_server_restart.rs | 31 +
.../tests/server/cluster_view_durability_vsr.rs | 266 +++
core/integration/tests/server/flush_vsr.rs | 4 +
.../server/metadata_checkpoint_recovery_vsr.rs | 138 ++
core/integration/tests/server/mod.rs | 8 +
.../scenarios/segment_rotation_race_scenario.rs | 9 -
core/journal/Cargo.toml | 2 +
core/journal/src/lib.rs | 31 +
core/journal/src/prepare_journal.rs | 508 +++-
core/journal/src/superblock.rs | 824 +++++++
core/message_bus/src/client_listener/mod.rs | 5 +-
core/message_bus/src/client_listener/quic.rs | 24 +-
core/message_bus/src/framing.rs | 8 +-
core/message_bus/src/installer/replica.rs | 2 +-
core/message_bus/src/replica/auth.rs | 106 +-
core/message_bus/src/replica/handshake.rs | 21 +-
core/message_bus/src/socket_opts.rs | 59 +-
core/message_bus/tests/common/mod.rs | 7 +-
core/message_bus/tests/replica_auth_handshake.rs | 28 +
core/metadata/src/impls/metadata.rs | 1389 +++++++++--
core/metadata/src/impls/recovery.rs | 985 +++++++-
core/metadata/src/lib.rs | 5 +-
core/metadata/src/stm/mod.rs | 10 +
core/metadata/src/stm/mux.rs | 40 +-
core/metadata/src/stm/snapshot.rs | 129 +-
core/metadata/src/stm/stream.rs | 268 ++-
core/metadata/src/stm/user.rs | 69 +-
core/partitions/src/iggy_index_reader.rs | 180 +-
core/partitions/src/iggy_partition.rs | 662 +++++-
core/partitions/src/iggy_partitions.rs | 25 +-
core/partitions/src/journal.rs | 23 +-
core/partitions/src/log.rs | 365 ++-
core/partitions/src/poll_plan.rs | 269 ++-
core/sdk/src/prelude.rs | 6 +-
core/server-ng/Cargo.toml | 8 +-
core/server-ng/config.toml | 18 +-
core/server-ng/src/auth.rs | 100 +-
core/server-ng/src/bootstrap.rs | 543 +++--
core/server-ng/src/dispatch.rs | 187 +-
core/server-ng/src/http.rs | 160 +-
core/server-ng/src/http/metrics.rs | 297 +++
core/server-ng/src/http/state.rs | 8 +
core/server-ng/src/http/submit.rs | 37 +-
core/server-ng/src/partition_reconciler.rs | 1533 +++++++++++-
core/server-ng/src/pat.rs | 182 +-
core/server-ng/src/responses.rs | 19 +-
core/server-ng/src/server_error.rs | 16 +-
core/server_common/src/consensus_message.rs | 34 +-
core/server_common/src/send_messages2.rs | 987 +++++++-
core/shard/src/lib.rs | 2452 ++++++++++++++++++--
core/shard/src/metrics.rs | 139 +-
core/shard/src/router.rs | 72 +-
core/simulator/src/client.rs | 4 +-
core/simulator/src/deps.rs | 131 ++
core/simulator/src/lib.rs | 534 ++++-
core/simulator/src/replica.rs | 126 +-
examples/go/go.mod | 2 +-
examples/go/go.sum | 4 +-
examples/java/README.md | 2 +-
examples/node/README.md | 2 +-
examples/python/README.md | 23 +
examples/python/basic/producer.py | 50 +-
examples/python/getting-started/producer.py | 50 +-
examples/python/message-headers/common.py | 240 ++
.../message-headers/plain-headers/consumer.py | 77 +
.../message-headers/plain-headers/producer.py | 59 +
.../message-headers/typed-headers/consumer.py | 77 +
.../message-headers/typed-headers/producer.py | 64 +
examples/python/pyproject.toml | 5 +
examples/rust/README.md | 2 +-
examples/rust/src/tcp-tls/consumer/main.rs | 2 +-
examples/rust/src/tcp-tls/producer/main.rs | 2 +-
foreign/csharp/Directory.Packages.props | 2 +-
foreign/go/go.mod | 2 +-
foreign/go/go.sum | 4 +-
foreign/node/README.md | 6 +-
foreign/node/package-lock.json | 1018 +++-----
foreign/node/package.json | 6 +-
foreign/node/src/client/client.config.test.ts | 19 +-
foreign/node/src/client/client.config.ts | 3 -
foreign/node/src/client/client.socket.test.ts | 50 +-
foreign/node/src/e2e/tls.system.e2e.ts | 4 +
foreign/python/Cargo.toml | 1 +
foreign/python/Dockerfile.test | 2 +-
foreign/python/apache_iggy.pyi | 731 +++++-
foreign/python/docker-compose.test.yml | 11 +-
foreign/python/src/client.rs | 103 +-
foreign/python/src/lib.rs | 11 +
foreign/python/src/permissions.rs | 458 ++++
foreign/python/src/receive_message.rs | 26 +
foreign/python/src/send_message.rs | 41 +-
foreign/python/src/user.rs | 9 +
foreign/python/src/user_headers.rs | 791 +++++++
foreign/python/tests/test_consumer_group.py | 98 +
foreign/python/tests/test_message_operations.py | 402 +++-
foreign/python/tests/test_permissions.py | 695 ++++++
foreign/python/tests/test_user.py | 113 +-
foreign/python/tests/utils.py | 23 +
scripts/ci/python-sdk-version-sync.sh | 2 +-
scripts/ci/sync-python-interpreter-version.sh | 2 +-
scripts/ci/sync-rustc-version.sh | 2 +-
scripts/ci/third-party-licenses.sh | 2 +-
scripts/ci/uv-lock-check.sh | 2 +-
scripts/run-examples-from-readme.sh | 31 +-
scripts/utils.sh | 8 +-
web/package-lock.json | 613 ++---
web/package.json | 30 +-
162 files changed, 24629 insertions(+), 2570 deletions(-)
copy core/connectors/sinks/{surrealdb_sink => meilisearch_sink}/Cargo.toml
(84%)
create mode 100644 core/connectors/sinks/meilisearch_sink/README.md
copy core/connectors/sinks/{elasticsearch_sink =>
meilisearch_sink}/config.toml (77%)
create mode 100644 core/connectors/sinks/meilisearch_sink/src/lib.rs
create mode 100644 core/consensus/src/le_cursor.rs
create mode 100644 core/consensus/src/state_manifest.rs
create mode 100644 core/consensus/src/vsr_state.rs
create mode 100644
core/integration/tests/cluster/metadata_checkpoint_restart.rs
create mode 100644 core/integration/tests/cluster/metadata_state_transfer.rs
create mode 100644
core/integration/tests/cluster/multi_shard_partition_convergence.rs
create mode 100644
core/integration/tests/connectors/fixtures/meilisearch/container.rs
copy core/integration/tests/connectors/fixtures/{quickwit =>
meilisearch}/mod.rs (89%)
create mode 100644
core/integration/tests/connectors/fixtures/meilisearch/sink.rs
create mode 100644
core/integration/tests/connectors/meilisearch/meilisearch_sink.rs
copy core/integration/tests/connectors/{delta => meilisearch}/mod.rs (97%)
copy core/integration/tests/connectors/{delta => meilisearch}/sink.toml (94%)
create mode 100644 core/integration/tests/server/cluster_view_durability_vsr.rs
create mode 100644
core/integration/tests/server/metadata_checkpoint_recovery_vsr.rs
create mode 100644 core/journal/src/superblock.rs
create mode 100644 core/server-ng/src/http/metrics.rs
create mode 100644 examples/python/message-headers/common.py
create mode 100644 examples/python/message-headers/plain-headers/consumer.py
create mode 100644 examples/python/message-headers/plain-headers/producer.py
create mode 100644 examples/python/message-headers/typed-headers/consumer.py
create mode 100644 examples/python/message-headers/typed-headers/producer.py
create mode 100644 foreign/python/src/permissions.rs
create mode 100644 foreign/python/src/user_headers.rs
create mode 100644 foreign/python/tests/test_permissions.py
