This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a change to branch state_transfer_metadata_clients_table
in repository https://gitbox.apache.org/repos/asf/iggy.git
from ea6c607e6 peer feetching during stale view restart
add b6a2f81c0 fix(sdk): forward unknown command codes instead of rejecting
them (#3766)
add ba541bb9c fix(server-ng): stop restart panic when WAL replay recovers
sessions (#3768)
add 91fcb9300 fix(ci): propagate failures in pre-commit hooks (#3743)
add 265bd265a chore(deps): Bump com.gradleup.shadow from 9.5.1 to 9.6.0 in
/foreign/java in the java group across 1 directory (#3771)
add 3153e456f chore(deps): Bump the github-actions group with 5 updates
(#3772)
add 3711d060d fix(ci): example-runner silently skips Python/Go examples
when exclude starts with a dash (#3770)
add f0eb0950d fix(server-ng): make partition reconvergence safe for
in-flight sends (#3761)
add f67f03873 fix(server-ng): fold io_uring remediation into
runtime-creation errors (#3775)
add 0fa657a94 fix(ci): stop trailing checks from skipping text files as
binary (#3778)
add 3b190c14e feat(js): add VSR framing to the Node SDK (#3763)
add 5ca0bbb40 chore(deps): Bump postcss from 8.5.16 to 8.5.23 in /web
(#3762)
add 74fb5e2a9 fix(server-ng): bind cluster TCP to [tcp].address, not the
roster ip (#3783)
add 53a7ff99e feat(server-ng): harden listener, auth, shutdown, and
disk-poll paths (#3774)
add 2afd4b12b feat(consensus): persist metadata VSR state in a durable
superblock (#3767)
add 25fe5711c merge master
No new revisions were added by this update.
Summary of changes:
.github/actions/node-npm/pre-merge/action.yml | 76 +-
.github/actions/utils/server-start/action.yml | 15 +-
.github/config/components.yml | 2 +-
.github/workflows/_build_python_wheels.yml | 14 +-
.github/workflows/_build_rust_artifacts.yml | 6 +-
.github/workflows/_common.yml | 32 +-
.github/workflows/_detect.yml | 2 +-
.github/workflows/_publish_rust_crates.yml | 2 +-
.github/workflows/_test.yml | 17 +-
.github/workflows/_test_bdd.yml | 2 +-
.github/workflows/_test_examples.yml | 6 +-
.github/workflows/coverage-baseline.yml | 26 +-
.github/workflows/edge-release.yml | 2 +-
.github/workflows/post-merge.yml | 4 +-
.github/workflows/publish.yml | 16 +-
.pre-commit-config.yaml | 8 +-
Cargo.lock | 11 +-
Cargo.toml | 11 +-
codecov.yml | 6 +
core/binary_protocol/Cargo.toml | 2 +-
core/binary_protocol/src/consensus/header.rs | 41 +-
core/common/Cargo.toml | 2 +-
core/common/src/error/iggy_error.rs | 2 -
core/configs/src/server_ng_config/cluster.rs | 65 +-
core/configs/src/server_ng_config/sharding.rs | 74 ++
core/consensus/Cargo.toml | 1 +
core/consensus/src/client_table.rs | 517 ++++++++++-
core/consensus/src/impls.rs | 445 +++++++++-
core/consensus/src/lib.rs | 7 +-
core/consensus/src/plane_helpers.rs | 42 +
core/consensus/src/vsr_state.rs | 224 +++++
core/integration/Cargo.toml | 8 +-
.../tests/cluster/metadata_state_transfer.rs | 154 ++++
.../integration/tests/connectors/http/http_sink.rs | 72 +-
core/integration/tests/sdk/raw.rs | 13 +-
.../tests/server/cluster_view_durability_vsr.rs | 266 ++++++
core/integration/tests/server/http_vsr.rs | 11 +
core/integration/tests/server/message_retrieval.rs | 51 +-
.../server/metadata_checkpoint_recovery_vsr.rs | 138 +++
core/integration/tests/server/mod.rs | 8 +
.../server/scenarios/message_cleanup_scenario.rs | 8 +-
.../server/scenarios/purge_delete_scenario.rs | 78 +-
core/journal/Cargo.toml | 2 +
core/journal/src/lib.rs | 15 +
core/journal/src/prepare_journal.rs | 495 ++++++++++-
core/journal/src/superblock.rs | 824 +++++++++++++++++
core/message_bus/src/client_listener/mod.rs | 5 +-
core/message_bus/src/client_listener/quic.rs | 24 +-
core/message_bus/src/framing.rs | 8 +-
core/message_bus/src/installer/replica.rs | 2 +-
core/message_bus/src/replica/auth.rs | 106 ++-
core/message_bus/src/replica/handshake.rs | 21 +-
core/message_bus/src/socket_opts.rs | 59 +-
core/message_bus/tests/common/mod.rs | 7 +-
core/message_bus/tests/replica_auth_handshake.rs | 28 +
core/metadata/src/impls/metadata.rs | 920 ++++++++++++++++---
core/metadata/src/impls/recovery.rs | 981 +++++++++++++++++++--
core/metadata/src/lib.rs | 1 +
core/metadata/src/stm/snapshot.rs | 64 +-
core/metadata/src/stm/stream.rs | 293 +++++-
core/metadata/src/stm/user.rs | 5 +
core/partitions/src/iggy_partition.rs | 88 +-
core/partitions/src/poll_plan.rs | 57 +-
core/sdk/Cargo.toml | 2 +-
core/sdk/src/clients/client.rs | 5 +-
core/sdk/src/vsr.rs | 69 +-
core/server-ng/config.toml | 26 +-
core/server-ng/src/auth.rs | 100 ++-
core/server-ng/src/bootstrap.rs | 699 ++++++++++-----
core/server-ng/src/dispatch.rs | 346 +++++++-
core/server-ng/src/dispatch/authz.rs | 10 +-
core/server-ng/src/http.rs | 30 +-
core/server-ng/src/main.rs | 13 +-
core/server-ng/src/partition_reconciler.rs | 194 +++-
core/server-ng/src/server_error.rs | 22 +-
core/server/src/main.rs | 40 +-
core/server_common/src/diagnostics.rs | 108 ++-
core/shard/src/lib.rs | 545 ++++++++++--
core/shard/src/router.rs | 5 +
core/simulator/src/deps.rs | 121 +++
core/simulator/src/lib.rs | 555 +++++++++++-
core/simulator/src/replica.rs | 122 ++-
.../data-population-explanation.md | 3 +-
foreign/java/gradle/libs.versions.toml | 2 +-
foreign/node/README.md | 63 +-
foreign/node/package-lock.json | 4 +-
foreign/node/package.json | 8 +-
foreign/node/scripts/check-vsr-protocol.mjs | 340 +++++++
foreign/node/src/bdd/raw.ts | 5 +-
foreign/node/src/client/client.config.test.ts | 95 ++
foreign/node/src/client/client.config.ts | 56 ++
foreign/node/src/client/client.connection.test.ts | 455 ++++++++++
foreign/node/src/client/client.connection.ts | 362 ++++++--
foreign/node/src/client/client.frame.test.ts | 160 ++++
foreign/node/src/client/client.frame.ts | 227 +++++
foreign/node/src/client/client.socket.test.ts | 677 ++++++++++++++
foreign/node/src/client/client.socket.ts | 410 ++++++++-
foreign/node/src/client/client.ts | 73 +-
foreign/node/src/client/client.type.ts | 24 +-
foreign/node/src/client/index.ts | 1 +
foreign/node/src/e2e/tcp.consumer-group.e2e.ts | 8 +-
foreign/node/src/e2e/tcp.consumer-stream.e2e.ts | 8 +-
foreign/node/src/e2e/tcp.raw.e2e.ts | 38 +-
foreign/node/src/e2e/test-client.utils.ts | 2 +
foreign/node/src/index.ts | 6 +
foreign/node/src/stream/consumer-stream.ts | 9 +-
.../wire/cluster/get-cluster-metadata.command.ts | 2 +-
foreign/node/src/wire/command-set.test.ts | 10 +-
foreign/node/src/wire/command-set.ts | 10 +-
foreign/node/src/wire/command.code.ts | 5 +-
.../src/wire/command.utils.test.ts} | 39 +-
foreign/node/src/wire/command.utils.ts | 8 +-
.../wire/consumer-group/delete-group.command.ts | 19 +-
.../src/wire/consumer-group/get-group.command.ts | 16 +-
.../node/src/wire/consumer-group/group.utils.ts | 7 +
foreign/node/src/wire/consumer-group/index.ts | 1 +
.../src/wire/consumer-group/join-group.command.ts | 19 +-
.../src/wire/consumer-group/leave-group.command.ts | 19 +-
.../wire/consumer-group/sync-group.command.test.ts | 99 +++
.../src/wire/consumer-group/sync-group.command.ts | 67 ++
foreign/node/src/wire/error.code.test.ts | 40 +
foreign/node/src/wire/error.code.ts | 32 +-
foreign/node/src/wire/error.utils.ts | 22 +-
.../src/wire/message/poll-messages.command.test.ts | 423 +++++++++
.../node/src/wire/message/poll-messages.command.ts | 211 ++++-
foreign/node/src/wire/offset/get-offset.command.ts | 2 +-
foreign/node/src/wire/offset/offset.utils.ts | 4 +-
.../node/src/wire/offset/store-offset.command.ts | 2 +-
.../topic.utils.test.ts} | 60 +-
foreign/node/src/wire/topic/topic.utils.ts | 4 +-
foreign/node/src/wire/vsr/header.test.ts | 99 +++
foreign/node/src/wire/vsr/header.ts | 165 ++++
foreign/node/src/wire/vsr/index.ts | 157 ++++
foreign/node/src/wire/vsr/namespace.test.ts | 342 +++++++
foreign/node/src/wire/vsr/namespace.ts | 216 +++++
foreign/node/src/wire/vsr/operation.test.ts | 107 +++
foreign/node/src/wire/vsr/operation.ts | 158 ++++
foreign/node/src/wire/vsr/register.test.ts | 93 ++
foreign/node/src/wire/vsr/register.ts | 116 +++
foreign/node/src/wire/vsr/reply.test.ts | 196 ++++
foreign/node/src/wire/vsr/reply.ts | 172 ++++
foreign/node/src/wire/vsr/session.test.ts | 73 ++
foreign/node/src/wire/vsr/session.ts | 117 +++
foreign/node/src/wire/vsr/vsr.test.ts | 131 +++
foreign/node/tsconfig.json | 2 +-
foreign/python/Cargo.toml | 2 +-
scripts/ci/trailing-newline.sh | 2 +-
scripts/ci/trailing-whitespace.sh | 2 +-
scripts/utils.sh | 2 +-
web/package-lock.json | 16 +-
web/package.json | 2 +-
151 files changed, 14470 insertions(+), 1361 deletions(-)
create mode 100644 core/consensus/src/vsr_state.rs
create mode 100644 core/integration/tests/server/cluster_view_durability_vsr.rs
create mode 100644
core/integration/tests/server/metadata_checkpoint_recovery_vsr.rs
create mode 100644 core/journal/src/superblock.rs
create mode 100644 foreign/node/scripts/check-vsr-protocol.mjs
create mode 100644 foreign/node/src/client/client.config.test.ts
create mode 100644 foreign/node/src/client/client.config.ts
create mode 100644 foreign/node/src/client/client.connection.test.ts
create mode 100644 foreign/node/src/client/client.frame.test.ts
create mode 100644 foreign/node/src/client/client.frame.ts
create mode 100644 foreign/node/src/client/client.socket.test.ts
copy foreign/{go/contracts/duration.go => node/src/wire/command.utils.test.ts}
(60%)
create mode 100644
foreign/node/src/wire/consumer-group/sync-group.command.test.ts
create mode 100644 foreign/node/src/wire/consumer-group/sync-group.command.ts
create mode 100644 foreign/node/src/wire/error.code.test.ts
create mode 100644 foreign/node/src/wire/message/poll-messages.command.test.ts
copy foreign/node/src/wire/{token/delete-token.command.test.ts =>
topic/topic.utils.test.ts} (51%)
create mode 100644 foreign/node/src/wire/vsr/header.test.ts
create mode 100644 foreign/node/src/wire/vsr/header.ts
create mode 100644 foreign/node/src/wire/vsr/index.ts
create mode 100644 foreign/node/src/wire/vsr/namespace.test.ts
create mode 100644 foreign/node/src/wire/vsr/namespace.ts
create mode 100644 foreign/node/src/wire/vsr/operation.test.ts
create mode 100644 foreign/node/src/wire/vsr/operation.ts
create mode 100644 foreign/node/src/wire/vsr/register.test.ts
create mode 100644 foreign/node/src/wire/vsr/register.ts
create mode 100644 foreign/node/src/wire/vsr/reply.test.ts
create mode 100644 foreign/node/src/wire/vsr/reply.ts
create mode 100644 foreign/node/src/wire/vsr/session.test.ts
create mode 100644 foreign/node/src/wire/vsr/session.ts
create mode 100644 foreign/node/src/wire/vsr/vsr.test.ts
