This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a change to branch global-metadata-leftright
in repository https://gitbox.apache.org/repos/asf/iggy.git
omit f7b7bc68c refactor(server): WIP, don't merge: replace slab ECS with
LeftRight-based global metadata
add ab7902eb3 perf(ci): reduce dev profile opt-level from 2 to 1 (#2573)
add a53a4ed3d fix(server): maintain message availability during async disk
I/O (#2575)
add ac6b30bb8 fix(integration): add watchdog to detect test server crashes
(#2576)
add f5f9bb51c fix(server): prevent message unavailability during
background persistence (#2578)
add 5b864bedc refactor(server): consolidate permissions into metadata
crate and fix inheritance bugs (#2564)
add 9319220e4 feat(connectors): extend Postgres sink & source connectors,
add integration tests (#2579)
add b13456c1e fix(server): chunk vectored writes to avoid exceeding
IOV_MAX limit (#2581)
add 75e30ab42 fix(server): prevent panic when segment rotates during async
persistence (#2588)
add 61eae146f refactor(server): WIP, don't merge: replace slab ECS with
LeftRight-based global metadata
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f7b7bc68c)
\
N -- N -- N refs/heads/global-metadata-leftright (61eae146f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
Cargo.lock | 152 +-
Cargo.toml | 37 +-
DEPENDENCIES.md | 24 +-
core/ai/mcp/Cargo.toml | 2 +-
core/bench/dashboard/server/Cargo.toml | 4 +-
core/cli/Cargo.toml | 2 +-
core/common/src/alloc/buffer.rs | 23 +-
core/common/src/configs/mod.rs | 113 +-
core/common/src/types/message/messages_batch.rs | 10 +
core/common/src/types/message/polled_messages.rs | 8 +-
core/connectors/runtime/src/configs/connectors.rs | 10 +
.../src/configs/connectors/local_provider.rs | 75 +-
core/connectors/runtime/src/main.rs | 2 +
core/connectors/runtime/src/sink.rs | 36 +-
core/connectors/runtime/src/source.rs | 26 +-
core/connectors/sdk/Cargo.toml | 4 +-
core/connectors/sdk/src/lib.rs | 2 +
core/connectors/sinks/README.md | 2 +
core/connectors/sinks/postgres_sink/Cargo.toml | 2 +
core/connectors/sinks/postgres_sink/README.md | 298 ++-
.../sinks/postgres_sink/config.toml} | 12 +-
core/connectors/sinks/postgres_sink/src/lib.rs | 741 +++++-
core/connectors/sources/README.md | 2 +
core/connectors/sources/postgres_source/Cargo.toml | 1 +
core/connectors/sources/postgres_source/README.md | 399 +++-
.../sources/postgres_source/config.toml} | 27 +-
core/connectors/sources/postgres_source/src/lib.rs | 1299 ++++++----
core/integration/Cargo.toml | 8 +-
core/integration/src/test_server.rs | 34 +-
core/integration/tests/connectors/mod.rs | 158 +-
core/integration/tests/connectors/postgres/mod.rs | 511 +++-
.../tests/connectors/postgres/postgres_sink.rs | 141 +-
.../tests/connectors/postgres/postgres_source.rs | 209 ++
.../connectors/postgres/{config.toml => sink.toml} | 2 +-
.../{random/config.toml => postgres/source.toml} | 2 +-
.../tests/connectors/random/random_source.rs | 5 +-
core/integration/tests/mcp/mod.rs | 1 +
core/integration/tests/server/general.rs | 3 +-
core/integration/tests/server/mod.rs | 7 +-
core/integration/tests/server/scenarios/mod.rs | 4 +
.../tests/server/scenarios/permissions_scenario.rs | 2511 ++++++++++++++++++++
.../scenarios/read_during_persistence_scenario.rs | 198 ++
.../scenarios/segment_rotation_race_scenario.rs | 196 ++
.../scenarios/single_message_per_batch_scenario.rs | 178 ++
core/integration/tests/server/specific.rs | 104 +-
core/integration/tests/streaming/mod.rs | 6 +-
core/metadata/src/lib.rs | 2 +-
core/metadata/src/permissioner/mod.rs | 8 +-
.../permissioner_rules/consumer_groups.rs | 4 -
.../permissioner_rules/consumer_offsets.rs | 4 -
.../permissioner/permissioner_rules/messages.rs | 84 +-
.../permissioner/permissioner_rules/partitions.rs | 4 -
.../permissioner/permissioner_rules/segments.rs | 4 -
.../src/permissioner/permissioner_rules/streams.rs | 4 -
.../src/permissioner/permissioner_rules/system.rs | 4 -
.../src/permissioner/permissioner_rules/topics.rs | 72 +-
.../src/permissioner/permissioner_rules/users.rs | 4 -
core/metadata/src/stm/user.rs | 6 +-
core/server/Cargo.toml | 9 +-
.../cluster/get_cluster_metadata_handler.rs | 6 +-
.../handlers/messages/poll_messages_handler.rs | 6 +-
.../binary/handlers/system/get_snapshot_handler.rs | 2 +-
core/server/src/http/messages.rs | 6 +-
core/server/src/http/system.rs | 26 +-
core/server/src/shard/builder.rs | 5 +-
core/server/src/shard/mod.rs | 3 +-
core/server/src/shard/system/cluster.rs | 11 +-
core/server/src/shard/system/messages.rs | 6 +-
core/server/src/shard/system/snapshot/mod.rs | 2 -
core/server/src/streaming/partitions/in_flight.rs | 100 +
core/server/src/streaming/partitions/log.rs | 23 +-
core/server/src/streaming/partitions/mod.rs | 1 +
.../src/streaming/segments/indexes/indexes_mut.rs | 13 +-
.../streaming/segments/messages/messages_writer.rs | 32 +-
core/server/src/streaming/segments/messages/mod.rs | 59 +-
core/server/src/streaming/segments/mod.rs | 2 +
.../streaming/segments/types/messages_batch_mut.rs | 16 +-
.../streaming/segments/types/messages_batch_set.rs | 37 +-
core/server/src/streaming/users/mod.rs | 2 -
core/server/src/streaming/users/permissioner.rs | 73 -
.../users/permissioner_rules/consumer_groups.rs | 76 -
.../users/permissioner_rules/consumer_offsets.rs | 49 -
.../streaming/users/permissioner_rules/messages.rs | 99 -
.../src/streaming/users/permissioner_rules/mod.rs | 27 -
.../users/permissioner_rules/partitions.rs | 40 -
.../streaming/users/permissioner_rules/segments.rs | 30 -
.../streaming/users/permissioner_rules/streams.rs | 98 -
.../streaming/users/permissioner_rules/system.rs | 46 -
.../streaming/users/permissioner_rules/topics.rs | 165 --
.../streaming/users/permissioner_rules/users.rs | 74 -
examples/rust/src/multi-tenant/consumer/main.rs | 18 +-
examples/rust/src/multi-tenant/producer/main.rs | 19 +-
92 files changed, 6894 insertions(+), 2078 deletions(-)
copy
core/{integration/tests/connectors/postgres/connectors_config/postgres.toml =>
connectors/sinks/postgres_sink/config.toml} (81%)
rename
core/{integration/tests/connectors/postgres/connectors_config/postgres.toml =>
connectors/sources/postgres_source/config.toml} (70%)
create mode 100644
core/integration/tests/connectors/postgres/postgres_source.rs
rename core/integration/tests/connectors/postgres/{config.toml => sink.toml}
(93%)
copy core/integration/tests/connectors/{random/config.toml =>
postgres/source.toml} (93%)
create mode 100644
core/integration/tests/server/scenarios/permissions_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/read_during_persistence_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs
create mode 100644
core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs
create mode 100644 core/server/src/streaming/partitions/in_flight.rs
delete mode 100644 core/server/src/streaming/users/permissioner.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/consumer_groups.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/consumer_offsets.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/messages.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/mod.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/partitions.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/segments.rs
delete mode 100644
core/server/src/streaming/users/permissioner_rules/streams.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/system.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/topics.rs
delete mode 100644 core/server/src/streaming/users/permissioner_rules/users.rs
