This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a change to branch zero-copy-no-batching
in repository https://gitbox.apache.org/repos/asf/iggy.git
from be058175 insane improvements
add 61b0b00b rename headers to user_headers
new 745899db feat(bench): improve benchmark by parallelizing actors (#1645)
new 1e78ba24 added indexes to IggyMessageBatch
new d510935a add memmap2 to deps
new 00f7566a fix more todos
new ce3cf664 another day, another refactor
new 5a7cde2e fixes in sdk
new 5a248f09 implement http (sadly with copy)
new 24b59da1 quic finished
new fecd2ed3 quic and http fixes
new 9f9c1209 new errors
new 78b85ba2 validation
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Cargo.lock | 17 +-
Cargo.toml | 2 +-
bench/Cargo.toml | 3 +-
bench/src/actors/consumer.rs | 94 ++--
bench/src/actors/producer.rs | 2 +-
bench/src/actors/producing_consumer.rs | 14 +-
bench/src/actors/utils.rs | 5 +-
bench/src/benchmarks/benchmark.rs | 15 +-
bench/src/benchmarks/consumer_benchmark.rs | 18 +-
bench/src/benchmarks/consumer_group_benchmark.rs | 19 +-
.../benchmarks/producer_and_consumer_benchmark.rs | 21 +-
.../producer_and_consumer_group_benchmark.rs | 23 +-
bench/src/benchmarks/producer_benchmark.rs | 17 +-
.../src/benchmarks/producing_consumer_benchmark.rs | 18 +-
.../producing_consumer_group_benchmark.rs | 16 +-
bench/src/runner.rs | 12 +-
cli/src/args/message.rs | 2 +-
configs/server.toml | 9 +-
integration/tests/state/mod.rs | 9 +-
integration/tests/streaming/get_by_offset.rs | 296 ++++++++----
integration/tests/streaming/get_by_timestamp.rs | 343 +++++++++----
integration/tests/streaming/mod.rs | 8 +-
integration/tests/streaming/partition.rs | 4 +-
integration/tests/streaming/segment.rs | 19 +-
sdk/src/binary/messages.rs | 11 +-
sdk/src/cli/message/poll_messages.rs | 74 ++-
sdk/src/cli/message/send_messages.rs | 13 +-
sdk/src/client.rs | 2 +-
sdk/src/clients/client.rs | 8 +-
sdk/src/clients/consumer.rs | 237 +++++----
sdk/src/clients/producer.rs | 2 +-
sdk/src/consumer_ext/consumer_message_ext.rs | 3 +-
sdk/src/consumer_ext/mod.rs | 7 +-
sdk/src/error.rs | 18 +-
sdk/src/http/messages.rs | 17 +-
sdk/src/messages/mod.rs | 2 +
sdk/src/messages/partitioning_kind.rs | 8 +-
sdk/src/messages/poll_messages.rs | 2 -
sdk/src/messages/polled_messages.rs | 112 +++++
sdk/src/messages/polling_strategy.rs | 6 -
sdk/src/messages/send_messages.rs | 273 ++---------
.../indexes => sdk/src/models/messaging}/index.rs | 0
.../src/models/messaging}/index_view.rs | 9 +-
sdk/src/models/messaging/indexes.rs | 180 +++++++
sdk/src/models/messaging/message.rs | 538 ++++++++++++++++-----
sdk/src/models/messaging/message_header.rs | 55 ++-
sdk/src/models/messaging/message_header_view.rs | 28 +-
sdk/src/models/messaging/message_view.rs | 95 +++-
sdk/src/models/messaging/messages_batch.rs | 500 +++++++++++++++++++
sdk/src/models/messaging/mod.rs | 17 +-
.../messaging/{header.rs => user_headers.rs} | 0
sdk/src/prelude.rs | 9 +-
sdk/src/utils/byte_size.rs | 5 +
sdk/src/utils/mod.rs | 1 -
sdk/src/utils/varint.rs | 71 ---
server/Cargo.toml | 1 +
server/src/binary/command.rs | 4 +-
.../handlers/messages/poll_messages_handler.rs | 36 +-
.../handlers/messages/send_messages_handler.rs | 24 +-
server/src/channels/commands/archive_state.rs | 4 +-
server/src/channels/commands/maintain_messages.rs | 18 +-
.../src/compat/index_rebuilding/index_rebuilder.rs | 64 ++-
server/src/configs/defaults.rs | 6 +
server/src/configs/system.rs | 3 +-
server/src/http/messages.rs | 47 +-
server/src/quic/listener.rs | 68 ++-
server/src/quic/quic_sender.rs | 53 +-
server/src/streaming/cache/buffer.rs | 177 -------
server/src/streaming/cache/memory_tracker.rs | 101 ----
server/src/streaming/cache/mod.rs | 3 -
server/src/streaming/mod.rs | 2 +-
server/src/streaming/partitions/messages.rs | 249 ++++------
server/src/streaming/partitions/partition.rs | 40 --
server/src/streaming/partitions/persistence.rs | 7 +-
server/src/streaming/partitions/segments.rs | 18 +-
server/src/streaming/partitions/storage.rs | 55 +--
.../src/streaming/segments/indexes/index_reader.rs | 331 ++++++-------
.../src/streaming/segments/indexes/index_writer.rs | 8 +-
.../src/streaming/segments/indexes/indexes_mut.rs | 338 ++++++-------
server/src/streaming/segments/indexes/mod.rs | 15 +-
.../streaming/segments/indexes/read_boundary.rs | 27 --
.../streaming/segments/messages/messages_reader.rs | 101 ++--
.../streaming/segments/messages/messages_writer.rs | 20 +-
server/src/streaming/segments/messages/mod.rs | 24 +-
.../streaming/segments/messages/persister_task.rs | 6 +-
.../src/streaming/segments/messages_accumulator.rs | 174 ++++---
server/src/streaming/segments/mod.rs | 7 +-
server/src/streaming/segments/reading_messages.rs | 285 ++++++-----
server/src/streaming/segments/segment.rs | 171 ++++---
.../streaming/segments/types/message_view_mut.rs | 40 +-
server/src/streaming/segments/types/messages.rs | 214 --------
.../src/streaming/segments/types/messages_batch.rs | 225 ---------
.../streaming/segments/types/messages_batch_mut.rs | 245 ++++++++++
.../streaming/segments/types/messages_batch_set.rs | 356 ++++++++++++++
.../src/streaming/segments/types/messages_mut.rs | 121 -----
server/src/streaming/segments/types/mod.rs | 10 +-
server/src/streaming/segments/writing_messages.rs | 88 ++--
server/src/streaming/streams/topics.rs | 2 +-
server/src/streaming/systems/messages.rs | 211 ++++----
server/src/streaming/systems/snapshot/mod.rs | 2 +-
server/src/streaming/systems/system.rs | 31 +-
server/src/streaming/topics/messages.rs | 75 ++-
server/src/streaming/utils/file.rs | 30 +-
server/src/tcp/connection_handler.rs | 6 -
server/src/tcp/sender.rs | 3 +-
tools/src/data-seeder/seeder.rs | 85 +++-
106 files changed, 4251 insertions(+), 3269 deletions(-)
create mode 100644 sdk/src/messages/polled_messages.rs
rename {server/src/streaming/segments/indexes =>
sdk/src/models/messaging}/index.rs (100%)
rename {server/src/streaming/segments/indexes =>
sdk/src/models/messaging}/index_view.rs (91%)
create mode 100644 sdk/src/models/messaging/indexes.rs
create mode 100644 sdk/src/models/messaging/messages_batch.rs
rename sdk/src/models/messaging/{header.rs => user_headers.rs} (100%)
delete mode 100644 sdk/src/utils/varint.rs
delete mode 100644 server/src/streaming/cache/buffer.rs
delete mode 100644 server/src/streaming/cache/memory_tracker.rs
delete mode 100644 server/src/streaming/cache/mod.rs
delete mode 100644 server/src/streaming/segments/indexes/read_boundary.rs
delete mode 100644 server/src/streaming/segments/types/messages.rs
delete mode 100644 server/src/streaming/segments/types/messages_batch.rs
create mode 100644 server/src/streaming/segments/types/messages_batch_mut.rs
create mode 100644 server/src/streaming/segments/types/messages_batch_set.rs
delete mode 100644 server/src/streaming/segments/types/messages_mut.rs
