haze518 commented on code in PR #1838:
URL: https://github.com/apache/iggy/pull/1838#discussion_r2133305896
##########
core/sdk/src/clients/producer_builder.rs:
##########
@@ -15,27 +15,234 @@
// specific language governing permissions and limitations
// under the License.
-use super::MAX_BATCH_SIZE;
+use super::MAX_BATCH_LENGTH;
+use crate::clients::producer_config::{BackgroundConfig, BackpressureMode,
SyncConfig};
+use crate::clients::producer_error_callback::ErrorCallback;
+use crate::clients::producer_error_callback::LogErrorCallback;
+use crate::clients::producer_sharding::{BalancedSharding, Sharding};
use crate::prelude::IggyProducer;
use iggy_binary_protocol::Client;
use iggy_common::locking::IggySharedMut;
use iggy_common::{
- EncryptorKind, Identifier, IggyDuration, IggyExpiry, MaxTopicSize,
Partitioner, Partitioning,
+ EncryptorKind, Identifier, IggyByteSize, IggyDuration, IggyExpiry,
MaxTopicSize, Partitioner,
+ Partitioning,
};
use std::sync::Arc;
-#[derive(Debug)]
+pub struct BackgroundBuilder {
+ num_shards: Option<usize>,
+ batch_size: Option<usize>,
+ batch_length: Option<usize>,
+ failure_mode: Option<BackpressureMode>,
+ max_buffer_size: Option<IggyByteSize>,
+ linger_time: Option<IggyDuration>,
+ max_in_flight: Option<usize>,
+
+ error_callback: Box<dyn ErrorCallback>,
+ sharding: Box<dyn Sharding>,
+}
+
+impl Default for BackgroundBuilder {
+ fn default() -> Self {
+ let num_shards = default_shard_count();
+ BackgroundBuilder {
+ num_shards: Some(num_shards),
Review Comment:
in this case, shards are created to parallelize serialization, encryption,
and network calls, removing the bottleneck of a single channel and thereby
increasing throughput. It’s similar to the writer-thread-per-partition approach
in kafka clients. However, since we currently don’t know the target partition
on the client side (except in the case of PartitioningKind::PartitionId), we
use artificial shard queues instead.
An additional advantage is that if partition calculation is moved to the
client in the future, the shards can be easily repurposed to represent actual
partitions without requiring a refactor of the rest of the code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
