This is an automated email from the ASF dual-hosted git repository.
jiacai2050 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-horaedb.git
The following commit(s) were added to refs/heads/dev by this push:
new 3359a9a1 fix: skip wal encoding when data wal is disabled (#1401)
3359a9a1 is described below
commit 3359a9a1c12a9e000e353401b3dc4438038c1240
Author: Jiacai Liu <[email protected]>
AuthorDate: Wed Dec 27 12:25:19 2023 +0800
fix: skip wal encoding when data wal is disabled (#1401)
## Rationale
When data wal is disable, data is still encoded, which waste cpu usage.
## Detailed Changes
Skip encode when data wal is disabled.
## Test Plan
CI.
---
analytic_engine/src/instance/mod.rs | 1 +
analytic_engine/src/instance/open.rs | 1 +
analytic_engine/src/instance/write.rs | 27 ++++++++++++++++++---------
analytic_engine/src/lib.rs | 1 +
analytic_engine/src/table/data.rs | 5 +++++
5 files changed, 26 insertions(+), 9 deletions(-)
diff --git a/analytic_engine/src/instance/mod.rs
b/analytic_engine/src/instance/mod.rs
index ab8df1ef..6eb257b8 100644
--- a/analytic_engine/src/instance/mod.rs
+++ b/analytic_engine/src/instance/mod.rs
@@ -189,6 +189,7 @@ pub struct Instance {
pub(crate) iter_options: Option<IterOptions>,
pub(crate) recover_mode: RecoverMode,
pub(crate) wal_encode: WalEncodeConfig,
+ pub(crate) disable_wal: bool,
}
impl Instance {
diff --git a/analytic_engine/src/instance/open.rs
b/analytic_engine/src/instance/open.rs
index 0f9ad18a..31702dc9 100644
--- a/analytic_engine/src/instance/open.rs
+++ b/analytic_engine/src/instance/open.rs
@@ -149,6 +149,7 @@ impl Instance {
scan_options,
recover_mode: ctx.config.recover_mode,
wal_encode: ctx.config.wal_encode,
+ disable_wal: ctx.config.wal.disable_data,
});
Ok(instance)
diff --git a/analytic_engine/src/instance/write.rs
b/analytic_engine/src/instance/write.rs
index e49ccaac..2e5d60d0 100644
--- a/analytic_engine/src/instance/write.rs
+++ b/analytic_engine/src/instance/write.rs
@@ -422,19 +422,28 @@ impl<'a> Writer<'a> {
self.preprocess_write(&mut encode_ctx).await?;
- let encoded_payload = {
- let _timer =
self.table_data.metrics.start_table_write_encode_timer();
- let schema = self.table_data.schema();
- encode_ctx.encode(&self.instance.wal_encode, &schema)?
- };
+ let table_data = self.table_data.clone();
+ let seq = if self.instance.disable_wal {
+ // When wal is disabled, just update the last_seq one by one.
+ table_data.next_sequence()
+ } else {
+ let encoded_payload = {
+ let _timer =
self.table_data.metrics.start_table_write_encode_timer();
+ let schema = self.table_data.schema();
+ encode_ctx.encode(&self.instance.wal_encode, &schema)?
+ };
- let seq = match encoded_payload {
- EncodedPayload::Rows(encoded_rows) =>
self.write_to_wal_in_rows(encoded_rows).await?,
- EncodedPayload::Cols(encoded_cols) =>
self.write_to_wal_in_cols(encoded_cols).await?,
+ match encoded_payload {
+ EncodedPayload::Rows(encoded_rows) => {
+ self.write_to_wal_in_rows(encoded_rows).await?
+ }
+ EncodedPayload::Cols(encoded_cols) => {
+ self.write_to_wal_in_cols(encoded_cols).await?
+ }
+ }
};
// Write the row group to the memtable and update the state in the mem.
- let table_data = self.table_data.clone();
let EncodeContext {
row_group,
index_in_writer,
diff --git a/analytic_engine/src/lib.rs b/analytic_engine/src/lib.rs
index e7d7f810..68f79683 100644
--- a/analytic_engine/src/lib.rs
+++ b/analytic_engine/src/lib.rs
@@ -110,6 +110,7 @@ pub struct Config {
/// The interval for sampling the memory usage
pub mem_usage_sampling_interval: ReadableDuration,
/// The config for log in the wal.
+ // TODO: move this to WalConfig.
pub wal_encode: WalEncodeConfig,
/// Wal storage config
diff --git a/analytic_engine/src/table/data.rs
b/analytic_engine/src/table/data.rs
index 2c011a9c..998c5c3b 100644
--- a/analytic_engine/src/table/data.rs
+++ b/analytic_engine/src/table/data.rs
@@ -446,6 +446,11 @@ impl TableData {
self.last_sequence.store(seq, Ordering::Release);
}
+ #[inline]
+ pub fn next_sequence(&self) -> SequenceNumber {
+ self.last_sequence.fetch_add(1, Ordering::Relaxed) + 1
+ }
+
/// Get last flush time
#[inline]
pub fn last_flush_time(&self) -> u64 {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
