This is an automated email from the ASF dual-hosted git repository.
xikai pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-horaedb.git
The following commit(s) were added to refs/heads/main by this push:
new d7239ed8 feat: add system_stats lib to collect system stats (#1442)
d7239ed8 is described below
commit d7239ed8d6b2d7bafbd4843133b6b28d22275451
Author: WEI Xikai <[email protected]>
AuthorDate: Wed Jan 17 17:18:34 2024 +0800
feat: add system_stats lib to collect system stats (#1442)
## Rationale
Part of #1438. The preparations for reporting system statistics.
## Detailed Changes
- Add a new component called `system_stats` to helps collect system
statistics.
- Introduce a dependency [`sysinfo`](https://crates.io/crates/sysinfo).
## Test Plan
Add a new unit test.
---
Cargo.lock | 119 ++++++++++++++++-
Cargo.toml | 4 +-
components/skiplist/Cargo.toml | 2 -
components/{skiplist => system_stats}/Cargo.toml | 19 +--
components/system_stats/src/lib.rs | 158 +++++++++++++++++++++++
5 files changed, 277 insertions(+), 25 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 9e29407d..5c08c3e5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3233,7 +3233,7 @@ dependencies = [
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
- "windows",
+ "windows 0.47.0",
]
[[package]]
@@ -4292,6 +4292,15 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@@ -5704,9 +5713,9 @@ dependencies = [
[[package]]
name = "rayon"
-version = "1.7.0"
+version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
+checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
dependencies = [
"either",
"rayon-core",
@@ -5714,14 +5723,12 @@ dependencies = [
[[package]]
name = "rayon-core"
-version = "1.11.0"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
+checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
dependencies = [
- "crossbeam-channel",
"crossbeam-deque 0.8.3",
"crossbeam-utils 0.8.15",
- "num_cpus",
]
[[package]]
@@ -6871,6 +6878,20 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+[[package]]
+name = "sysinfo"
+version = "0.30.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1fb4f3438c8f6389c864e61221cbc97e9bca98b4daf39a5beb7bea660f528bb2"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "windows 0.52.0",
+]
+
[[package]]
name = "system_catalog"
version = "1.2.6-alpha"
@@ -6893,6 +6914,14 @@ dependencies = [
"trace_metric",
]
+[[package]]
+name = "system_stats"
+version = "1.2.6-alpha"
+dependencies = [
+ "sysinfo",
+ "tokio",
+]
+
[[package]]
name = "table_engine"
version = "1.2.6-alpha"
@@ -8028,6 +8057,25 @@ dependencies = [
"windows-targets 0.47.0",
]
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.0",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets 0.52.0",
+]
+
[[package]]
name = "windows-sys"
version = "0.45.0"
@@ -8091,6 +8139,21 @@ dependencies = [
"windows_x86_64_msvc 0.48.0",
]
+[[package]]
+name = "windows-targets"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.0",
+ "windows_aarch64_msvc 0.52.0",
+ "windows_i686_gnu 0.52.0",
+ "windows_i686_msvc 0.52.0",
+ "windows_x86_64_gnu 0.52.0",
+ "windows_x86_64_gnullvm 0.52.0",
+ "windows_x86_64_msvc 0.52.0",
+]
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
@@ -8109,6 +8172,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
@@ -8127,6 +8196,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
@@ -8145,6 +8220,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
@@ -8163,6 +8244,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
@@ -8181,6 +8268,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
@@ -8199,6 +8292,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
@@ -8217,6 +8316,12 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+
[[package]]
name = "winnow"
version = "0.4.1"
diff --git a/Cargo.toml b/Cargo.toml
index 24a9b13a..9813f1c5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -53,6 +53,7 @@ members = [
"components/sampling_cache",
"components/size_ext",
"components/skiplist",
+ "components/system_stats",
"components/table_kv",
"components/test_util",
"components/time_ext",
@@ -164,7 +165,7 @@ router = { path = "router" }
runtime = { path = "components/runtime" }
sampling_cache = { path = "components/sampling_cache" }
snafu = { version = "0.6.10", features = ["backtraces"] }
-serde = "1.0"
+serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.60"
server = { path = "server" }
size_ext = { path = "components/size_ext" }
@@ -173,6 +174,7 @@ slog = "2.7"
spin = "0.9.6"
sqlparser = { version = "0.35", features = ["serde"] }
system_catalog = { path = "system_catalog" }
+system_statis = { path = "component/system_stats" }
table_engine = { path = "table_engine" }
table_kv = { path = "components/table_kv" }
tempfile = "3.1.0"
diff --git a/components/skiplist/Cargo.toml b/components/skiplist/Cargo.toml
index 544915ab..84b0a5ce 100644
--- a/components/skiplist/Cargo.toml
+++ b/components/skiplist/Cargo.toml
@@ -36,8 +36,6 @@ rand = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
yatp = { git = "https://github.com/tikv/yatp.git";, rev =
"4b71f8abd86890f0d1e95778c2b6bf5a9ee4c502" }
-# [target.'cfg(not(target_env = "msvc"))'.dev-dependencies]
-# tikv-jemallocator = "0.4.0"
[[bench]]
name = "bench"
diff --git a/components/skiplist/Cargo.toml b/components/system_stats/Cargo.toml
similarity index 66%
copy from components/skiplist/Cargo.toml
copy to components/system_stats/Cargo.toml
index 544915ab..b4ab6112 100644
--- a/components/skiplist/Cargo.toml
+++ b/components/system_stats/Cargo.toml
@@ -16,8 +16,8 @@
# under the License.
[package]
-name = "skiplist"
-authors = ["Jay Lee <[email protected]>", "HoraeDB Authors"]
+name = "system_stats"
+authors = ["HoraeDB Authors"]
[package.license]
workspace = true
@@ -29,16 +29,5 @@ workspace = true
workspace = true
[dependencies]
-arena = { workspace = true }
-bytes = { workspace = true }
-rand = { workspace = true }
-
-[dev-dependencies]
-criterion = { workspace = true }
-yatp = { git = "https://github.com/tikv/yatp.git";, rev =
"4b71f8abd86890f0d1e95778c2b6bf5a9ee4c502" }
-# [target.'cfg(not(target_env = "msvc"))'.dev-dependencies]
-# tikv-jemallocator = "0.4.0"
-
-[[bench]]
-name = "bench"
-harness = false
+sysinfo = { version = "0.30", default-features = false }
+tokio = { workspace = true }
diff --git a/components/system_stats/src/lib.rs
b/components/system_stats/src/lib.rs
new file mode 100644
index 00000000..a5680bfc
--- /dev/null
+++ b/components/system_stats/src/lib.rs
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Helps to collect and report statistics about the system.
+
+use std::{sync::Mutex, time::Duration};
+
+pub use sysinfo::LoadAvg;
+use sysinfo::{Cpu, CpuRefreshKind, MemoryRefreshKind, RefreshKind, System};
+
+/// The stats about the system.
+#[derive(Debug)]
+pub struct SystemStats {
+ /// The usage's range is [0, 1.0]
+ pub cpu_usage: f32,
+ /// The memory is counted in byte.
+ pub used_memory: u64,
+ /// The memory is counted in byte.
+ pub total_memory: u64,
+ pub load_avg: LoadAvg,
+}
+
+/// Collect the stats of the system for reporting.
+///
+/// One background thread will be spawned to run stats collection.
+pub struct SystemStatsCollector {
+ total_memory: u64,
+ system: Mutex<System>,
+}
+
+pub type ErrorMessage = String;
+
+impl SystemStatsCollector {
+ /// Create an new collector for the system stats.
+ pub fn try_new() -> Result<Self, ErrorMessage> {
+ if !sysinfo::IS_SUPPORTED_SYSTEM {
+ return Err("Unsupported system to collect system
metrics".to_string());
+ }
+
+ let system = System::new_with_specifics(Self::make_mem_refresh_kind());
+ Ok(Self {
+ total_memory: system.total_memory(),
+ system: Mutex::new(system),
+ })
+ }
+
+ /// Collect the system stats for `observe_dur`.
+ ///
+ /// The [`sysinfo::MINIMUM_CPU_UPDATE_INTERVAL`] will be used if
+ /// `observe_dur` is smaller.
+ pub async fn collect_and_report(&self, observe_dur: Duration) ->
SystemStats {
+ {
+ let mut system = self.system.lock().unwrap();
+ system.refresh_specifics(Self::make_cpu_refresh_kind());
+ }
+
+ let wait_dur = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.max(observe_dur);
+ tokio::time::sleep(wait_dur).await;
+
+ let mut system = self.system.lock().unwrap();
+ system.refresh_specifics(Self::make_cpu_and_mem_refresh_kind());
+
+ SystemStats {
+ cpu_usage: self.compute_cpu_usage(system.cpus()),
+ used_memory: system.used_memory(),
+ total_memory: self.total_memory,
+ load_avg: System::load_average(),
+ }
+ }
+
+ // Refresh and compute the latest cpu usage.
+ fn compute_cpu_usage(&self, cpus: &[Cpu]) -> f32 {
+ let mut num_cpus = 0;
+ let mut total_cpu_usage = 0.0;
+ let valid_cpus = cpus.iter().filter(|v| !v.cpu_usage().is_nan());
+ for cpu in valid_cpus {
+ total_cpu_usage += cpu.cpu_usage();
+ num_cpus += 1;
+ }
+
+ if num_cpus != 0 {
+ total_cpu_usage / (num_cpus as f32) / 100.0
+ } else {
+ 0f32
+ }
+ }
+
+ #[inline]
+ fn make_mem_refresh_kind() -> RefreshKind {
+ let mem_refresh_kind = MemoryRefreshKind::new().with_ram();
+ RefreshKind::new().with_memory(mem_refresh_kind)
+ }
+
+ #[inline]
+ fn make_cpu_refresh_kind() -> RefreshKind {
+ let cpu_refresh_kind = CpuRefreshKind::new().with_cpu_usage();
+ RefreshKind::new().with_cpu(cpu_refresh_kind)
+ }
+
+ #[inline]
+ fn make_cpu_and_mem_refresh_kind() -> RefreshKind {
+ let cpu_refresh_kind = CpuRefreshKind::new().with_cpu_usage();
+ let mem_refresh_kind = MemoryRefreshKind::new().with_ram();
+ RefreshKind::new()
+ .with_cpu(cpu_refresh_kind)
+ .with_memory(mem_refresh_kind)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn check_system_stats(stats: &SystemStats) {
+ assert!(stats.total_memory > 0);
+ assert!(stats.used_memory > 0);
+ assert!(stats.used_memory < stats.total_memory);
+ assert!(stats.cpu_usage > 0.0);
+ assert!(stats.load_avg.one > 0.0);
+ assert!(stats.load_avg.five > 0.0);
+ assert!(stats.load_avg.fifteen > 0.0);
+ }
+
+ #[tokio::test]
+ async fn test_normal_case() {
+ let collector = SystemStatsCollector::try_new().unwrap();
+ let stats = collector
+ .collect_and_report(Duration::from_millis(500))
+ .await;
+ check_system_stats(&stats);
+
+ let mut all_cpu_usages = Vec::with_capacity(5);
+ for _ in 0..5 {
+ let new_stats = collector
+ .collect_and_report(Duration::from_millis(200))
+ .await;
+ check_system_stats(&new_stats);
+ all_cpu_usages.push(new_stats.cpu_usage);
+ }
+
+ // Ensure the stats will be updated for every collection.
+ assert!(all_cpu_usages.into_iter().any(|v| v != stats.cpu_usage));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
