This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new 97dd6f68 Using tokio tracing for log file (#123)
97dd6f68 is described below
commit 97dd6f68f69c6ddb7e0f4dcc4ae928be13e9629f
Author: Yang Jiang <[email protected]>
AuthorDate: Thu Aug 11 12:40:28 2022 +0800
Using tokio tracing for log file (#123)
---
ballista/rust/executor/Cargo.toml | 4 +++-
ballista/rust/executor/executor_config_spec.toml | 15 ++++++++++++++-
ballista/rust/executor/src/main.rs | 18 +++++++++++++-----
ballista/rust/scheduler/Cargo.toml | 4 +++-
ballista/rust/scheduler/scheduler_config_spec.toml | 14 +++++++++++++-
ballista/rust/scheduler/src/main.rs | 18 +++++++++++++-----
6 files changed, 59 insertions(+), 14 deletions(-)
diff --git a/ballista/rust/executor/Cargo.toml
b/ballista/rust/executor/Cargo.toml
index d05138ce..7d6dad2e 100644
--- a/ballista/rust/executor/Cargo.toml
+++ b/ballista/rust/executor/Cargo.toml
@@ -42,7 +42,6 @@ chrono = { version = "0.4", default-features = false }
configure_me = "0.4.0"
datafusion = { git = "https://github.com/apache/arrow-datafusion";, rev =
"6a5de4fe08597896ab6375e3e4b76c5744dcfba7" }
datafusion-proto = { git = "https://github.com/apache/arrow-datafusion";, rev =
"6a5de4fe08597896ab6375e3e4b76c5744dcfba7" }
-env_logger = "0.9"
futures = "0.3"
hyper = "0.14.4"
log = "0.4"
@@ -52,6 +51,9 @@ tempfile = "3"
tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread",
"parking_lot"] }
tokio-stream = { version = "0.1", features = ["net"] }
tonic = "0.7"
+tracing = "0.1.36"
+tracing-appender = "0.2.2"
+tracing-subscriber = { version = "0.3.15", features = ["fmt", "env-filter",
"ansi"] }
uuid = { version = "1.0", features = ["v4"] }
[dev-dependencies]
diff --git a/ballista/rust/executor/executor_config_spec.toml
b/ballista/rust/executor/executor_config_spec.toml
index 4dcfc670..e34c27fa 100644
--- a/ballista/rust/executor/executor_config_spec.toml
+++ b/ballista/rust/executor/executor_config_spec.toml
@@ -103,8 +103,21 @@ type = "String"
doc = "plugin dir"
default = "std::string::String::from(\"\")"
+[[param]]
+name = "log_dir"
+type = "String"
+doc = "Log dir: a path to save log. This will create a new storage directory
at the specified path if it does not already exist."
+default = "std::string::String::from(\"./logs\")"
+
+[[param]]
+name = "print_thread_info"
+type = "bool"
+doc = "Enable print thread ids and names in log file."
+default = "true"
+
[[param]]
name = "log_level_setting"
type = "String"
doc = "special log level for sub mod. link:
https://docs.rs/env_logger/latest/env_logger/#enabling-logging. For example we
want whole level is INFO but datafusion mode is DEBUG"
-default = "std::string::String::from(\"INFO, datafusion=INFO\")"
+default = "std::string::String::from(\"INFO,datafusion=INFO\")"
+
diff --git a/ballista/rust/executor/src/main.rs
b/ballista/rust/executor/src/main.rs
index cc7f41e3..882600e8 100644
--- a/ballista/rust/executor/src/main.rs
+++ b/ballista/rust/executor/src/main.rs
@@ -74,15 +74,23 @@ async fn main() -> Result<()> {
}
let special_mod_log_level = opt.log_level_setting;
- env_logger::builder()
- .parse_filters(&*special_mod_log_level)
- .format_timestamp_millis()
- .init();
-
let external_host = opt.external_host;
let bind_host = opt.bind_host;
let port = opt.bind_port;
let grpc_port = opt.bind_grpc_port;
+ let log_dir = opt.log_dir;
+ let print_thread_info = opt.print_thread_info;
+
+ let scheduler_name = format!("executor_{}_{}", bind_host, port);
+ let log_file = tracing_appender::rolling::daily(log_dir, &scheduler_name);
+
+ tracing_subscriber::fmt()
+ .with_ansi(true)
+ .with_thread_names(print_thread_info)
+ .with_thread_ids(print_thread_info)
+ .with_writer(log_file)
+ .with_env_filter(special_mod_log_level)
+ .init();
let addr = format!("{}:{}", bind_host, port);
let addr = addr
diff --git a/ballista/rust/scheduler/Cargo.toml
b/ballista/rust/scheduler/Cargo.toml
index e0555a46..826daa9d 100644
--- a/ballista/rust/scheduler/Cargo.toml
+++ b/ballista/rust/scheduler/Cargo.toml
@@ -44,7 +44,6 @@ clap = { version = "3", features = ["derive", "cargo"] }
configure_me = "0.4.0"
datafusion = { git = "https://github.com/apache/arrow-datafusion";, rev =
"6a5de4fe08597896ab6375e3e4b76c5744dcfba7" }
datafusion-proto = { git = "https://github.com/apache/arrow-datafusion";, rev =
"6a5de4fe08597896ab6375e3e4b76c5744dcfba7" }
-env_logger = "0.9"
etcd-client = { version = "0.9", optional = true }
flatbuffers = { version = "2.1.2" }
futures = "0.3"
@@ -63,6 +62,9 @@ tokio = { version = "1.0", features = ["full"] }
tokio-stream = { version = "0.1", features = ["net"], optional = true }
tonic = "0.7"
tower = { version = "0.4" }
+tracing = "0.1.36"
+tracing-appender = "0.2.2"
+tracing-subscriber = { version = "0.3.15", features = ["fmt", "env-filter",
"ansi"] }
uuid = { version = "1.0", features = ["v4"] }
warp = "0.3"
diff --git a/ballista/rust/scheduler/scheduler_config_spec.toml
b/ballista/rust/scheduler/scheduler_config_spec.toml
index cff529d5..c3be0f9d 100644
--- a/ballista/rust/scheduler/scheduler_config_spec.toml
+++ b/ballista/rust/scheduler/scheduler_config_spec.toml
@@ -78,8 +78,20 @@ type = "String"
doc = "Sled dir: Opens a Db for saving schduler metadata at the specified
path. This will create a new storage directory at the specified path if it does
not already exist."
default = "std::string::String::from(\"\")"
+[[param]]
+name = "log_dir"
+type = "String"
+doc = "Log dir: a path to save log. This will create a new storage directory
at the specified path if it does not already exist."
+default = "std::string::String::from(\"./logs\")"
+
+[[param]]
+name = "print_thread_info"
+type = "bool"
+doc = "Enable print thread ids and names in log file."
+default = "true"
+
[[param]]
name = "log_level_setting"
type = "String"
doc = "special log level for sub mod. link:
https://docs.rs/env_logger/latest/env_logger/#enabling-logging. For example we
want whole level is INFO but datafusion mode is DEBUG"
-default = "std::string::String::from(\"INFO, datafusion=INFO\")"
+default = "std::string::String::from(\"INFO,datafusion=INFO\")"
diff --git a/ballista/rust/scheduler/src/main.rs
b/ballista/rust/scheduler/src/main.rs
index 417e1855..5a7a9db1 100644
--- a/ballista/rust/scheduler/src/main.rs
+++ b/ballista/rust/scheduler/src/main.rs
@@ -157,14 +157,22 @@ async fn main() -> Result<()> {
}
let special_mod_log_level = opt.log_level_setting;
- env_logger::builder()
- .parse_filters(&*special_mod_log_level)
- .format_timestamp_millis()
- .init();
-
let namespace = opt.namespace;
let bind_host = opt.bind_host;
let port = opt.bind_port;
+ let log_dir = opt.log_dir;
+ let print_thread_info = opt.print_thread_info;
+
+ let scheduler_name = format!("scheduler_{}_{}_{}", namespace, bind_host,
port);
+ let log_file = tracing_appender::rolling::daily(log_dir, &scheduler_name);
+
+ tracing_subscriber::fmt()
+ .with_ansi(true)
+ .with_thread_names(print_thread_info)
+ .with_thread_ids(print_thread_info)
+ .with_writer(log_file)
+ .with_env_filter(special_mod_log_level)
+ .init();
let addr = format!("{}:{}", bind_host, port);
let addr = addr.parse()?;
