This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new e3af1d4c another round of cleanup ... (#1089)
e3af1d4c is described below
commit e3af1d4ca554db1d92c729d265031c911153b0cb
Author: Marko Milenković <[email protected]>
AuthorDate: Wed Oct 16 23:57:39 2024 +0100
another round of cleanup ... (#1089)
- ballista.plugin_dir
- update to `actions/checkout@v4`
relates to #1067
---
.github/workflows/comment_bot.yml | 4 ++--
.github/workflows/dev.yml | 6 +++---
.github/workflows/dev_pr.yml | 2 +-
.github/workflows/docker.yml | 4 ++--
.github/workflows/rust.yml | 20 ++++++++++----------
ballista/client/src/prelude.rs | 6 +++---
ballista/core/src/config.rs | 13 +++----------
docs/source/user-guide/configs.md | 1 -
8 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/.github/workflows/comment_bot.yml
b/.github/workflows/comment_bot.yml
index 51e53b7a..81b15939 100644
--- a/.github/workflows/comment_bot.yml
+++ b/.github/workflows/comment_bot.yml
@@ -31,7 +31,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
repository: apache/arrow
- name: Set up Python
@@ -54,7 +54,7 @@ jobs:
if: startsWith(github.event.comment.body, '@github-actions rebase')
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: r-lib/actions/pr-fetch@master
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index b71c5d98..2c02c0b8 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -24,7 +24,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
repository: apache/arrow
submodules: true
@@ -43,7 +43,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
@@ -55,7 +55,7 @@ jobs:
name: Use prettier to check formatting of documents
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "14"
diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
index c898cbe1..b8ee4ed9 100644
--- a/.github/workflows/dev_pr.yml
+++ b/.github/workflows/dev_pr.yml
@@ -29,7 +29,7 @@ jobs:
name: Process
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Assign GitHub labels
if: |
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a50ac823..ffeb9fa5 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -24,8 +24,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Installs Rust and Cargo
run: curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
- name: Run script
- run: ./dev/build-ballista-docker.sh
\ No newline at end of file
+ run: ./dev/build-ballista-docker.sh
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 1ffb9e18..cbda2632 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -38,7 +38,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Cache Cargo
uses: actions/cache@v4
with:
@@ -88,7 +88,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf compiler
@@ -142,7 +142,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf compiler
@@ -192,7 +192,7 @@ jobs:
os: [windows-latest, macos-latest]
rust: [stable]
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf macos compiler
@@ -252,7 +252,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf compiler
@@ -286,7 +286,7 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Setup toolchain
run: |
rustup toolchain install stable
@@ -306,7 +306,7 @@ jobs:
contents: read
packages: write
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Restore rust artifacts
uses: actions/download-artifact@v4
with:
@@ -343,7 +343,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Cache Cargo
@@ -386,7 +386,7 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: Cache Cargo
@@ -435,7 +435,7 @@ jobs:
# arch: [amd64]
# rust: [stable]
# steps:
-# - uses: actions/checkout@v3
+# - uses: actions/checkout@v4
# with:
# submodules: true
# - name: Cache Cargo
diff --git a/ballista/client/src/prelude.rs b/ballista/client/src/prelude.rs
index 0314c128..acab6652 100644
--- a/ballista/client/src/prelude.rs
+++ b/ballista/client/src/prelude.rs
@@ -21,9 +21,9 @@ pub use ballista_core::{
config::{
BallistaConfig, BALLISTA_COLLECT_STATISTICS,
BALLISTA_DEFAULT_BATCH_SIZE,
BALLISTA_DEFAULT_SHUFFLE_PARTITIONS,
BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE,
- BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING, BALLISTA_PLUGIN_DIR,
- BALLISTA_REPARTITION_AGGREGATIONS, BALLISTA_REPARTITION_JOINS,
- BALLISTA_REPARTITION_WINDOWS, BALLISTA_WITH_INFORMATION_SCHEMA,
+ BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING,
BALLISTA_REPARTITION_AGGREGATIONS,
+ BALLISTA_REPARTITION_JOINS, BALLISTA_REPARTITION_WINDOWS,
+ BALLISTA_WITH_INFORMATION_SCHEMA,
},
error::{BallistaError, Result},
};
diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs
index 746a0be9..db47a7e5 100644
--- a/ballista/core/src/config.rs
+++ b/ballista/core/src/config.rs
@@ -39,8 +39,7 @@ pub const BALLISTA_PARQUET_PRUNING: &str =
"ballista.parquet.pruning";
pub const BALLISTA_COLLECT_STATISTICS: &str = "ballista.collect_statistics";
pub const BALLISTA_WITH_INFORMATION_SCHEMA: &str =
"ballista.with_information_schema";
-/// give a plugin files dir, and then the dynamic library files in this dir
will be load when scheduler state init.
-pub const BALLISTA_PLUGIN_DIR: &str = "ballista.plugin_dir";
+
/// max message size for gRPC clients
pub const BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE: &str =
"ballista.grpc_client_max_message_size";
@@ -205,9 +204,7 @@ impl BallistaConfig {
"Configuration for collecting statistics during
scan".to_string(),
DataType::Boolean, Some("false".to_string())
),
- ConfigEntry::new(BALLISTA_PLUGIN_DIR.to_string(),
- "Sets the plugin dir".to_string(),
- DataType::Utf8, Some("".to_string())),
+
ConfigEntry::new(BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE.to_string(),
"Configuration for max message size in gRPC
clients".to_string(),
DataType::UInt64,
@@ -227,10 +224,6 @@ impl BallistaConfig {
self.get_usize_setting(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS)
}
- pub fn default_plugin_dir(&self) -> String {
- self.get_string_setting(BALLISTA_PLUGIN_DIR)
- }
-
pub fn default_batch_size(&self) -> usize {
self.get_usize_setting(BALLISTA_DEFAULT_BATCH_SIZE)
}
@@ -290,6 +283,7 @@ impl BallistaConfig {
v.parse::<bool>().unwrap()
}
}
+ #[allow(dead_code)]
fn get_string_setting(&self, key: &str) -> String {
if let Some(v) = self.settings.get(key) {
// infallible because we validate all configs in the constructor
@@ -403,7 +397,6 @@ mod tests {
fn custom_config_invalid() -> Result<()> {
let config = BallistaConfig::builder()
.set(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, "true")
- .set(BALLISTA_PLUGIN_DIR, "test_dir")
.build();
assert!(config.is_err());
assert_eq!("General(\"Failed to parse user-supplied value
'ballista.shuffle.partitions' for configuration setting 'true': ParseIntError {
kind: InvalidDigit }\")", format!("{:?}", config.unwrap_err()));
diff --git a/docs/source/user-guide/configs.md
b/docs/source/user-guide/configs.md
index d112e7c8..880ee490 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -46,7 +46,6 @@ let ctx = BallistaContext::remote("localhost", 50050, &
config).await?;
| ballista.repartition.windows | Boolean | true | When set to true,
Ballista will repartition data using the partition keys to execute window
functions in parallel using the provided `ballista.shuffle.partitions` level. |
| ballista.parquet.pruning | Boolean | true | Determines whether
Parquet pruning should be enabled or not.
|
| ballista.with_information_schema | Boolean | true | Determines whether
the `information_schema` should be created in the context. This is necessary
for supporting DDL commands such as `SHOW TABLES`. |
-| ballista.plugin_dir | Boolean | true | Specified a path for
plugin files. Dynamic library files in this directory will be loaded when
scheduler state initializes. |
### DataFusion Configuration Settings
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
