This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 6e3154ba document private items and update docs CI (#1327)
6e3154ba is described below
commit 6e3154ba56ee59c7b0afb95a6ede4db55eb6d92c
Author: killzoner <[email protected]>
AuthorDate: Tue Oct 14 21:34:42 2025 +0200
document private items and update docs CI (#1327)
* document private items and update docs CI
* update checkout to v5
---
.github/workflows/build.yml | 16 ++++----
.github/workflows/dev.yml | 2 +-
.github/workflows/dev_pr.yml | 2 +-
.github/workflows/docker.yml | 2 +-
.github/workflows/docs.yaml | 4 +-
.github/workflows/rust.yml | 47 ++++++++++++++++++------
ballista/core/src/client.rs | 6 ++-
ballista/executor/src/cpu_bound_executor.rs | 2 +-
ballista/executor/src/executor.rs | 2 +-
ballista/scheduler/src/cluster/mod.rs | 2 +-
ballista/scheduler/src/config.rs | 4 +-
ballista/scheduler/src/display.rs | 2 +-
ballista/scheduler/src/state/execution_graph.rs | 6 +--
ballista/scheduler/src/state/executor_manager.rs | 4 +-
ci/scripts/rust_docs.sh | 22 +++++++++++
15 files changed, 86 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f985deca..d1663ce3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -40,7 +40,7 @@ jobs:
- "3.12"
- "3.13"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: actions/setup-python@v5
with:
@@ -67,7 +67,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Install Python
uses: actions/setup-python@v5
with:
@@ -94,7 +94,7 @@ jobs:
generate-license:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
@@ -115,7 +115,7 @@ jobs:
python-version: ["3.10"]
os: [macos-latest, windows-latest]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: actions/setup-python@v5
with:
@@ -172,7 +172,7 @@ jobs:
matrix:
python-version: ["3.10"]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: actions/setup-python@v5
with:
@@ -217,7 +217,7 @@ jobs:
name: Manylinux x86_64
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
@@ -257,7 +257,7 @@ jobs:
name: Manylinux arm64
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
@@ -297,7 +297,7 @@ jobs:
name: Source distribution
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 08d239de..3f7f51f1 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -28,7 +28,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
- name: Setup Python
uses: actions/setup-python@v4
with:
diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
index 87368111..fb5b9c08 100644
--- a/.github/workflows/dev_pr.yml
+++ b/.github/workflows/dev_pr.yml
@@ -33,7 +33,7 @@ jobs:
name: Process
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Assign GitHub labels
if: |
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 4c8d81ad..d59e9e56 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -28,7 +28,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
- name: Installs Rust and Cargo
run: curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
- name: Run script
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 257224f1..d7429607 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -32,10 +32,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout docs sources
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
- name: Checkout asf-site branch
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
ref: asf-site
path: asf-site
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index abe9d3f4..55308556 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -46,9 +46,10 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Prepare cargo build
@@ -64,9 +65,10 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Run tests
@@ -79,15 +81,32 @@ jobs:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
+ # Run `cargo doc` to ensure the rustdoc is clean
+ linux-rustdoc:
+ name: cargo doc
+ needs: linux-build-lib
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Run cargo doc
+ run: ci/scripts/rust_docs.sh
+
linux-build-workspace:
name: check linux workspace
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Try to compile when `--no-default-features` is selected
@@ -106,9 +125,10 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Run Ballista tests
@@ -127,9 +147,10 @@ jobs:
name: windows test
runs-on: windows-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-windows-builder
- name: Run tests
@@ -146,9 +167,10 @@ jobs:
name: macos test
runs-on: macos-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-macos-builder
- name: Run tests
@@ -166,9 +188,10 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Verify that benchmark queries return expected results
@@ -181,7 +204,7 @@ jobs:
container:
image: amd64/rust
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Setup toolchain
run: |
rustup toolchain install stable
@@ -207,9 +230,10 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
@@ -237,9 +261,10 @@ jobs:
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
with:
submodules: true
+ fetch-depth: 1
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
@@ -274,7 +299,7 @@ jobs:
# arch: [amd64]
# rust: [stable]
# steps:
-# - uses: actions/checkout@v4
+# - uses: actions/checkout@v5
# with:
# submodules: true
# - name: Cache Cargo
diff --git a/ballista/core/src/client.rs b/ballista/core/src/client.rs
index 528570f9..6775dce0 100644
--- a/ballista/core/src/client.rs
+++ b/ballista/core/src/client.rs
@@ -141,6 +141,7 @@ impl BallistaClient {
})
}
+ #[allow(rustdoc::private_intra_doc_links)]
/// Executes the specified action and retrieves the results from the
remote executor.
///
/// This method establishes a [FlightDataStream] to facilitate the
transfer of data
@@ -298,7 +299,7 @@ impl BallistaClient {
/// compared to file-level compression, as it operates on smaller data
segments.
///
/// For further discussion regarding performance implications, refer to:
-/// https://github.com/apache/datafusion-ballista/issues/1315
+/// <https://github.com/apache/datafusion-ballista/issues/1315>
struct FlightDataStream {
stream: Streaming<FlightData>,
schema: SchemaRef,
@@ -346,6 +347,7 @@ impl RecordBatchStream for FlightDataStream {
self.schema.clone()
}
}
+#[allow(rustdoc::private_intra_doc_links)]
/// [BlockDataStream] facilitates the transfer of original shuffle files in a
block-by-block manner.
/// This implementation utilizes a custom `do_action` method on the Arrow
Flight server.
/// The primary distinction from [FlightDataStream] is that it does not
decompress or decode
@@ -353,7 +355,7 @@ impl RecordBatchStream for FlightDataStream {
/// on the Flight server and enables the transmission of less data, owing to
improved file-level compression.
///
/// For a detailed discussion of the performance advantages, see:
-/// https://github.com/apache/datafusion-ballista/issues/1315
+/// <https://github.com/apache/datafusion-ballista/issues/1315>
pub struct BlockDataStream<S: Stream<Item = Result<prost::bytes::Bytes>> +
Unpin> {
decoder: StreamDecoder,
state_buffer: Buffer,
diff --git a/ballista/executor/src/cpu_bound_executor.rs
b/ballista/executor/src/cpu_bound_executor.rs
index 5f1ed229..83e2c1cd 100644
--- a/ballista/executor/src/cpu_bound_executor.rs
+++ b/ballista/executor/src/cpu_bound_executor.rs
@@ -84,7 +84,7 @@ impl std::fmt::Debug for DedicatedExecutor {
}
impl DedicatedExecutor {
- /// https://stackoverflow.com/questions/62536566
+ /// <https://stackoverflow.com/questions/62536566>
/// Creates a new `DedicatedExecutor` with a dedicated tokio
/// runtime that is separate from the `[tokio::main]` threadpool.
///
diff --git a/ballista/executor/src/executor.rs
b/ballista/executor/src/executor.rs
index a8fdce17..3723c42e 100644
--- a/ballista/executor/src/executor.rs
+++ b/ballista/executor/src/executor.rs
@@ -110,7 +110,7 @@ impl Executor {
}
/// Create a new executor instance with given [RuntimeEnv],
- /// [ScalarUDF], [AggregateUDF] and [WindowUDF]
+ /// [datafusion::logical_expr::ScalarUDF],
[datafusion::logical_expr::AggregateUDF] and
[datafusion::logical_expr::WindowUDF]
#[allow(clippy::too_many_arguments)]
pub fn new(
metadata: ExecutorRegistration,
diff --git a/ballista/scheduler/src/cluster/mod.rs
b/ballista/scheduler/src/cluster/mod.rs
index 64da28d2..4942d452 100644
--- a/ballista/scheduler/src/cluster/mod.rs
+++ b/ballista/scheduler/src/cluster/mod.rs
@@ -149,7 +149,7 @@ pub trait ClusterState: Send + Sync + 'static {
Ok(())
}
- /// Bind the ready to running tasks from [`active_jobs`] with available
executors.
+ /// Bind the ready to running tasks from `active_jobs` with available
executors.
///
/// If `executors` is provided, only bind slots from the specified
executor IDs
async fn bind_schedulable_tasks(
diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs
index e02a6d61..559f1d67 100644
--- a/ballista/scheduler/src/config.rs
+++ b/ballista/scheduler/src/config.rs
@@ -356,7 +356,7 @@ pub enum TaskDistribution {
/// Distribute tasks evenly across executors. This will try and iterate
through available executors
/// and assign one task to each executor until all tasks are assigned.
RoundRobin,
- /// 1. Firstly, try to bind tasks without scanning source files by
[`RoundRobin`] policy.
+ /// 1. Firstly, try to bind tasks without scanning source files by
`RoundRobin` policy.
/// 2. Then for a task for scanning source files, firstly calculate a hash
value based on input files.
/// And then bind it with an execute according to consistent hashing
policy.
/// 3. If needed, work stealing can be enabled based on the tolerance of
the consistent hashing.
@@ -391,7 +391,7 @@ pub enum TaskDistributionPolicy {
/// Distribute tasks evenly across executors. This will try and iterate
through available executors
/// and assign one task to each executor until all tasks are assigned.
RoundRobin,
- /// 1. Firstly, try to bind tasks without scanning source files by
[`RoundRobin`] policy.
+ /// 1. Firstly, try to bind tasks without scanning source files by
`RoundRobin` policy.
/// 2. Then for a task for scanning source files, firstly calculate a hash
value based on input files.
/// And then bind it with an execute according to consistent hashing
policy.
/// 3. If needed, work stealing can be enabled based on the tolerance of
the consistent hashing.
diff --git a/ballista/scheduler/src/display.rs
b/ballista/scheduler/src/display.rs
index fa26331e..ea6d488e 100644
--- a/ballista/scheduler/src/display.rs
+++ b/ballista/scheduler/src/display.rs
@@ -16,7 +16,7 @@
// under the License.
//! Implementation of ballista physical plan display with metrics. See
-//! [`crate::physical_plan::displayable`] for examples of how to
+//! [`datafusion::physical_plan::display`] for examples of how to
//! format
use ballista_core::utils::collect_plan_metrics;
diff --git a/ballista/scheduler/src/state/execution_graph.rs
b/ballista/scheduler/src/state/execution_graph.rs
index fb5d470d..38339a3d 100644
--- a/ballista/scheduler/src/state/execution_graph.rs
+++ b/ballista/scheduler/src/state/execution_graph.rs
@@ -66,7 +66,7 @@ use crate::state::task_manager::UpdatedStages;
///
/// CoalesceBatchesExec: target_batch_size=4096
/// RepartitionExec: partitioning=Hash([Column { name: "id", index: 0 }],
4)
-/// AggregateExec: mode=Partial, gby=[id@0 as id],
aggr=[SUM(some_table.gmv)]
+/// AggregateExec: mode=Partial, gby=[id\@0 as id],
aggr=[SUM(some_table.gmv)]
/// TableScan: some_table
///
/// The Ballista `DistributedPlanner` will turn this into a distributed plan
by creating a shuffle
@@ -78,12 +78,12 @@ use crate::state::task_manager::UpdatedStages;
/// =========UnResolvedStage[id=2, children=1]=========
/// Inputs{1: StageOutput { partition_locations: {}, complete: false }}
/// ShuffleWriterExec: None
-/// AggregateExec: mode=FinalPartitioned, gby=[id@0 as id],
aggr=[SUM(?table?.gmv)]
+/// AggregateExec: mode=FinalPartitioned, gby=[id\@0 as id],
aggr=[SUM(?table?.gmv)]
/// CoalesceBatchesExec: target_batch_size=4096
/// UnresolvedShuffleExec
/// =========ResolvedStage[id=1, partitions=1]=========
/// ShuffleWriterExec: Some(Hash([Column { name: "id", index: 0 }], 4))
-/// AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[SUM(?table?.gmv)]
+/// AggregateExec: mode=Partial, gby=[id\@0 as id], aggr=[SUM(?table?.gmv)]
/// TableScan: some_table
///
///
diff --git a/ballista/scheduler/src/state/executor_manager.rs
b/ballista/scheduler/src/state/executor_manager.rs
index 9f3e2f5d..9c515d8f 100644
--- a/ballista/scheduler/src/state/executor_manager.rs
+++ b/ballista/scheduler/src/state/executor_manager.rs
@@ -70,7 +70,7 @@ impl ExecutorManager {
}
///
- /// Bind the ready to run tasks from [`active_jobs`] to available
executors.
+ /// Bind the ready to run tasks from `active_jobs` to available executors.
pub async fn bind_schedulable_tasks(
&self,
running_jobs: Arc<HashMap<String, JobInfoCache>>,
@@ -226,7 +226,7 @@ impl ExecutorManager {
/// It's only used for pull-based task scheduling.
///
- /// For push-based one, we should use [`register_executor`], instead.
+ /// For push-based one, we should use [`Self::register_executor`], instead.
pub async fn save_executor_metadata(&self, metadata: ExecutorMetadata) ->
Result<()> {
trace!(
"save executor metadata {} with {} task slots (pull-based
registration)",
diff --git a/ci/scripts/rust_docs.sh b/ci/scripts/rust_docs.sh
new file mode 100755
index 00000000..e90bfdf8
--- /dev/null
+++ b/ci/scripts/rust_docs.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+export RUSTDOCFLAGS="-D warnings"
+cargo doc --document-private-items --no-deps --workspace
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
