This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new ab9274cf1 ci: publish all rust crates in dependency order (#7352)
ab9274cf1 is described below
commit ab9274cf197287239132ef02f6f73327c05a59bb
Author: Xuanwo <[email protected]>
AuthorDate: Sat Apr 4 02:16:39 2026 +0800
ci: publish all rust crates in dependency order (#7352)
* ci: publish all split Rust crates
* ci: publish all rust crates in dependency order
* ci: extract rust release planning script
---
.github/scripts/release_rust/README.md | 49 +++++++++
.github/scripts/release_rust/plan.py | 176 ++++++++++++++++++++++++++++++
.github/scripts/release_rust/test_plan.py | 135 +++++++++++++++++++++++
.github/workflows/release_rust.yml | 62 ++++++-----
4 files changed, 394 insertions(+), 28 deletions(-)
diff --git a/.github/scripts/release_rust/README.md
b/.github/scripts/release_rust/README.md
new file mode 100644
index 000000000..8840d7abf
--- /dev/null
+++ b/.github/scripts/release_rust/README.md
@@ -0,0 +1,49 @@
+# Release Rust Plan
+
+This directory contains the planning logic for the Rust crates.io release
workflow.
+
+## Why this script exists
+
+After the repository split, Rust crates are no longer represented by a short
hand-maintained list.
+The release workflow needs to:
+
+- discover all publishable Rust crates under `core/` and `integrations/`
+- exclude crates with `publish = false`
+- publish them in dependency order so local path dependencies are already
available on crates.io
+
+Keeping this logic in a standalone script makes it testable and keeps the
workflow YAML readable.
+
+## Planned crate roots
+
+The planner scans:
+
+- `core/Cargo.toml`
+- `core/core/Cargo.toml`
+- `core/layers/*/Cargo.toml`
+- `core/services/*/Cargo.toml`
+- `integrations/*/Cargo.toml`
+
+It reads local `dependencies`, `build-dependencies`, and target-specific
variants of those tables
+to build a dependency graph, then emits a deterministic topological order.
+
+## Usage
+
+Print the publish order as JSON:
+
+```bash
+python3 .github/scripts/release_rust/plan.py
+```
+
+Write the same JSON to GitHub Actions output as `packages=<json>`:
+
+```bash
+python3 .github/scripts/release_rust/plan.py --github-output
+```
+
+## Tests
+
+Run the unit tests with:
+
+```bash
+python3 .github/scripts/release_rust/test_plan.py
+```
diff --git a/.github/scripts/release_rust/plan.py
b/.github/scripts/release_rust/plan.py
new file mode 100644
index 000000000..c7c15c850
--- /dev/null
+++ b/.github/scripts/release_rust/plan.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+import tomllib
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).resolve()
+PROJECT_DIR = SCRIPT_PATH.parents[3]
+PUBLISH_GLOBS = (
+ "core/Cargo.toml",
+ "core/core/Cargo.toml",
+ "core/layers/*/Cargo.toml",
+ "core/services/*/Cargo.toml",
+ "integrations/*/Cargo.toml",
+)
+
+
+@dataclass(frozen=True)
+class Package:
+ manifest_path: Path
+ manifest_dir: Path
+ path: str
+
+
+def discover_publishable_packages(project_dir: Path) -> dict[Path, Package]:
+ packages: dict[Path, Package] = {}
+
+ for pattern in PUBLISH_GLOBS:
+ for manifest_path in sorted(project_dir.glob(pattern)):
+ with manifest_path.open("rb") as fp:
+ manifest = tomllib.load(fp)
+
+ package = manifest.get("package")
+ if not package or package.get("publish") is False:
+ continue
+
+ manifest_dir = manifest_path.parent.resolve()
+ packages[manifest_dir] = Package(
+ manifest_path=manifest_path.resolve(),
+ manifest_dir=manifest_dir,
+ path=manifest_dir.relative_to(project_dir).as_posix(),
+ )
+
+ return packages
+
+
+def iter_local_dependencies(manifest: dict, manifest_dir: Path) -> list[Path]:
+ dependencies: list[Path] = []
+
+ def visit_table(table: dict | None) -> None:
+ if not isinstance(table, dict):
+ return
+
+ for dependency in table.values():
+ if not isinstance(dependency, dict):
+ continue
+ path = dependency.get("path")
+ if not isinstance(path, str):
+ continue
+ dependencies.append((manifest_dir / path).resolve())
+
+ for name in ("dependencies", "build-dependencies"):
+ visit_table(manifest.get(name))
+
+ for target in manifest.get("target", {}).values():
+ if not isinstance(target, dict):
+ continue
+ for name in ("dependencies", "build-dependencies"):
+ visit_table(target.get(name))
+
+ return dependencies
+
+
+def plan(project_dir: Path = PROJECT_DIR) -> list[str]:
+ project_dir = project_dir.resolve()
+ packages = discover_publishable_packages(project_dir)
+
+ graph: dict[Path, set[Path]] = defaultdict(set)
+ indegree = {manifest_dir: 0 for manifest_dir in packages}
+
+ for manifest_dir, package in packages.items():
+ with package.manifest_path.open("rb") as fp:
+ manifest = tomllib.load(fp)
+
+ for dependency_dir in iter_local_dependencies(manifest, manifest_dir):
+ if dependency_dir not in packages:
+ continue
+ if manifest_dir in graph[dependency_dir]:
+ continue
+
+ graph[dependency_dir].add(manifest_dir)
+ indegree[manifest_dir] += 1
+
+ queue = deque(
+ sorted(
+ (manifest_dir for manifest_dir, degree in indegree.items() if
degree == 0),
+ key=lambda manifest_dir: packages[manifest_dir].path,
+ )
+ )
+
+ ordered: list[str] = []
+ while queue:
+ manifest_dir = queue.popleft()
+ ordered.append(packages[manifest_dir].path)
+
+ for dependent in sorted(
+ graph[manifest_dir], key=lambda dependent: packages[dependent].path
+ ):
+ indegree[dependent] -= 1
+ if indegree[dependent] == 0:
+ queue.append(dependent)
+
+ if len(ordered) != len(packages):
+ raise RuntimeError("failed to resolve publish order for Rust crates")
+
+ return ordered
+
+
+def write_github_output(packages: list[str]) -> None:
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if not github_output:
+ raise RuntimeError("GITHUB_OUTPUT is not set")
+
+ with Path(github_output).open("a", encoding="utf-8") as fp:
+ fp.write(f"packages={json.dumps(packages)}\n")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Plan the publish order for Rust crates released from this
repository."
+ )
+ parser.add_argument(
+ "--project-dir",
+ type=Path,
+ default=PROJECT_DIR,
+ help="Path to the repository root.",
+ )
+ parser.add_argument(
+ "--github-output",
+ action="store_true",
+ help="Write the planned package list to GITHUB_OUTPUT as
`packages=<json>`.",
+ )
+ args = parser.parse_args()
+
+ packages = plan(args.project_dir)
+ print(json.dumps(packages))
+
+ if args.github_output:
+ write_github_output(packages)
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/release_rust/test_plan.py
b/.github/scripts/release_rust/test_plan.py
new file mode 100644
index 000000000..f7c227d09
--- /dev/null
+++ b/.github/scripts/release_rust/test_plan.py
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+from plan import plan
+
+
+def write_manifest(path: Path, content: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(textwrap.dedent(content).strip() + "\n", encoding="utf-8")
+
+
+class ReleaseRustPlanTest(unittest.TestCase):
+ def test_fixture_plan_orders_path_dependencies(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ root = Path(tmpdir)
+
+ write_manifest(
+ root / "core" / "core" / "Cargo.toml",
+ """
+ [package]
+ name = "opendal-core"
+ version = "0.1.0"
+ """,
+ )
+ write_manifest(
+ root / "core" / "layers" / "retry" / "Cargo.toml",
+ """
+ [package]
+ name = "opendal-layer-retry"
+ version = "0.1.0"
+
+ [dependencies]
+ opendal-core = { path = "../../core", version = "0.1.0" }
+ """,
+ )
+ write_manifest(
+ root / "core" / "services" / "fs" / "Cargo.toml",
+ """
+ [package]
+ name = "opendal-service-fs"
+ version = "0.1.0"
+
+ [dependencies]
+ opendal-core = { path = "../../core", version = "0.1.0" }
+ """,
+ )
+ write_manifest(
+ root / "core" / "Cargo.toml",
+ """
+ [package]
+ name = "opendal"
+ version = "0.1.0"
+
+ [dependencies]
+ opendal-core = { path = "core", version = "0.1.0" }
+ opendal-layer-retry = { path = "layers/retry", version =
"0.1.0" }
+
+ [target.'cfg(unix)'.build-dependencies]
+ opendal-service-fs = { path = "services/fs", version = "0.1.0"
}
+ """,
+ )
+ write_manifest(
+ root / "integrations" / "object_store" / "Cargo.toml",
+ """
+ [package]
+ name = "object_store_opendal"
+ version = "0.1.0"
+
+ [dependencies]
+ opendal = { path = "../../core", version = "0.1.0" }
+ """,
+ )
+ write_manifest(
+ root / "core" / "services" / "private" / "Cargo.toml",
+ """
+ [package]
+ name = "opendal-service-private"
+ version = "0.1.0"
+ publish = false
+ """,
+ )
+
+ result = plan(root)
+ self.assertEqual(
+ result,
+ [
+ "core/core",
+ "core/layers/retry",
+ "core/services/fs",
+ "core",
+ "integrations/object_store",
+ ],
+ )
+
+ def test_repository_plan_excludes_non_release_paths(self):
+ result = plan()
+
+ self.assertIn("core/core", result)
+ self.assertIn("core", result)
+ self.assertIn("integrations/object_store", result)
+
+ self.assertNotIn("bindings/python", result)
+ self.assertNotIn("core/testkit", result)
+ self.assertNotIn("core/examples/basic", result)
+
+ def test_repository_plan_orders_core_before_root_and_integrations(self):
+ result = plan()
+
+ self.assertLess(result.index("core/core"), result.index("core"))
+ self.assertLess(result.index("core"),
result.index("integrations/object_store"))
+ self.assertLess(result.index("core"),
result.index("integrations/parquet"))
+ self.assertLess(result.index("core"),
result.index("integrations/unftp-sbe"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/workflows/release_rust.yml
b/.github/workflows/release_rust.yml
index fefab0605..549e7d963 100644
--- a/.github/workflows/release_rust.yml
+++ b/.github/workflows/release_rust.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-# This workflow is used for publish all rust based packages
+# This workflow publishes all Rust packages that are intended for crates.io.
name: Release Rust Packages
@@ -30,23 +30,26 @@ on:
- ".github/workflows/release_rust.yml"
workflow_dispatch:
+permissions:
+ contents: read
+
jobs:
+ plan:
+ runs-on: ubuntu-latest
+ outputs:
+ packages: ${{ steps.plan.outputs.packages }}
+ steps:
+ - uses: actions/checkout@v6
+ - name: Plan publish matrix
+ id: plan
+ run: python3 .github/scripts/release_rust/plan.py --github-output
+
publish:
+ if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-')
}}
+ needs: plan
runs-on: ubuntu-latest
permissions:
id-token: write
- strategy:
- # Publish package one by one instead of flooding the registry
- max-parallel: 1
- fail-fast: false
- matrix:
- # Order here is sensitive, as it will be used to determine the order
of publishing
- package:
- - "core"
- - "integrations/object_store"
- - "integrations/parquet"
- - "integrations/dav-server"
- - "integrations/unftp-sbe"
steps:
- uses: actions/checkout@v6
- name: Checkout python env
@@ -58,28 +61,31 @@ jobs:
with:
distribution: zulu
java-version: 25
-
- name: Setup Rust toolchain
uses: ./.github/actions/setup
with:
need-rocksdb: true
need-protoc: true
-
- - name: Dryrun ${{ matrix.package }}
- # Only dryrun test upon core.
- if: matrix.package == 'core'
- working-directory: ${{ matrix.package }}
- run: cargo publish --dry-run
- env:
- LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{
env.LD_LIBRARY_PATH }}
-
- uses:
rust-lang/crates-io-auth-action@b7e9a28eded4986ec6b1fa40eeee8f8f165559ec
id: auth
- - name: Publish ${{ matrix.package }}
- working-directory: ${{ matrix.package }}
- # Only publish if it's a tag and the tag is not a pre-release
- if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref,
'-') }}
- run: cargo publish --no-verify
+ - name: Publish Rust crates
+ shell: python
env:
+ PACKAGES: ${{ needs.plan.outputs.packages }}
LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{
env.LD_LIBRARY_PATH }}
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
+ run: |
+ import json
+ import os
+ import subprocess
+
+ packages = json.loads(os.environ["PACKAGES"])
+
+ for package in packages:
+ print(f"Publishing {package}")
+ subprocess.run(
+ ["cargo", "publish", "--no-verify"],
+ cwd=package,
+ check=True,
+ env=os.environ,
+ )
