This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new a8518bd47 ci: Move platform build checks from java binding to rust
core (#3060)
a8518bd47 is described below
commit a8518bd474d354b36693fce0c925843917160fe1
Author: Xuanwo <[email protected]>
AuthorDate: Thu Sep 14 12:19:27 2023 +0800
ci: Move platform build checks from java binding to rust core (#3060)
* ci(bindings/java): Add features support
Signed-off-by: Xuanwo <[email protected]>
* Remove TODO
Signed-off-by: Xuanwo <[email protected]>
* cover other platforms
Signed-off-by: Xuanwo <[email protected]>
* Use bash shell
Signed-off-by: Xuanwo <[email protected]>
* disable rocksdb build for temp
Signed-off-by: Xuanwo <[email protected]>
* setup protoc
Signed-off-by: Xuanwo <[email protected]>
* Fix typo
Signed-off-by: Xuanwo <[email protected]>
* Fix sftp
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
.github/workflows/bindings_java.yml | 12 ++---
.github/workflows/ci.yml | 89 +++++++++++++++++++++++++++++++++--
.github/workflows/release_java.yml | 1 +
bindings/java/Cargo.toml | 92 ++++++++++++++++++++++++++++---------
bindings/java/pom.xml | 3 ++
bindings/java/tools/build.py | 4 ++
6 files changed, 170 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/bindings_java.yml
b/.github/workflows/bindings_java.yml
index 4d5803fcd..0411a32a5 100644
--- a/.github/workflows/bindings_java.yml
+++ b/.github/workflows/bindings_java.yml
@@ -60,10 +60,10 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
- include:
- - os: ubuntu-latest
- - os: windows-latest
- - os: macos-latest
+ os:
+ - ubuntu-latest
+ - macos-latest
+ - windows-latest
steps:
- uses: actions/checkout@v3
- name: Set up JDK 8
@@ -86,5 +86,5 @@ jobs:
if [[ "${{ runner.os }}" != Linux ]]; then
export NO_DOCKER=true
fi
- ./mvnw clean install -DskipTests
- ./mvnw verify artifact:compare
+ ./mvnw clean install -DskipTests
-Dcargo-build.features=services-redis
+ ./mvnw verify artifact:compare -Dcargo-build.features=services-redis
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d5072c28f..cf1568056 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -107,7 +107,9 @@ jobs:
uses: ./.github/actions/setup-ocaml
- name: Cargo clippy
- run: cargo clippy --all-targets --all-features --workspace -- -D
warnings
+ # FIXME
+ # remove `-exclude opendal-php` after
https://github.com/apache/incubator-opendal/issues/3055 addressed
+ run: cargo clippy --all-targets --all-features --workspace --exclude
opendal-php -- -D warnings
- name: Check diff
run: git diff --exit-code
@@ -126,7 +128,7 @@ jobs:
- name: Check
run: cargo +${OPENDAL_MSRV} clippy -p opendal --no-deps -- -D warnings
- build:
+ build_default_features:
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -154,8 +156,6 @@ jobs:
with:
distribution: temurin
java-version: "11"
- - name: Setup OCaml toolchain
- uses: ./.github/actions/setup-ocaml
- name: Setup Rust toolchain
uses: ./.github/actions/setup
@@ -166,6 +166,87 @@ jobs:
- name: Build
run: cargo build --all-features
+ build_all_platforms:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os:
+ - ubuntu-latest
+ - macos-latest
+ - windows-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Checkout python env
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.8"
+ - name: Checkout java env
+ uses: actions/setup-java@v3
+ with:
+ distribution: temurin
+ java-version: "11"
+
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ with:
+ need-rocksdb: true
+
+ - name: Install Protoc
+ uses: arduino/setup-protoc@v2
+ with:
+ version: "23.4"
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Build
+ shell: bash
+ run: |
+ FEATURES=(
+ services-azblob
+ services-azdls
+ services-cacache
+ services-cos
+ services-dashmap
+ services-dropbox
+ services-etcd
+ # FIXME this requires a preinstalled fdb library
+ # services-foundationdb
+ services-fs
+ services-ftp
+ services-gcs
+ services-gdrive
+ services-ghac
+ # FIXME how to support HDFS services in other platforms?
+ # services-hdfs
+ services-http
+ services-ipfs
+ services-ipmfs
+ services-memcached
+ services-memory
+ services-mini-moka
+ services-moka
+ services-obs
+ services-onedrive
+ services-oss
+ services-persy
+ services-postgresql
+ services-redb
+ services-redis
+ services-redis-rustls
+ # TODO: we need to find ways to using pre-install rocksdb library
+ # services-rocksdb
+ services-s3
+ # TODO: sftp is known to not work on windows, waiting for
https://github.com/apache/incubator-opendal/issues/2963
+ # services-sftp
+ services-sled
+ services-supabase
+ services-tikv
+ services-vercel-artifacts
+ services-wasabi
+ services-webdav
+ services-webhdfs
+ )
+ cargo build --features "${FEATURES[*]}"
+
unit:
runs-on: ubuntu-latest
steps:
diff --git a/.github/workflows/release_java.yml
b/.github/workflows/release_java.yml
index 6afd3c607..cee529acd 100644
--- a/.github/workflows/release_java.yml
+++ b/.github/workflows/release_java.yml
@@ -68,6 +68,7 @@ jobs:
-DskipTests=true \
-Djni.classifier=${{ matrix.classifier }} \
-Dcargo-build.profile=release \
+ -Dcargo-build.features=services-all \
-DaltStagingDirectory=local-staging \
-DskipRemoteStaging=true \
-DserverId=apache.releases.https \
diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml
index ee094eda4..09364f268 100644
--- a/bindings/java/Cargo.toml
+++ b/bindings/java/Cargo.toml
@@ -31,57 +31,107 @@ version.workspace = true
crate-type = ["cdylib"]
doc = false
-[dependencies]
-anyhow = "1.0.71"
-jni = "0.21.1"
-num_cpus = "1.15.0"
-once_cell = "1.17.1"
-tokio = { version = "1.28.1", features = ["full"] }
-
-[dependencies.opendal]
-workspace = true
-features = [
+[features]
+# Enable all opendal default feature by default.
+default = [
"services-azblob",
"services-azdls",
- "services-cacache",
"services-cos",
+ "services-fs",
+ "services-gcs",
+ "services-ghac",
+ "services-http",
+ "services-ipmfs",
+ "services-memory",
+ "services-obs",
+ "services-oss",
+ "services-s3",
+ "services-webdav",
+ "services-webhdfs",
+]
+
+services-all = [
+ "default",
+
+ "services-cacache",
"services-dashmap",
"services-dropbox",
"services-etcd",
# FIXME this requires a preinstalled fdb library
# "services-foundationdb",
- "services-fs",
"services-ftp",
- "services-gcs",
"services-gdrive",
- "services-ghac",
# FIXME how to support HDFS services in bindings?
# "services-hdfs",
- "services-http",
"services-ipfs",
"services-memcached",
- "services-memory",
"services-mini-moka",
"services-moka",
- "services-obs",
"services-onedrive",
- "services-oss",
"services-persy",
"services-postgresql",
"services-redb",
"services-redis",
"services-redis-rustls",
"services-rocksdb",
- "services-s3",
"services-sled",
"services-supabase",
"services-tikv",
"services-vercel-artifacts",
"services-wasabi",
- "services-webdav",
- "services-webhdfs",
]
+# Default services provided by opendal.
+services-azblob = [ "opendal/services-azblob" ]
+services-azdls = [ "opendal/services-azdls" ]
+services-cos = [ "opendal/services-cos" ]
+services-fs = [ "opendal/services-fs" ]
+services-gcs = [ "opendal/services-gcs" ]
+services-ghac = [ "opendal/services-ghac" ]
+services-http = [ "opendal/services-http" ]
+services-ipmfs = [ "opendal/services-ipmfs" ]
+services-memory = [ "opendal/services-memory" ]
+services-obs = [ "opendal/services-obs" ]
+services-oss = [ "opendal/services-oss" ]
+services-s3 = [ "opendal/services-s3" ]
+services-webdav = [ "opendal/services-webdav" ]
+services-webhdfs = [ "opendal/services-webhdfs" ]
+
+# Optional services provided by opendal.
+services-cacache = ["opendal/services-cacache"]
+services-dashmap = ["opendal/services-dashmap"]
+services-dropbox = ["opendal/services-dropbox"]
+services-etcd = ["opendal/services-etcd"]
+services-foundationdb = ["opendal/services-foundationdb"]
+services-ftp = ["opendal/services-ftp"]
+services-gdrive = ["opendal/services-gdrive"]
+services-hdfs = ["opendal/services-hdfs"]
+services-ipfs = ["opendal/services-ipfs"]
+services-memcached = ["opendal/services-memcached"]
+services-mini-moka = ["opendal/services-mini-moka"]
+services-moka = ["opendal/services-moka"]
+services-onedrive = ["opendal/services-onedrive"]
+services-persy = ["opendal/services-persy"]
+services-postgresql = ["opendal/services-postgresql"]
+services-redb = ["opendal/services-redb"]
+services-redis = ["opendal/services-redis"]
+services-redis-rustls = ["opendal/services-redis-rustls"]
+services-rocksdb = ["opendal/services-rocksdb"]
+services-sftp = ["opendal/services-sftp"]
+services-sled = ["opendal/services-sled"]
+services-supabase = ["opendal/services-supabase"]
+services-tikv = ["opendal/services-tikv"]
+services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
+services-wasabi = ["opendal/services-wasabi"]
+
+[dependencies]
+anyhow = "1.0.71"
+jni = "0.21.1"
+num_cpus = "1.15.0"
+once_cell = "1.17.1"
+tokio = { version = "1.28.1", features = ["full"] }
+opendal = { workspace = true }
+
# This is not optimal. See also the Cargo issue:
# https://github.com/rust-lang/cargo/issues/1197#issuecomment-1641086954
[target.'cfg(unix)'.dependencies.opendal]
diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml
index 2dea1f63f..fd785c931 100644
--- a/bindings/java/pom.xml
+++ b/bindings/java/pom.xml
@@ -53,6 +53,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<cargo-build.profile>dev</cargo-build.profile>
+ <cargo-build.features>default</cargo-build.features>
<jni.classifier>${os.detected.classifier}</jni.classifier>
<assertj.version>3.23.1</assertj.version>
@@ -193,6 +194,8 @@
<argument>${jni.classifier}</argument>
<argument>--profile</argument>
<argument>${cargo-build.profile}</argument>
+ <argument>--features</argument>
+ <argument>${cargo-build.features}</argument>
</arguments>
</configuration>
</execution>
diff --git a/bindings/java/tools/build.py b/bindings/java/tools/build.py
index 1492d7dad..2d6b3d2b0 100755
--- a/bindings/java/tools/build.py
+++ b/bindings/java/tools/build.py
@@ -53,10 +53,14 @@ if __name__ == '__main__':
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--classifier', type=str, required=True)
parser.add_argument('--profile', type=str, default='dev')
+ parser.add_argument('--features', type=str, default='default')
args = parser.parse_args()
cmd = ['cargo', 'build', '--color=always', f'--profile={args.profile}']
+ if args.features:
+ cmd += ['--features', args.features]
+
target = classifier_to_target(args.classifier)
if target:
command = ['rustup', 'target', 'add', target]
