This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1b3c95fd7a ci: Migrate hdfs default setup (#4140)
1b3c95fd7a is described below
commit 1b3c95fd7a6792528528fbd4aae348d551c38142
Author: Xuanwo <[email protected]>
AuthorDate: Mon Feb 5 11:35:12 2024 +0800
ci: Migrate hdfs default setup (#4140)
* ci: Migrate hdfs default setup
Signed-off-by: Xuanwo <[email protected]>
* Fix build
Signed-off-by: Xuanwo <[email protected]>
* try fix classpath
Signed-off-by: Xuanwo <[email protected]>
* Revert "try fix classpath"
This reverts commit 7c7b85aa5a6634c8bb601368fee5099610d95648.
* trick for mvn
Signed-off-by: Xuanwo <[email protected]>
* try
Signed-off-by: Xuanwo <[email protected]>
* Don't run hdfs on java
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
.github/scripts/test_behavior/plan.py | 4 +++
.github/services/hdfs/hdfs_default/action.yml | 46 +++++++++++++++++++++++++++
.github/workflows/service_test_hdfs.yml | 36 ---------------------
3 files changed, 50 insertions(+), 36 deletions(-)
diff --git a/.github/scripts/test_behavior/plan.py
b/.github/scripts/test_behavior/plan.py
index 66a44b13e7..164d50c77b 100755
--- a/.github/scripts/test_behavior/plan.py
+++ b/.github/scripts/test_behavior/plan.py
@@ -198,6 +198,10 @@ def generate_language_binding_cases(
) -> list[dict[str, str]]:
cases = unique_cases(cases)
+ # Remove hdfs cases for java.
+ if language == "java":
+ cases = [v for v in cases if v["service"] != "hdfs"]
+
if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
diff --git a/.github/services/hdfs/hdfs_default/action.yml
b/.github/services/hdfs/hdfs_default/action.yml
new file mode 100644
index 0000000000..d4bdce0765
--- /dev/null
+++ b/.github/services/hdfs/hdfs_default/action.yml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default
+description: 'Behavior test for hdfs default'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup java env
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: "11"
+ - name: Setup
+ shell: bash
+ run: |
+ curl -LsSf
https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar
zxf - -C /home/runner
+
+ export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+ export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+ cp ./fixtures/hdfs/hdfs-site.xml
${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+ cat << EOF >> $GITHUB_ENV
+ HADOOP_HOME=${HADOOP_HOME}
+ CLASSPATH=${CLASSPATH}
+ LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+ OPENDAL_HDFS_ROOT=/tmp/opendal/
+ OPENDAL_HDFS_NAME_NODE=default
+ OPENDAL_HDFS_ENABLE_APPEND=false
+ EOF
diff --git a/.github/workflows/service_test_hdfs.yml
b/.github/workflows/service_test_hdfs.yml
index 1e1dae9f7e..c4447c2d45 100644
--- a/.github/workflows/service_test_hdfs.yml
+++ b/.github/workflows/service_test_hdfs.yml
@@ -37,42 +37,6 @@ concurrency:
cancel-in-progress: true
jobs:
- hdfs-default:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
-
- - name: Setup java env
- uses: actions/setup-java@v4
- with:
- distribution: temurin
- java-version: "11"
- - name: Setup hadoop env
- shell: bash
- run: |
- curl -LsSf
https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar
zxf - -C /home/runner
-
- - name: Test
- shell: bash
- working-directory: core
- run: |
- export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
- export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{
env.HADOOP_HOME }}/lib/native
- cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${{
env.HADOOP_HOME }}/etc/hadoop/hdfs-site.xml
-
- cargo test behavior --features tests,services-hdfs
- env:
- HADOOP_HOME: "/home/runner/hadoop-3.3.5"
- OPENDAL_TEST: hdfs
- OPENDAL_HDFS_ROOT: /tmp/opendal/
- OPENDAL_HDFS_NAME_NODE: default
- OPENDAL_HDFS_ENABLE_APPEND: false
-
hdfs-cluster:
runs-on: ubuntu-latest
steps:
