This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1bfc47b2c refactor(services/webhdfs): migrate to test planner (#3578)
1bfc47b2c is described below
commit 1bfc47b2c6415df5e1c7d86eacf3aba4365fd30c
Author: G-XD <[email protected]>
AuthorDate: Tue Nov 14 01:13:47 2023 +0800
refactor(services/webhdfs): migrate to test planner (#3578)
---
.github/services/webhdfs/webhdfs/action.yml | 35 ++++++
.../webhdfs_with_list_batch_disabled/action.yml | 36 +++++++
.github/workflows/service_test_webhdfs.yml | 117 ---------------------
fixtures/webhdfs/docker-compose-webhdfs.yml | 42 ++++++++
4 files changed, 113 insertions(+), 117 deletions(-)
diff --git a/.github/services/webhdfs/webhdfs/action.yml
b/.github/services/webhdfs/webhdfs/action.yml
new file mode 100644
index 000000000..ae9cdfa73
--- /dev/null
+++ b/.github/services/webhdfs/webhdfs/action.yml
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: webhdfs
+description: 'Behavior test for webhdfs'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup webhdfs
+ shell: bash
+ working-directory: fixtures/webhdfs
+ run: |
+ docker compose -f docker-compose-webhdfs.yml up -d --wait
+ - name: Setup
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_WEBHDFS_ROOT=/
+ OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870
+ EOF
diff --git
a/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml
b/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml
new file mode 100644
index 000000000..7fb294b04
--- /dev/null
+++ b/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: webhdfs_with_list_batch_disabled
+description: 'Behavior test for webhdfs with list batch disabled'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup webhdfs
+ shell: bash
+ working-directory: fixtures/webhdfs
+ run: |
+ docker compose -f docker-compose-webhdfs.yml up -d --wait
+ - name: Setup
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_WEBHDFS_ROOT=/
+ OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870
+ OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=true
+ EOF
diff --git a/.github/workflows/service_test_webhdfs.yml
b/.github/workflows/service_test_webhdfs.yml
deleted file mode 100644
index 4db1f3a29..000000000
--- a/.github/workflows/service_test_webhdfs.yml
+++ /dev/null
@@ -1,117 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Service Test WebHDFS
-
-on:
- push:
- branches:
- - main
- pull_request:
- branches:
- - main
- paths:
- - "core/src/**"
- - "core/tests/**"
- - "!core/src/docs/**"
- - "!core/src/services/**"
- - "core/src/services/webhdfs/**"
- - ".github/workflows/service_test_webhdfs.yml"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
- cancel-in-progress: true
-
-jobs:
- webhdfs:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Configure Webhdfs
- # namenode will use ports: 9870 and 9000
- # datanode will use ports: 9864
- run: |
- docker run -d \
- --name namenode \
- --network host \
- -e CLUSTER_NAME=test \
- -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
- -e CORE_CONF_hadoop_http_staticuser_user=root \
- bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
-
- docker run -d \
- --name datanode \
- --network host \
- -e CLUSTER_NAME=test \
- -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
- -e CORE_CONF_hadoop_http_staticuser_user=root \
- bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
-
- curl --retry 30 --retry-delay 1 --retry-connrefused
http://localhost:9870
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
-
- - name: Test
- shell: bash
- working-directory: core
- run: cargo test behavior --features tests
- env:
- OPENDAL_TEST: webhdfs
- OPENDAL_WEBHDFS_ROOT: /
- OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870
-
- webhdfs_with_list_batch_disabled:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Configure Webhdfs
- # namenode will use ports: 9870 and 9000
- # datanode will use ports: 9864
- run: |
- docker run -d \
- --name namenode \
- --network host \
- -e CLUSTER_NAME=test \
- -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
- -e CORE_CONF_hadoop_http_staticuser_user=root \
- bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
-
- docker run -d \
- --name datanode \
- --network host \
- -e CLUSTER_NAME=test \
- -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
- -e CORE_CONF_hadoop_http_staticuser_user=root \
- bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
-
- curl --retry 30 --retry-delay 1 --retry-connrefused
http://localhost:9870
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
-
- - name: Test with disable_list_batch
- shell: bash
- working-directory: core
- run: cargo test behavior --features tests
- env:
- OPENDAL_TEST: webhdfs
- OPENDAL_WEBHDFS_ROOT: /
- OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870
- OPENDAL_WEBHDFS_DISABLE_LIST_BATCH: true
diff --git a/fixtures/webhdfs/docker-compose-webhdfs.yml
b/fixtures/webhdfs/docker-compose-webhdfs.yml
new file mode 100644
index 000000000..24f16a28a
--- /dev/null
+++ b/fixtures/webhdfs/docker-compose-webhdfs.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3.8'
+
+services:
+ namenode:
+ image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+ container_name: namenode
+ ports:
+ - 9870:9870
+ environment:
+ - CLUSTER_NAME=test
+ - WEBHDFS_CONF_dfs_webhdfs_enabled=true
+ - CORE_CONF_hadoop_http_staticuser_user=root
+
+ datanode:
+ image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+ container_name: datanode
+ hostname: localhost
+ ports:
+ - 9864:9864
+ environment:
+ - SERVICE_PRECONDITION=namenode:9870
+ - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
+ - CLUSTER_NAME=test
+ - WEBHDFS_CONF_dfs_webhdfs_enabled=true
+ - CORE_CONF_hadoop_http_staticuser_user=root
