This is an automated email from the ASF dual-hosted git repository. gxd pushed a commit to branch webhdfs_test_planner in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit a682fdd21b67a28a199bd8b975328f642ee24641 Author: G-XD <[email protected]> AuthorDate: Tue Nov 14 00:16:59 2023 +0800 refactor(services/webhdfs): migrate to test planner --- .github/services/webhdfs/webhdfs/action.yml | 35 ++++++ .../webhdfs_with_list_batch_disabled/action.yml | 36 +++++++ .github/workflows/service_test_webhdfs.yml | 117 --------------------- fixtures/webhdfs/docker-compose-webhdfs.yml | 42 ++++++++ 4 files changed, 113 insertions(+), 117 deletions(-) diff --git a/.github/services/webhdfs/webhdfs/action.yml b/.github/services/webhdfs/webhdfs/action.yml new file mode 100644 index 000000000..ae9cdfa73 --- /dev/null +++ b/.github/services/webhdfs/webhdfs/action.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: webhdfs +description: 'Behavior test for webhdfs' + +runs: + using: "composite" + steps: + - name: Setup webhdfs + shell: bash + working-directory: fixtures/webhdfs + run: | + docker compose -f docker-compose-webhdfs.yml up -d --wait + - name: Setup + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_WEBHDFS_ROOT=/ + OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870 + EOF diff --git a/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml b/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml new file mode 100644 index 000000000..7fb294b04 --- /dev/null +++ b/.github/services/webhdfs/webhdfs_with_list_batch_disabled/action.yml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: webhdfs_with_list_batch_disabled +description: 'Behavior test for webhdfs with list batch disabled' + +runs: + using: "composite" + steps: + - name: Setup webhdfs + shell: bash + working-directory: fixtures/webhdfs + run: | + docker compose -f docker-compose-webhdfs.yml up -d --wait + - name: Setup + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_WEBHDFS_ROOT=/ + OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870 + OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=true + EOF diff --git a/.github/workflows/service_test_webhdfs.yml b/.github/workflows/service_test_webhdfs.yml deleted file mode 100644 index 4db1f3a29..000000000 --- a/.github/workflows/service_test_webhdfs.yml +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Service Test WebHDFS - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "core/src/**" - - "core/tests/**" - - "!core/src/docs/**" - - "!core/src/services/**" - - "core/src/services/webhdfs/**" - - ".github/workflows/service_test_webhdfs.yml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - webhdfs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Configure Webhdfs - # namenode will use ports: 9870 and 9000 - # datanode will use ports: 9864 - run: | - docker run -d \ - --name namenode \ - --network host \ - -e CLUSTER_NAME=test \ - -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ - -e CORE_CONF_hadoop_http_staticuser_user=root \ - bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 - - docker run -d \ - --name datanode \ - --network host \ - -e CLUSTER_NAME=test \ - -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ - -e CORE_CONF_hadoop_http_staticuser_user=root \ - bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 - - curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Test - shell: bash - working-directory: core - run: cargo test behavior --features tests - env: - OPENDAL_TEST: webhdfs - OPENDAL_WEBHDFS_ROOT: / - OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870 - - webhdfs_with_list_batch_disabled: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Configure Webhdfs - # namenode will use ports: 9870 and 9000 - # datanode will use ports: 9864 - run: | - docker run -d \ - --name namenode \ - --network host \ - -e CLUSTER_NAME=test \ - -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ - -e CORE_CONF_hadoop_http_staticuser_user=root \ - bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 - - docker run -d \ - --name datanode \ - --network host \ - -e CLUSTER_NAME=test \ - -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ - -e CORE_CONF_hadoop_http_staticuser_user=root \ - bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 - - curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Test with disable_list_batch - shell: bash - working-directory: core - run: cargo test behavior --features tests - env: - OPENDAL_TEST: webhdfs - OPENDAL_WEBHDFS_ROOT: / - OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870 - OPENDAL_WEBHDFS_DISABLE_LIST_BATCH: true diff --git a/fixtures/webhdfs/docker-compose-webhdfs.yml b/fixtures/webhdfs/docker-compose-webhdfs.yml new file mode 100644 index 000000000..24f16a28a --- /dev/null +++ b/fixtures/webhdfs/docker-compose-webhdfs.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.8' + +services: + namenode: + image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 + container_name: namenode + ports: + - 9870:9870 + environment: + - CLUSTER_NAME=test + - WEBHDFS_CONF_dfs_webhdfs_enabled=true + - CORE_CONF_hadoop_http_staticuser_user=root + + datanode: + image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 + container_name: datanode + hostname: localhost + ports: + - 9864:9864 + environment: + - SERVICE_PRECONDITION=namenode:9870 + - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 + - CLUSTER_NAME=test + - WEBHDFS_CONF_dfs_webhdfs_enabled=true + - CORE_CONF_hadoop_http_staticuser_user=root
