This is an automated email from the ASF dual-hosted git repository. hoslo pushed a commit to branch fix-chainsafe-423 in repository https://gitbox.apache.org/repos/asf/opendal.git
commit ff25333b9718b8b9beba2d5143f1760bf174a47a Author: Xuanwo <[email protected]> AuthorDate: Mon Feb 5 11:35:12 2024 +0800 fix(services/chainsafe): fix 423 http status --- .github/scripts/test_behavior/plan.py | 4 +++ .github/services/hdfs/hdfs_default/action.yml | 46 +++++++++++++++++++++++++++ .github/workflows/service_test_hdfs.yml | 36 --------------------- core/src/services/chainsafe/error.rs | 4 +++ 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/.github/scripts/test_behavior/plan.py b/.github/scripts/test_behavior/plan.py index 66a44b13e7..164d50c77b 100755 --- a/.github/scripts/test_behavior/plan.py +++ b/.github/scripts/test_behavior/plan.py @@ -198,6 +198,10 @@ def generate_language_binding_cases( ) -> list[dict[str, str]]: cases = unique_cases(cases) + # Remove hdfs cases for java. + if language == "java": + cases = [v for v in cases if v["service"] != "hdfs"] + if os.getenv("GITHUB_IS_PUSH") == "true": return cases diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml new file mode 100644 index 0000000000..d4bdce0765 --- /dev/null +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: hdfs_default +description: 'Behavior test for hdfs default' + +runs: + using: "composite" + steps: + - name: Setup java env + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + - name: Setup + shell: bash + run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + export HADOOP_HOME="/home/runner/hadoop-3.3.5" + export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) + + cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + + cat << EOF >> $GITHUB_ENV + HADOOP_HOME=${HADOOP_HOME} + CLASSPATH=${CLASSPATH} + LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native + OPENDAL_HDFS_ROOT=/tmp/opendal/ + OPENDAL_HDFS_NAME_NODE=default + OPENDAL_HDFS_ENABLE_APPEND=false + EOF diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml index 1e1dae9f7e..c4447c2d45 100644 --- a/.github/workflows/service_test_hdfs.yml +++ b/.github/workflows/service_test_hdfs.yml @@ -37,42 +37,6 @@ concurrency: cancel-in-progress: true jobs: - hdfs-default: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - - name: Setup java env - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "11" - - name: Setup hadoop env - shell: bash - run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - - name: Test - shell: bash - working-directory: core - run: | - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${{ env.HADOOP_HOME }}/etc/hadoop/hdfs-site.xml - - cargo test behavior --features tests,services-hdfs - env: - HADOOP_HOME: "/home/runner/hadoop-3.3.5" - OPENDAL_TEST: hdfs - OPENDAL_HDFS_ROOT: /tmp/opendal/ - OPENDAL_HDFS_NAME_NODE: default - OPENDAL_HDFS_ENABLE_APPEND: false - hdfs-cluster: runs-on: ubuntu-latest steps: diff --git a/core/src/services/chainsafe/error.rs b/core/src/services/chainsafe/error.rs index 5df3cebb3c..f998b625a4 100644 --- a/core/src/services/chainsafe/error.rs +++ b/core/src/services/chainsafe/error.rs @@ -46,6 +46,10 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> { 401 | 403 => (ErrorKind::PermissionDenied, false), 404 => (ErrorKind::NotFound, false), 304 | 412 => (ErrorKind::ConditionNotMatch, false), + // https://github.com/apache/opendal/issues/4146 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423 + // We should retry it when we get 423 error. + 423 => (ErrorKind::RateLimited, true), // Service like Upyun could return 499 error with a message like: // Client Disconnect, we should retry it. 499 => (ErrorKind::Unexpected, true),
