This is an automated email from the ASF dual-hosted git repository. hoslo pushed a commit to branch fix-chainsafe-423 in repository https://gitbox.apache.org/repos/asf/opendal.git
commit d3fa53ec1e88245ccff02cf10b2c389883bcf1d4 Author: Xuanwo <[email protected]> AuthorDate: Mon Feb 5 11:35:12 2024 +0800 fix(services/chainsafe): fix 423 http status --- .github/services/hdfs/hdfs_default/action.yml | 46 +++++++++++++++++++++++++++ .github/workflows/service_test_hdfs.yml | 36 --------------------- core/src/services/chainsafe/error.rs | 4 +++ 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml new file mode 100644 index 0000000000..d4bdce0765 --- /dev/null +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: hdfs_default +description: 'Behavior test for hdfs default' + +runs: + using: "composite" + steps: + - name: Setup java env + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + - name: Setup + shell: bash + run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + export HADOOP_HOME="/home/runner/hadoop-3.3.5" + export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) + + cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + + cat << EOF >> $GITHUB_ENV + HADOOP_HOME=${HADOOP_HOME} + CLASSPATH=${CLASSPATH} + LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native + OPENDAL_HDFS_ROOT=/tmp/opendal/ + OPENDAL_HDFS_NAME_NODE=default + OPENDAL_HDFS_ENABLE_APPEND=false + EOF diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml index 1e1dae9f7e..c4447c2d45 100644 --- a/.github/workflows/service_test_hdfs.yml +++ b/.github/workflows/service_test_hdfs.yml @@ -37,42 +37,6 @@ concurrency: cancel-in-progress: true jobs: - hdfs-default: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - - name: Setup java env - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "11" - - name: Setup hadoop env - shell: bash - run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - - name: Test - shell: bash - working-directory: core - run: | - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${{ env.HADOOP_HOME }}/etc/hadoop/hdfs-site.xml - - cargo test behavior --features tests,services-hdfs - env: - HADOOP_HOME: "/home/runner/hadoop-3.3.5" - OPENDAL_TEST: hdfs - OPENDAL_HDFS_ROOT: /tmp/opendal/ - OPENDAL_HDFS_NAME_NODE: default - OPENDAL_HDFS_ENABLE_APPEND: false - hdfs-cluster: runs-on: ubuntu-latest steps: diff --git a/core/src/services/chainsafe/error.rs b/core/src/services/chainsafe/error.rs index 5df3cebb3c..f998b625a4 100644 --- a/core/src/services/chainsafe/error.rs +++ b/core/src/services/chainsafe/error.rs @@ -46,6 +46,10 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> { 401 | 403 => (ErrorKind::PermissionDenied, false), 404 => (ErrorKind::NotFound, false), 304 | 412 => (ErrorKind::ConditionNotMatch, false), + // https://github.com/apache/opendal/issues/4146 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423 + // We should retry it when we get 423 error. + 423 => (ErrorKind::RateLimited, true), // Service like Upyun could return 499 error with a message like: // Client Disconnect, we should retry it. 499 => (ErrorKind::Unexpected, true),
