This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch planner in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ae4b5592c47933227f937f1c666f5bb775b689dd Author: Xuanwo <[email protected]> AuthorDate: Wed Oct 18 16:22:24 2023 +0800 migrate existing s3 tests Signed-off-by: Xuanwo <[email protected]> --- .github/services/s3/aws_s3_with_sse_c/action.yml | 8 +- .github/services/s3/minio_s3/action.yml | 44 ++++++++++ .../services/s3/minio_s3_with_anonymous/action.yml | 50 +++++++++++ .../s3/{aws_s3_with_sse_c => r2}/action.yml | 23 ++--- .github/workflows/service_test_s3.yml | 99 ---------------------- 5 files changed, 111 insertions(+), 113 deletions(-) diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml index c28416a44..507fc4922 100644 --- a/.github/services/s3/aws_s3_with_sse_c/action.yml +++ b/.github/services/s3/aws_s3_with_sse_c/action.yml @@ -36,6 +36,8 @@ runs: - name: Add extra settings shell: bash run: - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256" >> $GITHUB_ENV - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=" >> $GITHUB_ENV - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw==" >> $GITHUB_ENV + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256 + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw== + EOF diff --git a/.github/services/s3/minio_s3/action.yml b/.github/services/s3/minio_s3/action.yml new file mode 100644 index 000000000..e74a63449 --- /dev/null +++ b/.github/services/s3/minio_s3/action.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: minio_s3 +description: 'Behavior test for Minio S3.' + +runs: + using: "composite" + steps: + - name: Setup MinIO Server + shell: bash + working-directory: fixtures/s3 + run: docker-compose -f docker-compose-minio.yml up -d + - name: Setup test bucket + shell: bash + env: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + AWS_EC2_METADATA_DISABLED: "true" + run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test + - name: Setup + shell: bash + run: + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_BUCKET=test + OPENDAL_S3_ENDPOINT="http://127.0.0.1:9000"; + OPENDAL_S3_ACCESS_KEY_ID=minioadmin + OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin + OPENDAL_S3_REGION=us-east-1 + EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml new file mode 100644 index 000000000..99308dd77 --- /dev/null +++ b/.github/services/s3/minio_s3_with_anonymous/action.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: minio_s3_with_anonymous +description: 'Behavior test for Minio S3 with anonymous access.' + +runs: + using: "composite" + steps: + - name: Setup MinIO Server + shell: bash + working-directory: fixtures/s3 + run: | + docker-compose -f docker-compose-minio.yml up -d + - name: Setup test bucket + shell: bash + env: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + AWS_EC2_METADATA_DISABLED: "true" + run: | + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test + + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin + ./mc anonymous set public local/test + - name: Setup + shell: bash + run: + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_BUCKET=test + OPENDAL_S3_ENDPOINT="http://127.0.0.1:9000"; + OPENDAL_S3_REGION=us-east-1 + OPENDAL_S3_ALLOW_ANONYMOUS=on + EOF diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/r2/action.yml similarity index 56% copy from .github/services/s3/aws_s3_with_sse_c/action.yml copy to .github/services/s3/r2/action.yml index c28416a44..cfbd1e4fd 100644 --- a/.github/services/s3/aws_s3_with_sse_c/action.yml +++ b/.github/services/s3/r2/action.yml @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -name: aws_s3_with_sse_c -description: 'Behavior test for AWS S3 with SSE-C. This service is sponsored by @datafuse_labs.' +name: r2 +description: 'Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo.' runs: using: "composite" @@ -26,16 +26,17 @@ runs: with: export-env: true env: - OPENDAL_S3_ROOT: op://services/s3/root - OPENDAL_S3_BUCKET: op://services/s3/bucket - OPENDAL_S3_ENDPOINT: op://services/s3/endpoint - OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id - OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key - OPENDAL_S3_REGION: op://services/s3/region + OPENDAL_S3_BUCKET: op://services/r2/bucket + OPENDAL_S3_ENDPOINT: op://services/r2/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key + # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation + # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information - name: Add extra settings shell: bash run: - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256" >> $GITHUB_ENV - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=" >> $GITHUB_ENV - echo "OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw==" >> $GITHUB_ENV + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_REGION=auto + OPENDAL_S3_BATCH_MAX_OPERATIONS=700 + EOF diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 0a42c0b7c..27d493bc7 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -71,105 +71,6 @@ jobs: OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing OPENDAL_S3_REGION: ap-northeast-1 - minio_s3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup MinIO Server - shell: bash - working-directory: fixtures/s3 - run: docker-compose -f docker-compose-minio.yml up -d - - name: Setup test bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"; - OPENDAL_S3_ACCESS_KEY_ID: minioadmin - OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin - OPENDAL_S3_REGION: us-east-1 - - anonymous_minio_s3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup MinIO Server - shell: bash - working-directory: fixtures/s3 - run: | - docker-compose -f docker-compose-minio.yml up -d - - name: Setup test bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: | - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin - ./mc anonymous set public local/test - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"; - OPENDAL_S3_ALLOW_ANONYMOUS: on - OPENDAL_S3_REGION: us-east-1 - - r2: - runs-on: ubuntu-latest - if: (github.event_name == 'push' && github.repository == 'apache/incubator-opendal') || !github.event.pull_request.head.repo.fork - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - - name: Load secret - uses: 1password/load-secrets-action@v1 - with: - export-env: true - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_S3_BUCKET: op://services/r2/bucket - OPENDAL_S3_ENDPOINT: op://services/r2/endpoint - OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id - OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key - - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_REGION: auto - # This is the R2's limitation - # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information - OPENDAL_S3_BATCH_MAX_OPERATIONS: 700 - java: runs-on: ubuntu-latest steps:
