This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch polish-sftp
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/polish-sftp by this push:
new d37e3e126 chore(test): Fix sftp test failed in python (#3436)
d37e3e126 is described below
commit d37e3e1269b08912c6719b28c93a3e54cbad5a93
Author: Nadeshiko Manju <[email protected]>
AuthorDate: Wed Nov 1 00:20:28 2023 +0800
chore(test): Fix sftp test failed in python (#3436)
---
.github/scripts/behavior_test/plan.py | 5 -----
.github/services/azblob/azurite_azblob/action.yml | 2 +-
.github/services/etcd/etcd-cluster/action.yml | 2 +-
.github/services/etcd/etcd-tls/action.yml | 2 +-
.github/services/etcd/etcd/action.yml | 2 +-
.github/services/http/caddy/action.yml | 2 +-
.github/services/http/nginx/action.yml | 2 +-
.github/services/memcached/memcached/action.yml | 2 +-
.../mongodb/mongodb_with_basic_auth/action.yml | 2 +-
.../mongodb/mongodb_with_no_auth/action.yml | 2 +-
.github/services/mysql/mysql/action.yml | 2 +-
.github/services/postgresql/postgresql/action.yml | 2 +-
.github/services/redis/dragonfly/action.yml | 2 +-
.github/services/redis/kvrocks/action.yml | 2 +-
.github/services/redis/redis/action.yml | 2 +-
.../services/redis/redis_with_cluster/action.yml | 2 +-
.github/services/s3/minio_s3/action.yml | 2 +-
.../services/s3/minio_s3_with_anonymous/action.yml | 2 +-
.github/services/sftp/sftp/action.yml | 3 ++-
.../sftp/sftp_with_default_root/action.yml | 3 ++-
.../sftp/docker-compose-sftp-with-default-root.yml | 15 ++++++++++++++-
fixtures/sftp/docker-compose-sftp.yml | 15 ++++++++++++++-
fixtures/sftp/health-check.sh | 22 ++++++++++++++++++++++
23 files changed, 71 insertions(+), 26 deletions(-)
diff --git a/.github/scripts/behavior_test/plan.py
b/.github/scripts/behavior_test/plan.py
index e734a3d88..c7de32140 100755
--- a/.github/scripts/behavior_test/plan.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -214,11 +214,6 @@ def generate_binding_python_cases(
) -> list[dict[str, str]]:
cases = unique_cases(cases)
- # REMOVE ME after https://github.com/apache/incubator-opendal/issues/3429
addressed.
- #
- # Sftp can't pass on python bindings, remove it for now.
- cases = [v for v in cases if v["service"] != "sftp"]
-
if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
diff --git a/.github/services/azblob/azurite_azblob/action.yml
b/.github/services/azblob/azurite_azblob/action.yml
index ac1c1abed..b5e255c02 100644
--- a/.github/services/azblob/azurite_azblob/action.yml
+++ b/.github/services/azblob/azurite_azblob/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup azurite azblob service
shell: bash
working-directory: fixtures/azblob
- run: docker-compose -f docker-compose-azurite.yml up -d
+ run: docker compose -f docker-compose-azurite.yml up -d --wait
--wait-timeout 20
- name: Setup test bucket
shell: bash
run: |
diff --git a/.github/services/etcd/etcd-cluster/action.yml
b/.github/services/etcd/etcd-cluster/action.yml
index bf7e5886e..0a1e68ea7 100644
--- a/.github/services/etcd/etcd-cluster/action.yml
+++ b/.github/services/etcd/etcd-cluster/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup etcd cluster
shell: bash
working-directory: fixtures/etcd
- run: docker-compose -f docker-compose-cluster.yml up -d
+ run: docker compose -f docker-compose-cluster.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/etcd/etcd-tls/action.yml
b/.github/services/etcd/etcd-tls/action.yml
index 8402ac90d..c72b70d2c 100644
--- a/.github/services/etcd/etcd-tls/action.yml
+++ b/.github/services/etcd/etcd-tls/action.yml
@@ -30,7 +30,7 @@ runs:
- name: Setup etcd tls
shell: bash
working-directory: fixtures/etcd
- run: docker-compose -f docker-compose-standalone-tls.yml up -d
+ run: docker compose -f docker-compose-standalone-tls.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/etcd/etcd/action.yml
b/.github/services/etcd/etcd/action.yml
index cb810a9ea..9f0a1eeb6 100644
--- a/.github/services/etcd/etcd/action.yml
+++ b/.github/services/etcd/etcd/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup etcd cluster
shell: bash
working-directory: fixtures/etcd
- run: docker-compose -f docker-compose-standalone.yml up -d
+ run: docker compose -f docker-compose-standalone.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/http/caddy/action.yml
b/.github/services/http/caddy/action.yml
index aeb887786..374d6e7c9 100644
--- a/.github/services/http/caddy/action.yml
+++ b/.github/services/http/caddy/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup http in caddy
shell: bash
working-directory: fixtures/http
- run: docker-compose -f docker-compose-caddy.yml up -d
+ run: docker compose -f docker-compose-caddy.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/http/nginx/action.yml
b/.github/services/http/nginx/action.yml
index 2dba9a28b..de95c3611 100644
--- a/.github/services/http/nginx/action.yml
+++ b/.github/services/http/nginx/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup http in nginx
shell: bash
working-directory: fixtures/http
- run: docker-compose -f docker-compose-nginx.yml up -d
+ run: docker compose -f docker-compose-nginx.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/memcached/memcached/action.yml
b/.github/services/memcached/memcached/action.yml
index 360de1d79..c6c0ca457 100644
--- a/.github/services/memcached/memcached/action.yml
+++ b/.github/services/memcached/memcached/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup memcached server
shell: bash
working-directory: fixtures/memcached
- run: docker-compose -f docker-compose-memcached.yml up -d
+ run: docker compose -f docker-compose-memcached.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/mongodb/mongodb_with_basic_auth/action.yml
b/.github/services/mongodb/mongodb_with_basic_auth/action.yml
index c1517dd3a..17cd74204 100644
--- a/.github/services/mongodb/mongodb_with_basic_auth/action.yml
+++ b/.github/services/mongodb/mongodb_with_basic_auth/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup MongoDB Server
shell: bash
working-directory: fixtures/mongodb
- run: docker-compose -f docker-compose-basic-auth.yml up -d
+ run: docker compose -f docker-compose-basic-auth.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/mongodb/mongodb_with_no_auth/action.yml
b/.github/services/mongodb/mongodb_with_no_auth/action.yml
index 8d885bb5e..4f197b655 100644
--- a/.github/services/mongodb/mongodb_with_no_auth/action.yml
+++ b/.github/services/mongodb/mongodb_with_no_auth/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup MongoDB Server
shell: bash
working-directory: fixtures/mongodb
- run: docker-compose -f docker-compose-no-auth.yml up -d
+ run: docker compose -f docker-compose-no-auth.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/mysql/mysql/action.yml
b/.github/services/mysql/mysql/action.yml
index a41e7d2be..f8eb9d250 100644
--- a/.github/services/mysql/mysql/action.yml
+++ b/.github/services/mysql/mysql/action.yml
@@ -26,7 +26,7 @@ runs:
working-directory: fixtures/mysql
run: |
apt update && apt install -y mysql-client
- docker-compose -f docker-compose.yml up -d
+ docker compose -f docker-compose.yml up -d --wait --wait-timeout 20
while ! mysql -h localhost -P 3306 --protocol=tcp -u root -p'password'
-e "SELECT 1"; do
echo "Waiting for MySQL..."
sleep 1
diff --git a/.github/services/postgresql/postgresql/action.yml
b/.github/services/postgresql/postgresql/action.yml
index 8fd495054..527b43ec7 100644
--- a/.github/services/postgresql/postgresql/action.yml
+++ b/.github/services/postgresql/postgresql/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup PostgreSQL Server
shell: bash
working-directory: fixtures/postgresql
- run: docker-compose -f docker-compose.yml up -d
+ run: docker compose -f docker-compose.yml up -d --wait --wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/redis/dragonfly/action.yml
b/.github/services/redis/dragonfly/action.yml
index 71e271f5c..f6ef0ad9a 100644
--- a/.github/services/redis/dragonfly/action.yml
+++ b/.github/services/redis/dragonfly/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup Dragonfly Server
shell: bash
working-directory: fixtures/redis
- run: docker-compose -f docker-compose-dragonfly.yml up -d
+ run: docker compose -f docker-compose-dragonfly.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/redis/kvrocks/action.yml
b/.github/services/redis/kvrocks/action.yml
index 64978bf44..ab8d01d68 100644
--- a/.github/services/redis/kvrocks/action.yml
+++ b/.github/services/redis/kvrocks/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup Kvrocks Server
shell: bash
working-directory: fixtures/redis
- run: docker-compose -f docker-compose-kvrocks.yml up -d
+ run: docker compose -f docker-compose-kvrocks.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/redis/redis/action.yml
b/.github/services/redis/redis/action.yml
index cf3ac0071..b7198cd6a 100644
--- a/.github/services/redis/redis/action.yml
+++ b/.github/services/redis/redis/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup Redis Server
shell: bash
working-directory: fixtures/redis
- run: docker-compose -f docker-compose-redis.yml up -d
+ run: docker compose -f docker-compose-redis.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/redis/redis_with_cluster/action.yml
b/.github/services/redis/redis_with_cluster/action.yml
index fba13bb1e..7fd453b92 100644
--- a/.github/services/redis/redis_with_cluster/action.yml
+++ b/.github/services/redis/redis_with_cluster/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup Redis Cluster
shell: bash
working-directory: fixtures/redis
- run: docker-compose -f docker-compose-redis-cluster.yml up -d
+ run: docker compose -f docker-compose-redis-cluster.yml up -d --wait
--wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/s3/minio_s3/action.yml
b/.github/services/s3/minio_s3/action.yml
index ed237bb37..f9414aeee 100644
--- a/.github/services/s3/minio_s3/action.yml
+++ b/.github/services/s3/minio_s3/action.yml
@@ -24,7 +24,7 @@ runs:
- name: Setup MinIO Server
shell: bash
working-directory: fixtures/s3
- run: docker-compose -f docker-compose-minio.yml up -d
+ run: docker compose -f docker-compose-minio.yml up -d --wait
--wait-timeout 20
- name: Setup test bucket
shell: bash
env:
diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml
b/.github/services/s3/minio_s3_with_anonymous/action.yml
index 407289d9c..3c6093bae 100644
--- a/.github/services/s3/minio_s3_with_anonymous/action.yml
+++ b/.github/services/s3/minio_s3_with_anonymous/action.yml
@@ -25,7 +25,7 @@ runs:
shell: bash
working-directory: fixtures/s3
run: |
- docker-compose -f docker-compose-minio.yml up -d
+ docker compose -f docker-compose-minio.yml up -d --wait --wait-timeout
20
- name: Setup test bucket
shell: bash
env:
diff --git a/.github/services/sftp/sftp/action.yml
b/.github/services/sftp/sftp/action.yml
index 3caab8e87..cc529269b 100644
--- a/.github/services/sftp/sftp/action.yml
+++ b/.github/services/sftp/sftp/action.yml
@@ -24,7 +24,8 @@ runs:
- name: Setup Sftp
shell: bash
working-directory: fixtures/sftp
- run: docker-compose -f docker-compose-sftp.yml up -d
+ run: |
+ docker compose -f docker-compose-sftp.yml up -d --wait --wait-timeout
20
- name: Setup
shell: bash
run: |
diff --git a/.github/services/sftp/sftp_with_default_root/action.yml
b/.github/services/sftp/sftp_with_default_root/action.yml
index 1371bb4e1..ec538c953 100644
--- a/.github/services/sftp/sftp_with_default_root/action.yml
+++ b/.github/services/sftp/sftp_with_default_root/action.yml
@@ -24,7 +24,8 @@ runs:
- name: Setup Sftp
shell: bash
working-directory: fixtures/sftp
- run: docker-compose -f docker-compose-sftp-with-default-root.yml up -d
+ run: |
+ docker compose -f docker-compose-sftp-with-default-root.yml up -d
--wait --wait-timeout 20
- name: Setup
shell: bash
run: |
diff --git a/fixtures/sftp/docker-compose-sftp-with-default-root.yml
b/fixtures/sftp/docker-compose-sftp-with-default-root.yml
index 6e12002eb..a02e91ad8 100644
--- a/fixtures/sftp/docker-compose-sftp-with-default-root.yml
+++ b/fixtures/sftp/docker-compose-sftp-with-default-root.yml
@@ -19,14 +19,27 @@ version: '3.8'
services:
sftp-with-default-root:
- image: atmoz/sftp
+ image: atmoz/sftp:debian
ports:
- "2222:22"
volumes:
- ./change_root_dir.sh:/etc/sftp.d/change_root_dir
- ./test_ssh_key.pub:/home/foo/.ssh/keys/id_rsa.pub:ro
+ - ./test_ssh_key:/home/foo/.ssh/keys/id_rsa:ro
+ - ./health-check.sh:/health-check.sh
ulimits:
nofile:
soft: 65536
hard: 65536
command: foo::::upload
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "bash",
+ "-c",
+ "/health-check.sh"
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/fixtures/sftp/docker-compose-sftp.yml
b/fixtures/sftp/docker-compose-sftp.yml
index 804f6c97f..d090565ae 100644
--- a/fixtures/sftp/docker-compose-sftp.yml
+++ b/fixtures/sftp/docker-compose-sftp.yml
@@ -19,13 +19,26 @@ version: '3.8'
services:
sftp:
- image: atmoz/sftp
+ image: atmoz/sftp:debian
ports:
- "2222:22"
volumes:
- ./test_ssh_key.pub:/home/foo/.ssh/keys/id_rsa.pub:ro
+ - ./test_ssh_key:/home/foo/.ssh/keys/id_rsa:ro
+ - ./health-check.sh:/health-check.sh
ulimits:
nofile:
soft: 65536
hard: 65536
command: foo::::upload
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "bash",
+ "-c",
+ "/health-check.sh"
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/fixtures/sftp/health-check.sh b/fixtures/sftp/health-check.sh
new file mode 100755
index 000000000..da7332240
--- /dev/null
+++ b/fixtures/sftp/health-check.sh
@@ -0,0 +1,22 @@
+# !/bin/bash
+
+set +ex
+username="foo"
+server="127.0.0.1"
+port=22
+know_hosts_strategy="no"
+identity_file="/home/foo/.ssh/keys/id_rsa"
+while :; do
+ (
+ sftp -oPort=$port -o StrictHostKeyChecking=$know_hosts_strategy -o
IdentityFile=$identity_file $username@$server << EOF
+ bye
+EOF
+ ) 2>/dev/null
+ if [[ $? -eq 0 ]]; then
+ echo "SFTP is available, proceeding..."
+ break
+ else
+ echo "Waiting for SFTP to be available..."
+ sleep 1
+ fi
+done
\ No newline at end of file
