This is an automated email from the ASF dual-hosted git repository. maxyang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 9ca5f55c9106c45dd9b3bd47f06a7b8e9819ec95 Author: liuxiaoyu <[email protected]> AuthorDate: Wed Dec 17 22:41:19 2025 +0800 feat: Add MinIO S3 support and improve Docker test environment - Configure S3/MinIO integration with PXF servers - Add Parquet profile support in pxf-profiles.xml - Update cloud access test cases to use MinIO protocol - Configure Gradle and Go to use aliyun mirrors for faster builds - Improve Docker entrypoint with MinIO deployment - Update test utilities to support protocol switching (s3/minio/hdfs) - Fix OutOfMemory test server configuration --- .../expected/query01.ans | 3 + .../sql/query01.sql | 3 + .../expected/query01.ans | 6 ++ .../sql/query01.sql | 6 ++ .../expected/query01.ans | 8 +- .../sql/query01.sql | 8 +- .../expected/query01.ans | 6 ++ .../sql/query01.sql | 6 ++ .../general/outOfMemory/expected/query01.ans | 9 +++ .../features/general/outOfMemory/sql/query01.sql | 9 +++ .../expected/query04.ans | 8 +- .../match_string_from_string_set/sql/query04.sql | 3 + .../automation/features/BaseWritableFeature.java | 8 +- .../docker/pxf-cbdb-dev/ubuntu/docker-compose.yml | 2 + .../docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh | 24 ++++++ .../pxf-cbdb-dev/ubuntu/script/entrypoint.sh | 86 +++++++++++++++++++++- .../docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh | 11 +++ concourse/singlecluster/Dockerfile | 13 ++++ dev/start_minio.bash | 43 ++++++++--- 19 files changed, 243 insertions(+), 19 deletions(-) diff --git a/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans b/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans index f0f8fcc3..05a98da6 100644 --- a/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans +++ b/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans @@ -15,6 +15,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE=s3:text&.*/pxf:\/\/pxf_automation_data?PROFILE=s3:text&ACCESS_AND_SECRET_KEY/ -- diff --git a/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql b/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql index ffa82957..683cba9b 100644 --- a/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql +++ b/automation/sqlrepo/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql @@ -15,6 +15,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE=s3:text&.*/pxf:\/\/pxf_automation_data?PROFILE=s3:text&ACCESS_AND_SECRET_KEY/ -- diff --git a/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans b/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans index 60420c32..a870f6fa 100644 --- a/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans +++ b/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans @@ -12,12 +12,18 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/default_s3/ -- s/default_s3/default/ -- diff --git a/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql b/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql index 09347239..1e0202c7 100644 --- a/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql +++ b/automation/sqlrepo/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql @@ -12,12 +12,18 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/default_s3/ -- s/default_s3/default/ -- diff --git a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans index 92428d9a..76e02e02 100644 --- a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans +++ b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans @@ -6,18 +6,24 @@ -- -- # create a match/subs -- --- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden).*/ +-- m/PXF server error.*(doesBucketExist|com.amazonaws).*/ -- s/PXF server error.*/PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/ -- -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/, file.*pxf_automation_data/ -- s/, file.*pxf_automation_data.*/pxf_automation_data/ -- diff --git a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql index f28f4e18..92309109 100644 --- a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql +++ b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql @@ -6,18 +6,24 @@ -- -- # create a match/subs -- --- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden).*/ +-- m/PXF server error.*(doesBucketExist|com.amazonaws).*/ -- s/PXF server error.*/PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/ -- -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/, file.*pxf_automation_data/ -- s/, file.*pxf_automation_data.*/pxf_automation_data/ -- diff --git a/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans b/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans index 0af50070..7ad4715e 100644 --- a/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans +++ b/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans @@ -15,12 +15,18 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/, file.*pxf_automation_data/ -- s/, file.*pxf_automation_data.*/pxf_automation_data/ -- diff --git a/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql b/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql index 9d09b486..ded78705 100644 --- a/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql +++ b/automation/sqlrepo/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql @@ -15,12 +15,18 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/CONTEXT:.*line.*/ +-- s/line \d* of //g +-- -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ -- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/, file.*pxf_automation_data/ -- s/, file.*pxf_automation_data.*/pxf_automation_data/ -- diff --git a/automation/sqlrepo/features/general/outOfMemory/expected/query01.ans b/automation/sqlrepo/features/general/outOfMemory/expected/query01.ans index 8f6348d5..b019b60a 100644 --- a/automation/sqlrepo/features/general/outOfMemory/expected/query01.ans +++ b/automation/sqlrepo/features/general/outOfMemory/expected/query01.ans @@ -3,6 +3,9 @@ -- -- # create a match/subs -- +-- m/PXF server error.*(java.lang.OutOfMemoryError|java.lang.RuntimeException: Class).*/ +-- s/PXF server error.*/PXF server error : java.lang.OutOfMemoryError: Java heap space/ +-- -- m/Check the PXF logs located in the.*/ -- s/Check the PXF logs located in the.*/Check the PXF logs located in the 'log' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details./ -- @@ -12,9 +15,15 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/Failed (to )?connect to/ -- s/Failed (to )?connect to.*/Failed to connect to server, must be down/ -- +-- m/, file.*pxf:\/\// +-- s/, file.*pxf:\/\//pxf:\/\// +-- -- end_matchsubs SELECT * from test_out_of_memory; ERROR: PXF server error : java.lang.OutOfMemoryError: Java heap space diff --git a/automation/sqlrepo/features/general/outOfMemory/sql/query01.sql b/automation/sqlrepo/features/general/outOfMemory/sql/query01.sql index d23eff51..b08ad3bb 100644 --- a/automation/sqlrepo/features/general/outOfMemory/sql/query01.sql +++ b/automation/sqlrepo/features/general/outOfMemory/sql/query01.sql @@ -3,6 +3,9 @@ -- -- # create a match/subs -- +-- m/PXF server error.*(java.lang.OutOfMemoryError|java.lang.RuntimeException: Class).*/ +-- s/PXF server error.*/PXF server error : java.lang.OutOfMemoryError: Java heap space/ +-- -- m/Check the PXF logs located in the.*/ -- s/Check the PXF logs located in the.*/Check the PXF logs located in the 'log' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details./ -- @@ -12,9 +15,15 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*External table.*/ +-- s/CONTEXT:.*External table.*// +-- -- m/Failed (to )?connect to/ -- s/Failed (to )?connect to.*/Failed to connect to server, must be down/ -- +-- m/, file.*pxf:\/\// +-- s/, file.*pxf:\/\//pxf:\/\// +-- -- end_matchsubs SELECT * from test_out_of_memory; diff --git a/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans b/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans index d65aeb74..1d24e93b 100644 --- a/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans +++ b/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans @@ -249,10 +249,10 @@ select * from hcfs_glob_match_string_from_string_set_10 order by name, num; -- -- end_matchsubs select * from hcfs_glob_match_string_from_string_set_11 order by name, num; -ERROR: PXF server error : Illegal file pattern: Unclosed group near index xxx -GP_IGNORE:-- start_ignore -GP_IGNORE:HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -GP_IGNORE:-- end_ignore +ERROR: PXF server error : Unclosed group near index xxx +GP_IGNORE:GP_IGNORE:-- start_ignore +GP_IGNORE:GP_IGNORE:HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. +GP_IGNORE:GP_IGNORE:-- end_ignore CONTEXT: External table hcfs_glob_match_string_from_string_set_11 -- }\{bc will match }{bc but it will not match }bc diff --git a/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql b/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql index 98d2aed3..845a31a3 100644 --- a/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql +++ b/automation/sqlrepo/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql @@ -52,6 +52,9 @@ select * from hcfs_glob_match_string_from_string_set_10 order by name, num; -- m/CONTEXT:.*file.*/ -- s/, file.*//g -- +-- m/CONTEXT:.*External table.*line \d* of pxf.*/ +-- s/, line \d* of pxf.*//g +-- -- end_matchsubs select * from hcfs_glob_match_string_from_string_set_11 order by name, num; diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/BaseWritableFeature.java b/automation/src/test/java/org/greenplum/pxf/automation/features/BaseWritableFeature.java index a5cb1a8b..fe5207ab 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/BaseWritableFeature.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/BaseWritableFeature.java @@ -18,7 +18,9 @@ public class BaseWritableFeature extends BaseFeature { @Override protected void beforeClass() throws Exception { super.beforeClass(); - hdfsWritePath = hdfs.getWorkingDirectory() + "/writable_results/"; + if (hdfs != null) { + hdfsWritePath = hdfs.getWorkingDirectory() + "/writable_results/"; + } } /** @@ -27,6 +29,8 @@ public class BaseWritableFeature extends BaseFeature { @Override protected void afterMethod() throws Exception { super.afterMethod(); - hdfs.removeDirectory(hdfsWritePath); + if (hdfs != null && hdfsWritePath != null) { + hdfs.removeDirectory(hdfsWritePath); + } } } diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml b/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml index b138a22e..f6d4d688 100644 --- a/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml @@ -14,6 +14,8 @@ services: hostname: mdw depends_on: - singlecluster + ports: + - "2222:22" volumes: - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh index 78ad67f2..b7d2028c 100755 --- a/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh @@ -13,9 +13,33 @@ sudo apt install -y openjdk-11-jdk maven cd /home/gpadmin/workspace/cloudberry-pxf +# mirror +# If the download fails, you can uncomment the line to switch to another mirror address. +# Configure Gradle to use Aliyun mirror +# mkdir -p ~/.gradle +# cat > ~/.gradle/init.gradle <<'EOF' +# allprojects { +# repositories { +# maven { url 'https://maven.aliyun.com/repository/public/' } +# maven { url 'https://maven.aliyun.com/repository/gradle-plugin' } +# mavenCentral() +# } +# buildscript { +# repositories { +# maven { url 'https://maven.aliyun.com/repository/public/' } +# maven { url 'https://maven.aliyun.com/repository/gradle-plugin' } +# mavenCentral() +# } +# } +# } +# EOF + # Set Go environment export GOPATH=$HOME/go export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin +# mirror +# If the download fails, you can uncomment the line to switch to another mirror address. +# export GOPROXY=https://goproxy.cn,direct mkdir -p $GOPATH export PXF_HOME=/usr/local/pxf sudo mkdir -p "$PXF_HOME" diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh index 146092d2..b180c10e 100755 --- a/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh @@ -87,7 +87,7 @@ build_cloudberry() { log "cleanup stale gpdemo data and PG locks" rm -rf /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs rm -f /tmp/.s.PGSQL.700* - sudo chown -R gpadmin:gpadmin "${ROOT_DIR}" || true + find "${ROOT_DIR}" -not -path '*/.git/*' -exec sudo chown gpadmin:gpadmin {} + 2>/dev/null || true "${PXF_SCRIPTS}/build_cloudberrry.sh" } @@ -122,6 +122,84 @@ XML if ! grep -q "pxf.service.user.name" "$PXF_BASE/servers/default-no-impersonation/pxf-site.xml"; then sed -i 's#</configuration># <property>\n <name>pxf.service.user.name</name>\n <value>foobar</value>\n </property>\n <property>\n <name>pxf.service.user.impersonation</name>\n <value>false</value>\n </property>\n</configuration>#' "$PXF_BASE/servers/default-no-impersonation/pxf-site.xml" fi + + # Configure pxf-profiles.xml for Parquet support + cat > "$PXF_BASE/conf/pxf-profiles.xml" <<'EOF' +<?xml version="1.0" encoding="UTF-8"?> +<profiles> + <profile> + <name>pxf:parquet</name> + <description>Profile for reading and writing Parquet files</description> + <plugins> + <fragmenter>org.greenplum.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.greenplum.pxf.plugins.hdfs.ParquetFileAccessor</accessor> + <resolver>org.greenplum.pxf.plugins.hdfs.ParquetResolver</resolver> + </plugins> + </profile> +</profiles> +EOF + + cat > "$PXF_HOME/conf/pxf-profiles.xml" <<'EOF' +<?xml version="1.0" encoding="UTF-8"?> +<profiles> + <profile> + <name>pxf:parquet</name> + <description>Profile for reading and writing Parquet files</description> + <plugins> + <fragmenter>org.greenplum.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.greenplum.pxf.plugins.hdfs.ParquetFileAccessor</accessor> + <resolver>org.greenplum.pxf.plugins.hdfs.ParquetResolver</resolver> + </plugins> + </profile> +</profiles> +EOF + + # Configure S3 settings + mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3" + + for s3_site in "$PXF_BASE/servers/s3/s3-site.xml" "$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do + mkdir -p "$(dirname "$s3_site")" + cat > "$s3_site" <<'EOF' +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + <property> + <name>fs.s3a.endpoint</name> + <value>http://localhost:9000</value> + </property> + <property> + <name>fs.s3a.access.key</name> + <value>admin</value> + </property> + <property> + <name>fs.s3a.secret.key</name> + <value>password</value> + </property> + <property> + <name>fs.s3a.path.style.access</name> + <value>true</value> + </property> + <property> + <name>fs.s3a.connection.ssl.enabled</name> + <value>false</value> + </property> + <property> + <name>fs.s3a.impl</name> + <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> + </property> + <property> + <name>fs.s3a.aws.credentials.provider</name> + <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value> + </property> +</configuration> +EOF + done + mkdir -p /home/gpadmin/.aws/ + cat > "/home/gpadmin/.aws/credentials" <<'EOF' +[default] +aws_access_key_id = admin +aws_secret_access_key = password +EOF + } prepare_hadoop_stack() { @@ -222,6 +300,11 @@ run_tests() { "${PXF_SCRIPTS}/run_tests.sh" "${GROUP:-}" } +deploy_minio() { + log "deploying MinIO" + bash "${REPO_DIR}/dev/start_minio.bash" +} + main() { detect_java_paths setup_locale_and_packages @@ -231,6 +314,7 @@ main() { build_pxf configure_pxf prepare_hadoop_stack + deploy_minio health_check #run_tests log "entrypoint finished; keeping container alive" diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh index 357a63df..54588516 100755 --- a/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh @@ -36,6 +36,17 @@ export COMMON_JAVA_OPTS=${COMMON_JAVA_OPTS:-} export PGHOST=${PGHOST:-localhost} export PGPORT=${PGPORT:-7000} export MASTER_DATA_DIRECTORY=${MASTER_DATA_DIRECTORY:-/home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1} +# set cloudberry timezone utc +export PGTZ=UTC + +# -------------------------------------------------------------------- +# Minio defaults +# -------------------------------------------------------------------- +export AWS_ACCESS_KEY_ID=admin +export AWS_SECRET_ACCESS_KEY=password +export PROTOCOL=minio +export ACCESS_KEY_ID=admin +export SECRET_ACCESS_KEY=password # -------------------------------------------------------------------- # PXF defaults diff --git a/concourse/singlecluster/Dockerfile b/concourse/singlecluster/Dockerfile index 5232f990..8e8c4621 100644 --- a/concourse/singlecluster/Dockerfile +++ b/concourse/singlecluster/Dockerfile @@ -85,6 +85,19 @@ RUN set -e; \ wget -O go.tgz -q "https://go.dev/dl/go1.23.3.linux-${go_arch}.tar.gz"; && \ sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go.tgz && rm go.tgz +# Install MinIO and mc +RUN set -e; \ + arch="${TARGETARCH:-$(uname -m)}"; \ + case "$arch" in \ + amd64|x86_64) minio_arch="amd64" ;; \ + arm64|aarch64) minio_arch="arm64" ;; \ + *) echo "Unsupported architecture: ${arch}"; exit 1 ;; \ + esac; \ + mkdir -p /home/gpadmin/workspace && \ + wget -O /home/gpadmin/workspace/minio "https://dl.min.io/server/minio/release/linux-${minio_arch}/minio"; && \ + wget -O /home/gpadmin/workspace/mc "https://dl.min.io/client/mc/release/linux-${minio_arch}/mc"; && \ + chmod +x /home/gpadmin/workspace/minio /home/gpadmin/workspace/mc + COPY ./templates $GPHD_ROOT COPY ./conf $GPHD_ROOT/conf diff --git a/dev/start_minio.bash b/dev/start_minio.bash index 764da02a..f9a43951 100755 --- a/dev/start_minio.bash +++ b/dev/start_minio.bash @@ -1,16 +1,39 @@ #!/bin/bash -echo 'Adding test bucket gpdb-ud-scratch to Minio ...' -sudo mkdir -p /opt/minio/data/gpdb-ud-scratch +set -e -export MINIO_ACCESS_KEY=admin -export MINIO_SECRET_KEY=password -echo "Minio credentials: accessKey=${MINIO_ACCESS_KEY} secretKey=${MINIO_SECRET_KEY}" +WORKSPACE_DIR=${WORKSPACE_DIR:-/home/gpadmin/workspace} +MINIO_BIN=${WORKSPACE_DIR}/minio +MC_BIN=${WORKSPACE_DIR}/mc +MINIO_DATA_DIR=${MINIO_DATA_DIR:-${WORKSPACE_DIR}/minio-data} +MINIO_PORT=${MINIO_PORT:-9000} +MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT:-9001} -echo 'Starting Minio ...' -sudo /opt/minio/bin/minio server /opt/minio/data & +export MINIO_ROOT_USER=${MINIO_ROOT_USER:-admin} +export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-password} + +echo "MinIO credentials: rootUser=${MINIO_ROOT_USER} rootPassword=${MINIO_ROOT_PASSWORD}" + +mkdir -p ${MINIO_DATA_DIR} + +echo "Starting MinIO server on port ${MINIO_PORT}..." +${MINIO_BIN} server ${MINIO_DATA_DIR} \ + --address ":${MINIO_PORT}" \ + --console-address ":${MINIO_CONSOLE_PORT}" & + +MINIO_PID=$! +echo "MinIO started with PID: ${MINIO_PID}" + +sleep 3 + +echo "Creating test bucket 'gpdb-ud-scratch'..." +${MC_BIN} alias set local http://localhost:${MINIO_PORT} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} +${MC_BIN} mb local/gpdb-ud-scratch --ignore-existing -# set variables used by automation export PROTOCOL=minio -export ACCESS_KEY_ID=${MINIO_ACCESS_KEY} -export SECRET_ACCESS_KEY=${MINIO_SECRET_KEY} +export ACCESS_KEY_ID=${MINIO_ROOT_USER} +export SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD} + +echo "MinIO is ready!" +echo " Console: http://localhost:${MINIO_CONSOLE_PORT}"; +echo " API: http://localhost:${MINIO_PORT}"; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
